From ec1452fd1cdb9cb1db9b8bad872916d3213489e2 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Thu, 14 May 2020 21:36:31 +0200
Subject: [PATCH 001/208] Pleroma.MIME: use gen_magic

---
 .gitlab-ci.yml                       |  2 +
 CHANGELOG.md                         |  1 +
 config/config.exs                    |  2 +
 config/description.exs               | 14 +++++
 docs/installation/alpine_linux_en.md |  1 +
 docs/installation/arch_linux_en.md   |  1 +
 docs/installation/debian_based_en.md |  1 +
 docs/installation/gentoo_en.md       |  1 +
 docs/installation/otp_en.md          |  5 +-
 lib/pleroma/application.ex           |  1 +
 lib/pleroma/mime.ex                  | 84 ++++++++--------------------
 mix.exs                              |  1 +
 mix.lock                             |  2 +
 13 files changed, 52 insertions(+), 64 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aad28a2d8..685106969 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,6 +24,8 @@ stages:
 before_script:
   - mix local.hex --force
   - mix local.rebar --force
+  - apt-get -qq update
+  - apt-get install -y libmagic-dev
 
 build:
   stage: build
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2629bf84..8925f31f6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Changed
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard
+- **Breaking** Requires `libmagic` (or `file`) to guess file types.
 <details>
   <summary>API Changes</summary>
 - **Breaking:** Emoji API: changed methods and renamed routes.
diff --git a/config/config.exs b/config/config.exs
index 6a7bb9e06..65aa8643e 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -678,6 +678,8 @@
     timeout: 300_000
   ]
 
+config :pleroma, :gen_magic_pool, size: 2
+
 config :pleroma, :restrict_unauthenticated,
   timelines: %{local: false, federated: false},
   profiles: %{local: false, remote: false},
diff --git a/config/description.exs b/config/description.exs
index b21d7840c..a318afbe6 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3325,5 +3325,19 @@
         suggestions: [false]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: :gen_magic_pool,
+    type: :group,
+    description: "GenMagic/libmagic configuration",
+    children: [
+      %{
+        key: :size,
+        type: :integer,
+        description: "Number of gen_magic workers to start.",
+        suggestions: [2]
+      }
+    ]
   }
 ]
diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md
index 2a9b8f6ff..1d503ff9f 100644
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@@ -13,6 +13,7 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-parsetools`
 * `erlang-xmerl`
 * `git`
+* `file-dev`
 * Development Tools
 
 #### Optional packages used in this guide
diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md
index 8370986ad..428ed6bd7 100644
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@@ -9,6 +9,7 @@ This guide will assume that you have administrative rights, either as root or a
 * `elixir`
 * `git`
 * `base-devel`
+* `file`
 
 #### Optional packages used in this guide
 
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 2c20d521a..21e89d1a0 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -10,6 +10,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
 * `erlang-dev`
 * `erlang-nox`
+* `libmagic-dev`
 * `git`
 * `build-essential`
 
diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md
index 1e61373cc..e936469d0 100644
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@@ -28,6 +28,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-db/postgresql`
 * `dev-lang/elixir`
 * `dev-vcs/git`
+* `sys-apps/file`
 
 #### Optional ebuilds used in this guide
 
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 86135cd20..0cad29bc8 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -27,15 +27,16 @@ Other than things bundled in the OTP release Pleroma depends on:
 * PostgreSQL (also utilizes extensions in postgresql-contrib)
 * nginx (could be swapped with another reverse proxy but this guide covers only it)
 * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
+* libmagic/file
 
 ```sh tab="Alpine"
 echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
 apk update
-apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
+apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot libmagic
 ```
 
 ```sh tab="Debian/Ubuntu"
-apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic
 ```
 
 ## Setup
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9d3d92b38..c74255629 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -80,6 +80,7 @@ def start(_type, _args) do
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
+          Pleroma.MIME,
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
diff --git a/lib/pleroma/mime.ex b/lib/pleroma/mime.ex
index 6ee055f50..3b406630e 100644
--- a/lib/pleroma/mime.ex
+++ b/lib/pleroma/mime.ex
@@ -6,8 +6,21 @@ defmodule Pleroma.MIME do
   @moduledoc """
   Returns the mime-type of a binary and optionally a normalized file-name.
   """
-  @default "application/octet-stream"
   @read_bytes 35
+  @pool __MODULE__.GenMagicPool
+
+  def child_spec(_) do
+    pool_size = Pleroma.Config.get!([:gen_magic_pool, :size])
+    name = @pool
+
+    %{
+      id: __MODULE__,
+      start: {GenMagic.Pool, :start_link, [[name: name, pool_size: pool_size]]},
+      type: :worker,
+      restart: :permanent,
+      shutdown: 500
+    }
+  end
 
   @spec file_mime_type(String.t(), String.t()) ::
           {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
@@ -20,9 +33,10 @@ def file_mime_type(path, filename) do
 
   @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
   def file_mime_type(filename) do
-    File.open(filename, [:read], fn f ->
-      check_mime_type(IO.binread(f, @read_bytes))
-    end)
+    case GenMagic.Pool.perform(@pool, filename) do
+      {:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
+      error -> error
+    end
   end
 
   def bin_mime_type(binary, filename) do
@@ -34,13 +48,14 @@ def bin_mime_type(binary, filename) do
 
   @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
   def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
-    {:ok, check_mime_type(head)}
+    case GenMagic.Pool.perform(@pool, {:bytes, head}) do
+      {:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
+      error -> error
+    end
   end
 
   def bin_mime_type(_), do: :error
 
-  def mime_type(<<_::binary>>), do: {:ok, @default}
-
   defp fix_extension(filename, content_type) do
     parts = String.split(filename, ".")
 
@@ -62,59 +77,4 @@ defp fix_extension(filename, content_type) do
         Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
     end
   end
-
-  defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>) do
-    "image/png"
-  end
-
-  defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _::binary>>) do
-    "image/gif"
-  end
-
-  defp check_mime_type(<<0xFF, 0xD8, 0xFF, _::binary>>) do
-    "image/jpeg"
-  end
-
-  defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _::binary>>) do
-    "video/webm"
-  end
-
-  defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
-    "video/mp4"
-  end
-
-  defp check_mime_type(<<0x49, 0x44, 0x33, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(
-         <<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::size(160), 0x80, 0x74, 0x68, 0x65,
-           0x6F, 0x72, 0x61, _::binary>>
-       ) do
-    "video/ogg"
-  end
-
-  defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary>>) do
-    "audio/ogg"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
-    "audio/wav"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
-    "image/webp"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
-    "video/avi"
-  end
-
-  defp check_mime_type(_) do
-    @default
-  end
 end
diff --git a/mix.exs b/mix.exs
index 03b060bc0..2d7e16e94 100644
--- a/mix.exs
+++ b/mix.exs
@@ -197,6 +197,7 @@ defp deps do
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
       {:mox, "~> 0.5", only: :test},
       {:restarter, path: "./restarter"},
+      {:gen_magic, git: "https://github.com/hrefhref/gen_magic", branch: "develop"},
       {:open_api_spex,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
        ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}
diff --git a/mix.lock b/mix.lock
index 5383c2c6e..7b7f3c934 100644
--- a/mix.lock
+++ b/mix.lock
@@ -45,6 +45,7 @@
   "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"},
   "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"},
   "floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "631f4e627c46d5ecd347df5a2accdaf0621c77c3693c5b75a8ad58e84c61f242"},
+  "gen_magic": {:git, "https://github.com/hrefhref/gen_magic", "eafdc2ea156433ccd87d1d99eaf1be758064a1db", [branch: "develop"]},
   "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"},
   "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
   "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
@@ -74,6 +75,7 @@
   "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"},
   "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
   "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"},
+  "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
   "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"},
   "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},

From f124f6820582d50be83ba7a1709b14ce8ee1abcc Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Tue, 16 Jun 2020 15:11:45 +0200
Subject: [PATCH 002/208] Switch from gen_magic to majic, use Majic.Plug,
 remove Pleroma.MIME

---
 config/config.exs                             |  2 +-
 config/description.exs                        |  6 +-
 docs/installation/alpine_linux_en.md          |  2 +-
 docs/installation/arch_linux_en.md            |  2 +-
 docs/installation/debian_based_en.md          |  2 +-
 docs/installation/debian_based_jp.md          |  3 +-
 docs/installation/gentoo_en.md                |  2 +-
 docs/installation/otp_en.md                   |  4 +-
 lib/pleroma/application.ex                    |  2 +-
 lib/pleroma/mime.ex                           | 80 -------------------
 lib/pleroma/upload.ex                         | 19 ++---
 .../activity_pub/activity_pub_controller.ex   |  2 +
 .../controllers/media_controller.ex           |  1 +
 .../controllers/account_controller.ex         |  5 ++
 .../controllers/mascot_controller.ex          | 10 ++-
 mix.exs                                       |  2 +-
 mix.lock                                      |  1 +
 test/upload_test.exs                          | 66 ++++-----------
 .../activity_pub_controller_test.exs          |  5 +-
 .../activity_pub/object_validator_test.exs    |  2 +-
 20 files changed, 60 insertions(+), 158 deletions(-)
 delete mode 100644 lib/pleroma/mime.ex

diff --git a/config/config.exs b/config/config.exs
index 65aa8643e..f58b99faa 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -678,7 +678,7 @@
     timeout: 300_000
   ]
 
-config :pleroma, :gen_magic_pool, size: 2
+config :pleroma, :majic_pool, size: 2
 
 config :pleroma, :restrict_unauthenticated,
   timelines: %{local: false, federated: false},
diff --git a/config/description.exs b/config/description.exs
index a318afbe6..2afc5e366 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3328,14 +3328,14 @@
   },
   %{
     group: :pleroma,
-    key: :gen_magic_pool,
+    key: :majic_pool,
     type: :group,
-    description: "GenMagic/libmagic configuration",
+    description: "Majic/libmagic configuration",
     children: [
       %{
         key: :size,
         type: :integer,
-        description: "Number of gen_magic workers to start.",
+        description: "Number of majic workers to start.",
         suggestions: [2]
       }
     ]
diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md
index 1d503ff9f..d3b3e8279 100644
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@@ -40,7 +40,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base
+sudo apk add git build-base file-dev
 ```
 
 ### Install Elixir and Erlang
diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md
index 428ed6bd7..36ef3e345 100644
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@@ -27,7 +27,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir
+sudo pacman -S git base-devel elixir file
 ```
 
 ### Install PostgreSQL
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 21e89d1a0..9357df5d2 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -31,7 +31,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib libmagic-devel
 ```
 
 ### Install Elixir and Erlang
diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 1e5a9be91..21cd4d2ad 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -16,6 +16,7 @@
 - `erlang-nox`
 - `git`
 - `build-essential`
+- `libmagic-dev`
 
 #### このガイドで利用している追加パッケージ
 
@@ -32,7 +33,7 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib libmagic-dev
 ```
 
 
diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md
index e936469d0..d143e068f 100644
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@@ -47,7 +47,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx sys-apps/file
 ```
 
 If you would not like to install the optional packages, remove them from this line. 
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 0cad29bc8..e086cc1c4 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -32,11 +32,11 @@ Other than things bundled in the OTP release Pleroma depends on:
 ```sh tab="Alpine"
 echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
 apk update
-apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot libmagic
+apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev
 ```
 
 ```sh tab="Debian/Ubuntu"
-apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic
+apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev
 ```
 
 ## Setup
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index c74255629..9c74fa00e 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -80,7 +80,7 @@ def start(_type, _args) do
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
-          Pleroma.MIME,
+          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
diff --git a/lib/pleroma/mime.ex b/lib/pleroma/mime.ex
deleted file mode 100644
index 3b406630e..000000000
--- a/lib/pleroma/mime.ex
+++ /dev/null
@@ -1,80 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.MIME do
-  @moduledoc """
-  Returns the mime-type of a binary and optionally a normalized file-name.
-  """
-  @read_bytes 35
-  @pool __MODULE__.GenMagicPool
-
-  def child_spec(_) do
-    pool_size = Pleroma.Config.get!([:gen_magic_pool, :size])
-    name = @pool
-
-    %{
-      id: __MODULE__,
-      start: {GenMagic.Pool, :start_link, [[name: name, pool_size: pool_size]]},
-      type: :worker,
-      restart: :permanent,
-      shutdown: 500
-    }
-  end
-
-  @spec file_mime_type(String.t(), String.t()) ::
-          {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
-  def file_mime_type(path, filename) do
-    with {:ok, content_type} <- file_mime_type(path),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
-  def file_mime_type(filename) do
-    case GenMagic.Pool.perform(@pool, filename) do
-      {:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
-      error -> error
-    end
-  end
-
-  def bin_mime_type(binary, filename) do
-    with {:ok, content_type} <- bin_mime_type(binary),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
-  def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
-    case GenMagic.Pool.perform(@pool, {:bytes, head}) do
-      {:ok, %GenMagic.Result{mime_type: content_type}} -> {:ok, content_type}
-      error -> error
-    end
-  end
-
-  def bin_mime_type(_), do: :error
-
-  defp fix_extension(filename, content_type) do
-    parts = String.split(filename, ".")
-
-    new_filename =
-      if length(parts) > 1 do
-        Enum.drop(parts, -1) |> Enum.join(".")
-      else
-        Enum.join(parts)
-      end
-
-    cond do
-      content_type == "application/octet-stream" ->
-        filename
-
-      ext = List.first(MIME.extensions(content_type)) ->
-        new_filename <> "." <> ext
-
-      true ->
-        Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
-    end
-  end
-end
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 797555bff..a0ba2f4c0 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -57,6 +57,7 @@ defmodule Pleroma.Upload do
   defstruct [:id, :name, :tempfile, :content_type, :path]
 
   @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()}
+  @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct."
   def store(upload, opts \\ []) do
     opts = get_opts(opts)
 
@@ -123,14 +124,13 @@ defp get_opts(opts) do
   end
 
   defp prepare_upload(%Plug.Upload{} = file, opts) do
-    with :ok <- check_file_size(file.path, opts.size_limit),
-         {:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do
+    with :ok <- check_file_size(file.path, opts.size_limit) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: file.filename,
          tempfile: file.path,
-         content_type: content_type
+         content_type: file.content_type
        }}
     end
   end
@@ -138,16 +138,17 @@ defp prepare_upload(%Plug.Upload{} = file, opts) do
   defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
     data = Base.decode64!(parsed["data"], ignore: :whitespace)
-    hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))
+    hash = Base.encode16(:crypto.hash(:sha256, data), lower: true)
 
     with :ok <- check_binary_size(data, opts.size_limit),
          tmp_path <- tempfile_for_image(data),
-         {:ok, content_type, name} <-
-           Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do
+         {:ok, %{mime_type: content_type}} <-
+           Majic.perform({:bytes, data}, pool: Pleroma.MajicPool),
+         [ext | _] <- MIME.extensions(content_type) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: hash <> "." <> ext,
          tempfile: tmp_path,
          content_type: content_type
        }}
@@ -156,7 +157,7 @@ defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
 
   # For Mix.Tasks.MigrateLocalUploads
   defp prepare_upload(%__MODULE__{tempfile: path} = upload, _opts) do
-    with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
+    with {:ok, %{mime_type: content_type}} <- Majic.perform(path, pool: Pleroma.MajicPool) do
       {:ok, %__MODULE__{upload | content_type: content_type}}
     end
   end
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index f0b5c6e93..e2a5fb9e9 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -45,6 +45,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media])
+
   plug(
     Pleroma.Plugs.Cache,
     [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2]
diff --git a/lib/pleroma/web/mastodon_api/controllers/media_controller.ex b/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
index 513de279f..06bb718ef 100644
--- a/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Web.MastodonAPI.MediaController do
 
   plug(OAuthScopesPlug, %{scopes: ["read:media"]} when action == :show)
   plug(OAuthScopesPlug, %{scopes: ["write:media"]} when action != :show)
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:create, :create2])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MediaOperation
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
index f3554d919..97a6ae60d 100644
--- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
@@ -56,6 +56,11 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do
   plug(:assign_account_by_id when action in [:favourites, :subscribe, :unsubscribe])
   plug(:put_view, Pleroma.Web.MastodonAPI.AccountView)
 
+  plug(
+    Majic.Plug,
+    [pool: Pleroma.MajicPool] when action in [:update_avatar, :update_background, :update_banner]
+  )
+
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaAccountOperation
 
   @doc "POST /api/v1/pleroma/accounts/confirmation_resend"
diff --git a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
index df6c50ca5..4ba4154dd 100644
--- a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
@@ -12,6 +12,7 @@ defmodule Pleroma.Web.PleromaAPI.MascotController do
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action == :show)
   plug(OAuthScopesPlug, %{scopes: ["write:accounts"]} when action != :show)
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:update])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaMascotOperation
 
@@ -22,14 +23,15 @@ def show(%{assigns: %{user: user}} = conn, _params) do
 
   @doc "PUT /api/v1/pleroma/mascot"
   def update(%{assigns: %{user: user}, body_params: %{file: file}} = conn, _) do
-    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
-         # Reject if not an image
-         %{type: "image"} = attachment <- render_attachment(object) do
+    with {:content_type, "image" <> _} <- {:content_type, file.content_type},
+         {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)) do
+      attachment = render_attachment(object)
       {:ok, _user} = User.mascot_update(user, attachment)
 
       json(conn, attachment)
     else
-      %{type: _} -> render_error(conn, :unsupported_media_type, "mascots can only be images")
+      {:content_type, _} ->
+        render_error(conn, :unsupported_media_type, "mascots can only be images")
     end
   end
 
diff --git a/mix.exs b/mix.exs
index 2d7e16e94..8fcbaa34e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -197,7 +197,7 @@ defp deps do
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
       {:mox, "~> 0.5", only: :test},
       {:restarter, path: "./restarter"},
-      {:gen_magic, git: "https://github.com/hrefhref/gen_magic", branch: "develop"},
+      {:majic, git: "https://github.com/hrefhref/majic", branch: "develop"},
       {:open_api_spex,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
        ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}
diff --git a/mix.lock b/mix.lock
index 7b7f3c934..8ff41ebf6 100644
--- a/mix.lock
+++ b/mix.lock
@@ -63,6 +63,7 @@
   "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
+  "majic": {:git, "https://github.com/hrefhref/majic", "1c723300364cd014866c6c1bd5260e03965865a2", [branch: "develop"]},
   "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},
diff --git a/test/upload_test.exs b/test/upload_test.exs
index 2abf0edec..c7ad177d9 100644
--- a/test/upload_test.exs
+++ b/test/upload_test.exs
@@ -11,7 +11,7 @@ defmodule Pleroma.UploadTest do
   alias Pleroma.Uploaders.Uploader
 
   @upload_file %Plug.Upload{
-    content_type: "image/jpg",
+    content_type: "image/jpeg",
     path: Path.absname("test/fixtures/image_tmp.jpg"),
     filename: "image.jpg"
   }
@@ -111,7 +111,7 @@ test "returns a media url" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "image.jpg"
       }
@@ -127,7 +127,7 @@ test "copies the file to the configured folder with deduping" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an [image.jpg"
       }
@@ -143,7 +143,7 @@ test "copies the file to the configured folder without deduping" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an [image.jpg"
       }
@@ -152,63 +152,31 @@ test "copies the file to the configured folder without deduping" do
       assert data["name"] == "an [image.jpg"
     end
 
-    test "fixes incorrect content type" do
-      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
-
-      file = %Plug.Upload{
-        content_type: "application/octet-stream",
-        path: Path.absname("test/fixtures/image_tmp.jpg"),
-        filename: "an [image.jpg"
+    test "fixes incorrect content type when base64 is given" do
+      params = %{
+        img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}"
       }
 
-      {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
+      {:ok, data} = Upload.store(params)
       assert hd(data["url"])["mediaType"] == "image/jpeg"
     end
 
-    test "adds missing extension" do
+    test "adds extension when base64 is given" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
-      file = %Plug.Upload{
-        content_type: "image/jpg",
-        path: Path.absname("test/fixtures/image_tmp.jpg"),
-        filename: "an [image"
+      params = %{
+        img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}"
       }
 
-      {:ok, data} = Upload.store(file)
-      assert data["name"] == "an [image.jpg"
-    end
-
-    test "fixes incorrect file extension" do
-      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
-
-      file = %Plug.Upload{
-        content_type: "image/jpg",
-        path: Path.absname("test/fixtures/image_tmp.jpg"),
-        filename: "an [image.blah"
-      }
-
-      {:ok, data} = Upload.store(file)
-      assert data["name"] == "an [image.jpg"
-    end
-
-    test "don't modify filename of an unknown type" do
-      File.cp("test/fixtures/test.txt", "test/fixtures/test_tmp.txt")
-
-      file = %Plug.Upload{
-        content_type: "text/plain",
-        path: Path.absname("test/fixtures/test_tmp.txt"),
-        filename: "test.txt"
-      }
-
-      {:ok, data} = Upload.store(file)
-      assert data["name"] == "test.txt"
+      {:ok, data} = Upload.store(params)
+      assert String.ends_with?(data["name"], ".jpg")
     end
 
     test "copies the file to the configured folder with anonymizing filename" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an [image.jpg"
       }
@@ -222,7 +190,7 @@ test "escapes invalid characters in url" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an… image.jpg"
       }
@@ -237,7 +205,7 @@ test "escapes reserved uri characters" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: ":?#[]@!$&\\'()*+,;=.jpg"
       }
@@ -259,7 +227,7 @@ test "returns a media url with configured base_url" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "image.jpg"
       }
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index e490a5744..8c6ee68b2 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -1428,9 +1428,9 @@ test "POST /api/ap/upload_media", %{conn: conn} do
       desc = "Description of the image"
 
       image = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "bad/content-type",
         path: Path.absname("test/fixtures/image.jpg"),
-        filename: "an_image.jpg"
+        filename: "an_image.png"
       }
 
       object =
@@ -1445,6 +1445,7 @@ test "POST /api/ap/upload_media", %{conn: conn} do
       assert [%{"href" => object_href, "mediaType" => object_mediatype}] = object["url"]
       assert is_binary(object_href)
       assert object_mediatype == "image/jpeg"
+      assert String.ends_with?(object_href, ".jpg")
 
       activity_request = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 31224abe0..ee1e1bcfe 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -58,7 +58,7 @@ test "it handles our own uploads" do
       user = insert(:user)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }

From 39f7fc5b8ef781c98136d1f9be50a14bff394233 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Tue, 16 Jun 2020 19:00:54 +0200
Subject: [PATCH 003/208] Update majic & call plug before OpenApiSpex

---
 .../web/mastodon_api/controllers/media_controller.ex   |  2 +-
 .../web/pleroma_api/controllers/account_controller.ex  | 10 +++++-----
 .../web/pleroma_api/controllers/mascot_controller.ex   |  2 +-
 mix.lock                                               |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/controllers/media_controller.ex b/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
index 06bb718ef..09acea7f4 100644
--- a/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/media_controller.ex
@@ -11,12 +11,12 @@ defmodule Pleroma.Web.MastodonAPI.MediaController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:create, :create2])
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(:put_view, Pleroma.Web.MastodonAPI.StatusView)
 
   plug(OAuthScopesPlug, %{scopes: ["read:media"]} when action == :show)
   plug(OAuthScopesPlug, %{scopes: ["write:media"]} when action != :show)
-  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:create, :create2])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MediaOperation
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
index 97a6ae60d..c76cbfc48 100644
--- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex
@@ -18,6 +18,11 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do
 
   require Pleroma.Constants
 
+  plug(
+    Majic.Plug,
+    [pool: Pleroma.MajicPool] when action in [:update_avatar, :update_background, :update_banner]
+  )
+
   plug(
     OpenApiSpex.Plug.PutApiSpec,
     [module: Pleroma.Web.ApiSpec] when action == :confirmation_resend
@@ -56,11 +61,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do
   plug(:assign_account_by_id when action in [:favourites, :subscribe, :unsubscribe])
   plug(:put_view, Pleroma.Web.MastodonAPI.AccountView)
 
-  plug(
-    Majic.Plug,
-    [pool: Pleroma.MajicPool] when action in [:update_avatar, :update_background, :update_banner]
-  )
-
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaAccountOperation
 
   @doc "POST /api/v1/pleroma/accounts/confirmation_resend"
diff --git a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
index 4ba4154dd..7e2f6c328 100644
--- a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
@@ -9,10 +9,10 @@ defmodule Pleroma.Web.PleromaAPI.MascotController do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
 
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:update])
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action == :show)
   plug(OAuthScopesPlug, %{scopes: ["write:accounts"]} when action != :show)
-  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:update])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaMascotOperation
 
diff --git a/mix.lock b/mix.lock
index 8ff41ebf6..b69f7eb8e 100644
--- a/mix.lock
+++ b/mix.lock
@@ -63,7 +63,7 @@
   "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
-  "majic": {:git, "https://github.com/hrefhref/majic", "1c723300364cd014866c6c1bd5260e03965865a2", [branch: "develop"]},
+  "majic": {:git, "https://github.com/hrefhref/majic", "91a10cfa0c902eb889584238a3ef5aa595c21ef8", [branch: "develop"]},
   "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},

From afa8b469ed0a71247f27efec08d6eeac24b6674f Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 1 Jul 2020 21:12:59 -0500
Subject: [PATCH 004/208] Allow restricting public timeline by instance

---
 lib/pleroma/web/activity_pub/activity_pub.ex         | 12 ++----------
 .../web/api_spec/operations/timeline_operation.ex    | 10 ++++++++++
 .../mastodon_api/controllers/timeline_controller.ex  |  1 +
 .../controllers/timeline_controller_test.exs         | 12 ++++++++++++
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bc7b5d95a..9ce2b04dd 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -927,16 +927,8 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{instance: instance}) do
-    users =
-      from(
-        u in User,
-        select: u.ap_id,
-        where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}")
-      )
-      |> Repo.all()
-
-    from(activity in query, where: activity.actor in ^users)
+  defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
+    from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%"))
   end
 
   defp restrict_instance(query, _), do: query
diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 8e19bace7..83cdbad69 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -59,6 +59,7 @@ def public_operation do
       security: [%{"oAuth" => ["read:statuses"]}],
       parameters: [
         local_param(),
+        instance_param(),
         only_media_param(),
         with_muted_param(),
         exclude_visibilities_param(),
@@ -158,6 +159,15 @@ defp local_param do
     )
   end
 
+  defp instance_param do
+    Operation.parameter(
+      :instance,
+      :query,
+      %Schema{type: :string},
+      "Show only statuses from the given domain"
+    )
+  end
+
   defp with_muted_param do
     Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
index ab7b1d6aa..7dccc0005 100644
--- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -110,6 +110,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
         |> Map.put(:blocking_user, user)
         |> Map.put(:muting_user, user)
         |> Map.put(:reply_filtering_user, user)
+        |> Map.put(:instance, params[:instance])
         |> ActivityPub.fetch_public_activities()
 
       conn
diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs
index 50e0d783d..6acd512c7 100644
--- a/test/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -140,6 +140,18 @@ test "doesn't return replies if follow is posting with users from blocked domain
       activities = json_response_and_validate_schema(res_conn, 200)
       [%{"id" => ^activity_id}] = activities
     end
+
+    test "can be filtered by instance", %{conn: conn} do
+      user = insert(:user, ap_id: "https://lain.com/users/lain")
+      insert(:note_activity, local: false)
+      insert(:note_activity, local: false)
+
+      {:ok, _} = CommonAPI.post(user, %{status: "test"})
+
+      conn = get(conn, "/api/v1/timelines/public?instance=lain.com")
+
+      assert length(json_response_and_validate_schema(conn, :ok)) == 1
+    end
   end
 
   defp local_and_remote_activities do

From e9cff69bceb5cfb14a13ef09d1d7ab1079028a58 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Sun, 2 Aug 2020 12:24:40 -0500
Subject: [PATCH 005/208] Add TagPolicy as default MRF, #2010

---
 config/config.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 134dccf49..68a050315 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -719,7 +719,7 @@
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
 config :pleroma, :mrf,
-  policies: Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
+  policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy],
   transparency: true,
   transparency_exclusions: []
 

From ad9c925efb77287316f5dbac26f6a1b16662910a Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 5 Aug 2020 13:08:13 -0500
Subject: [PATCH 006/208] Speed up instance timeline query

---
 lib/pleroma/web/activity_pub/activity_pub.ex                 | 5 ++++-
 test/web/admin_api/controllers/admin_api_controller_test.exs | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 9ce2b04dd..76fc9c3ee 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -928,7 +928,10 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
   defp restrict_muted_reblogs(query, _), do: query
 
   defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
-    from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%"))
+    from(
+      activity in query,
+      where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance)
+    )
   end
 
   defp restrict_instance(query, _), do: query
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index da91cd552..26194eb81 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -1647,8 +1647,8 @@ test "sets password_reset_pending to true", %{conn: conn} do
 
   describe "instances" do
     test "GET /instances/:instance/statuses", %{conn: conn} do
-      user = insert(:user, local: false, nickname: "archaeme@archae.me")
-      user2 = insert(:user, local: false, nickname: "test@test.com")
+      user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme")
+      user2 = insert(:user, local: false, ap_id: "https://test.com/users/test")
       insert_pair(:note_activity, user: user)
       activity = insert(:note_activity, user: user2)
 

From 24ce9c011caf7401fb261c7df4196b2ef9ba3d90 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 5 Aug 2020 19:33:51 +0000
Subject: [PATCH 007/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/api_spec/operations/timeline_operation.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 83cdbad69..95720df9f 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -169,7 +169,7 @@ defp instance_param do
   end
 
   defp with_muted_param do
-    Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
+    Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end
 
   defp exclude_visibilities_param do

From 0d5088c2b83fafd9d8da1f1b04936f831ac5ee87 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 1 Sep 2020 09:37:08 +0300
Subject: [PATCH 008/208] remove `unread_conversation_count` from User

---
 lib/pleroma/conversation.ex                   |  6 +--
 lib/pleroma/conversation/participation.ex     | 27 +++++-------
 lib/pleroma/user.ex                           | 42 -------------------
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 ...ve_unread_conversation_count_from_user.exs | 38 +++++++++++++++++
 ...ad_index_to_conversation_participation.exs | 12 ++++++
 test/conversation/participation_test.exs      | 32 +++++++-------
 .../conversation_controller_test.exs          | 23 +++++-----
 .../conversation_controller_test.exs          |  4 +-
 9 files changed, 92 insertions(+), 94 deletions(-)
 create mode 100644 priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
 create mode 100644 priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs

diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex
index e76eb0087..77933f0be 100644
--- a/lib/pleroma/conversation.ex
+++ b/lib/pleroma/conversation.ex
@@ -43,7 +43,7 @@ def get_for_ap_id(ap_id) do
   def maybe_create_recipientships(participation, activity) do
     participation = Repo.preload(participation, :recipients)
 
-    if participation.recipients |> Enum.empty?() do
+    if Enum.empty?(participation.recipients) do
       recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
@@ -69,10 +69,6 @@ def create_or_bump_for(activity, opts \\ []) do
         Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
-          unless invisible_conversation do
-            User.increment_unread_conversation_count(conversation, user)
-          end
-
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =
diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index 8bc3e85d6..4c32b273a 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -63,21 +63,10 @@ def mark_as_read(%User{} = user, %Conversation{} = conversation) do
     end
   end
 
-  def mark_as_read(participation) do
-    __MODULE__
-    |> where(id: ^participation.id)
-    |> update(set: [read: true])
-    |> select([p], p)
-    |> Repo.update_all([])
-    |> case do
-      {1, [participation]} ->
-        participation = Repo.preload(participation, :user)
-        User.set_unread_conversation_count(participation.user)
-        {:ok, participation}
-
-      error ->
-        error
-    end
+  def mark_as_read(%__MODULE__{} = participation) do
+    participation
+    |> change(read: true)
+    |> Repo.update()
   end
 
   def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
@@ -93,7 +82,6 @@ def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
     |> update([p], set: [read: true])
     |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, []}
   end
 
@@ -108,7 +96,6 @@ def mark_all_as_read(%User{} = user) do
       |> select([p], p)
       |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, participations}
   end
 
@@ -220,6 +207,12 @@ def set_recipients(participation, user_ids) do
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
 
+  @spec unread_count(User.t()) :: integer()
+  def unread_count(%User{id: user_id}) do
+    from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false)
+    |> Repo.aggregate(:count, :id)
+  end
+
   def unread_conversation_count_for_user(user) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d2ad9516f..7fc7a533e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -129,7 +129,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
@@ -1295,47 +1294,6 @@ def update_following_count(%User{local: true} = user) do
     |> update_and_set_cache()
   end
 
-  def set_unread_conversation_count(%User{local: true} = user) do
-    unread_query = Participation.unread_conversation_count_for_user(user)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      set: [unread_conversation_count: p.count]
-    )
-    |> where([u], u.id == ^user.id)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def set_unread_conversation_count(user), do: {:ok, user}
-
-  def increment_unread_conversation_count(conversation, %User{local: true} = user) do
-    unread_query =
-      Participation.unread_conversation_count_for_user(user)
-      |> where([p], p.conversation_id == ^conversation.id)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      inc: [unread_conversation_count: 1]
-    )
-    |> where([u], u.id == ^user.id)
-    |> where([u, p], p.count == 0)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def increment_unread_conversation_count(_, user), do: {:ok, user}
-
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 864c0417f..1bf53600c 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -386,7 +386,7 @@ defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{
     data
     |> Kernel.put_in(
       [:pleroma, :unread_conversation_count],
-      user.unread_conversation_count
+      Pleroma.Conversation.Participation.unread_count(user)
     )
   end
 
diff --git a/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
new file mode 100644
index 000000000..b7bdb9166
--- /dev/null
+++ b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
@@ -0,0 +1,38 @@
+defmodule Pleroma.Repo.Migrations.RemoveUnreadConversationCountFromUser do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    alter table(:users) do
+      remove_if_exists(:unread_conversation_count, :integer)
+    end
+  end
+
+  def down do
+    alter table(:users) do
+      add_if_not_exists(:unread_conversation_count, :integer, default: 0)
+    end
+
+    flush()
+    recalc_unread_conversation_count()
+  end
+
+  defp recalc_unread_conversation_count do
+    participations_subquery =
+      from(
+        p in "conversation_participations",
+        where: p.read == false,
+        group_by: p.user_id,
+        select: %{user_id: p.user_id, unread_conversation_count: count(p.id)}
+      )
+
+    from(
+      u in "users",
+      join: p in subquery(participations_subquery),
+      on: p.user_id == u.id,
+      update: [set: [unread_conversation_count: p.unread_conversation_count]]
+    )
+    |> Repo.update_all([])
+  end
+end
diff --git a/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs
new file mode 100644
index 000000000..68771c655
--- /dev/null
+++ b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs
@@ -0,0 +1,12 @@
+defmodule Pleroma.Repo.Migrations.AddUnreadIndexToConversationParticipation do
+  use Ecto.Migration
+
+  def change do
+    create(
+      index(:conversation_participations, [:user_id],
+        where: "read = false",
+        name: "unread_conversation_participation_count_index"
+      )
+    )
+  end
+end
diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs
index 59a1b6492..5a603dcc1 100644
--- a/test/conversation/participation_test.exs
+++ b/test/conversation/participation_test.exs
@@ -37,9 +37,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat
 
     [%{read: true}] = Participation.for_user(user)
     [%{read: false} = participation] = Participation.for_user(other_user)
-
-    assert User.get_cached_by_id(user.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 1
+    assert Participation.unread_count(user) == 0
+    assert Participation.unread_count(other_user) == 1
 
     {:ok, _} =
       CommonAPI.post(other_user, %{
@@ -54,8 +53,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat
     [%{read: false}] = Participation.for_user(user)
     [%{read: true}] = Participation.for_user(other_user)
 
-    assert User.get_cached_by_id(user.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0
+    assert Participation.unread_count(user) == 1
+    assert Participation.unread_count(other_user) == 0
   end
 
   test "for a new conversation, it sets the recipents of the participation" do
@@ -264,7 +263,7 @@ test "when the user blocks a recipient, the existing conversations with them are
       assert [%{read: false}, %{read: false}, %{read: false}, %{read: false}] =
                Participation.for_user(blocker)
 
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 4
+      assert Participation.unread_count(blocker) == 4
 
       {:ok, _user_relationship} = User.block(blocker, blocked)
 
@@ -272,15 +271,15 @@ test "when the user blocks a recipient, the existing conversations with them are
       assert [%{read: true}, %{read: true}, %{read: true}, %{read: false}] =
                Participation.for_user(blocker)
 
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocker) == 1
 
       # The conversation is not marked as read for the blocked user
       assert [_, _, %{read: false}] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocker) == 1
 
       # The conversation is not marked as read for the third user
       assert [%{read: false}, _, _] = Participation.for_user(third_user)
-      assert User.get_cached_by_id(third_user.id).unread_conversation_count == 1
+      assert Participation.unread_count(third_user) == 1
     end
 
     test "the new conversation with the blocked user is not marked as unread " do
@@ -298,7 +297,7 @@ test "the new conversation with the blocked user is not marked as unread " do
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       # When the blocked user is a recipient
       {:ok, _direct2} =
@@ -308,10 +307,10 @@ test "the new conversation with the blocked user is not marked as unread " do
         })
 
       assert [%{read: true}, %{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       assert [%{read: false}, _] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocked) == 1
     end
 
     test "the conversation with the blocked user is not marked as unread on a reply" do
@@ -327,8 +326,8 @@ test "the conversation with the blocked user is not marked as unread on a reply"
 
       {:ok, _user_relationship} = User.block(blocker, blocked)
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
 
+      assert Participation.unread_count(blocker) == 0
       assert [blocked_participation] = Participation.for_user(blocked)
 
       # When it's a reply from the blocked user
@@ -340,8 +339,8 @@ test "the conversation with the blocked user is not marked as unread on a reply"
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
 
+      assert Participation.unread_count(blocker) == 0
       assert [third_user_participation] = Participation.for_user(third_user)
 
       # When it's a reply from the third user
@@ -353,11 +352,12 @@ test "the conversation with the blocked user is not marked as unread on a reply"
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       # Marked as unread for the blocked user
       assert [%{read: false}] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+
+      assert Participation.unread_count(blocked) == 1
     end
   end
 end
diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs
index 3e21e6bf1..b23b22752 100644
--- a/test/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do
   use Pleroma.Web.ConnCase
 
+  alias Pleroma.Conversation.Participation
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
@@ -28,10 +29,10 @@ test "returns correct conversations", %{
       user_three: user_three,
       conn: conn
     } do
-      assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+      assert Participation.unread_count(user_two) == 0
       {:ok, direct} = create_direct_message(user_one, [user_two, user_three])
 
-      assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1
+      assert Participation.unread_count(user_two) == 1
 
       {:ok, _follower_only} =
         CommonAPI.post(user_one, %{
@@ -59,7 +60,7 @@ test "returns correct conversations", %{
       assert is_binary(res_id)
       assert unread == false
       assert res_last_status["id"] == direct.id
-      assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
+      assert Participation.unread_count(user_one) == 0
     end
 
     test "observes limit params", %{
@@ -134,8 +135,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
     user_two = insert(:user)
     {:ok, direct} = create_direct_message(user_one, [user_two])
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1
+    assert Participation.unread_count(user_one) == 0
+    assert Participation.unread_count(user_two) == 1
 
     user_two_conn =
       build_conn()
@@ -155,8 +156,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
       |> post("/api/v1/conversations/#{direct_conversation_id}/read")
       |> json_response_and_validate_schema(200)
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 0
+    assert Participation.unread_count(user_two) == 0
 
     # The conversation is marked as unread on reply
     {:ok, _} =
@@ -171,8 +172,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
       |> get("/api/v1/conversations")
       |> json_response_and_validate_schema(200)
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 1
+    assert Participation.unread_count(user_two) == 0
 
     # A reply doesn't increment the user's unread_conversation_count if the conversation is unread
     {:ok, _} =
@@ -182,8 +183,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
         in_reply_to_status_id: direct.id
       })
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 1
+    assert Participation.unread_count(user_two) == 0
   end
 
   test "(vanilla) Mastodon frontend behaviour", %{user: user_one, conn: conn} do
diff --git a/test/web/pleroma_api/controllers/conversation_controller_test.exs b/test/web/pleroma_api/controllers/conversation_controller_test.exs
index e6d0b3e37..f2feeaaef 100644
--- a/test/web/pleroma_api/controllers/conversation_controller_test.exs
+++ b/test/web/pleroma_api/controllers/conversation_controller_test.exs
@@ -121,7 +121,7 @@ test "POST /api/v1/pleroma/conversations/read" do
     [participation2, participation1] = Participation.for_user(other_user)
     assert Participation.get(participation2.id).read == false
     assert Participation.get(participation1.id).read == false
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 2
+    assert Participation.unread_count(other_user) == 2
 
     [%{"unread" => false}, %{"unread" => false}] =
       conn
@@ -131,6 +131,6 @@ test "POST /api/v1/pleroma/conversations/read" do
     [participation2, participation1] = Participation.for_user(other_user)
     assert Participation.get(participation2.id).read == true
     assert Participation.get(participation1.id).read == true
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0
+    assert Participation.unread_count(other_user) == 0
   end
 end

From c56e3d4f3bfb090d19bdbe93dac6cede7616cc4d Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Tue, 8 Sep 2020 13:26:44 +0300
Subject: [PATCH 009/208] Add expires_in param for account mutes

---
 config/config.exs                             |  5 ++-
 lib/pleroma/user.ex                           | 45 ++++++++++---------
 .../api_spec/operations/account_operation.ex  | 15 ++++++-
 .../controllers/account_controller.ex         |  2 +-
 lib/pleroma/workers/mute_expire_worker.ex     | 22 +++++++++
 test/notification_test.exs                    |  4 +-
 test/user_test.exs                            |  2 +-
 .../notification_controller_test.exs          |  2 +-
 .../mastodon_api/views/account_view_test.exs  |  2 +-
 9 files changed, 69 insertions(+), 30 deletions(-)
 create mode 100644 lib/pleroma/workers/mute_expire_worker.ex

diff --git a/config/config.exs b/config/config.exs
index ed37b93c0..0649f3078 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -541,7 +541,8 @@
     background: 5,
     remote_fetcher: 2,
     attachments_cleanup: 5,
-    new_users_digest: 1
+    new_users_digest: 1,
+    mute_expire: 5
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [
@@ -672,7 +673,7 @@
 # With no frontend configuration, the bundled files from the `static` directory will
 # be used.
 #
-# config :pleroma, :frontends, 
+# config :pleroma, :frontends,
 # primary: %{"name" => "pleroma-fe", "ref" => "develop"},
 # admin: %{"name" => "admin-fe", "ref" => "stable"},
 # available: %{...}
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 94c96de8d..040db8d80 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1356,14 +1356,34 @@ def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
     |> Repo.all()
   end
 
-  @spec mute(User.t(), User.t(), boolean()) ::
+  @spec mute(User.t(), User.t(), map()) ::
           {:ok, list(UserRelationship.t())} | {:error, String.t()}
-  def mute(%User{} = muter, %User{} = mutee, notifications? \\ true) do
-    add_to_mutes(muter, mutee, notifications?)
+  def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
+    notifications? = Map.get(params, :notifications, true)
+    expires_in = Map.get(params, :expires_in, 0)
+
+    with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           (notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
+             {:ok, nil} do
+      with seconds when seconds > 0 <- expires_in do
+        Pleroma.Workers.MuteExpireWorker.enqueue(
+          "unmute",
+          %{"muter" => muter.id, "mutee" => mutee.id},
+          schedule_in: expires_in
+        )
+      end
+
+      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
+    end
   end
 
   def unmute(%User{} = muter, %User{} = mutee) do
-    remove_from_mutes(muter, mutee)
+    with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           UserRelationship.delete_notification_mute(muter, mutee) do
+      {:ok, [user_mute, user_notification_mute]}
+    end
   end
 
   def subscribe(%User{} = subscriber, %User{} = target) do
@@ -2379,23 +2399,6 @@ defp remove_from_block(%User{} = user, %User{} = blocked) do
     UserRelationship.delete_block(user, blocked)
   end
 
-  defp add_to_mutes(%User{} = user, %User{} = muted_user, notifications?) do
-    with {:ok, user_mute} <- UserRelationship.create_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           (notifications? && UserRelationship.create_notification_mute(user, muted_user)) ||
-             {:ok, nil} do
-      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
-    end
-  end
-
-  defp remove_from_mutes(user, %User{} = muted_user) do
-    with {:ok, user_mute} <- UserRelationship.delete_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           UserRelationship.delete_notification_mute(user, muted_user) do
-      {:ok, [user_mute, user_notification_mute]}
-    end
-  end
-
   def set_invisible(user, invisible) do
     params = %{invisible: invisible}
 
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index aaebc9b5c..de715a077 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -262,6 +262,12 @@ def mute_operation do
           :query,
           %Schema{allOf: [BooleanLike], default: true},
           "Mute notifications in addition to statuses? Defaults to `true`."
+        ),
+        Operation.parameter(
+          :expires_in,
+          :query,
+          %Schema{type: :integer, default: 0},
+          "Expire the mute in `expires_in` seconds. Default 0 for infinity"
         )
       ],
       responses: %{
@@ -718,10 +724,17 @@ defp mute_request do
           nullable: true,
           description: "Mute notifications in addition to statuses? Defaults to true.",
           default: true
+        },
+        expires_in: %Schema{
+          type: :integer,
+          nullable: true,
+          description: "Expire the mute in `expires_in` seconds. Default 0 for infinity",
+          default: 0
         }
       },
       example: %{
-        "notifications" => true
+        "notifications" => true,
+        "expires_in" => 86_400
       }
     }
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 95d8452df..ca1a79f5e 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -394,7 +394,7 @@ def unfollow(%{assigns: %{user: follower, account: followed}} = conn, _params) d
 
   @doc "POST /api/v1/accounts/:id/mute"
   def mute(%{assigns: %{user: muter, account: muted}, body_params: params} = conn, _params) do
-    with {:ok, _user_relationships} <- User.mute(muter, muted, params.notifications) do
+    with {:ok, _user_relationships} <- User.mute(muter, muted, params) do
       render(conn, "relationship.json", user: muter, target: muted)
     else
       {:error, message} -> json_response(conn, :forbidden, %{error: message})
diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex
new file mode 100644
index 000000000..b8ec939a9
--- /dev/null
+++ b/lib/pleroma/workers/mute_expire_worker.ex
@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.MuteExpireWorker do
+  use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
+
+  require Logger
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"op" => "unmute", "muter" => muter_id, "mutee" => mutee_id}}) do
+    muter = Pleroma.User.get_by_id(muter_id)
+    mutee = Pleroma.User.get_by_id(mutee_id)
+    Pleroma.User.unmute(muter, mutee)
+    :ok
+  end
+
+  def perform(any) do
+    Logger.error("Got call to perform(#{inspect(any)})")
+    :ok
+  end
+end
diff --git a/test/notification_test.exs b/test/notification_test.exs
index a09b08675..ffd737969 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -227,7 +227,7 @@ test "notification created if user is muted without notifications" do
       muter = insert(:user)
       muted = insert(:user)
 
-      {:ok, _user_relationships} = User.mute(muter, muted, false)
+      {:ok, _user_relationships} = User.mute(muter, muted, %{notifications: false})
 
       {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"})
 
@@ -1013,7 +1013,7 @@ test "move activity generates a notification" do
 
     test "it returns notifications for muted user without notifications", %{user: user} do
       muted = insert(:user)
-      {:ok, _user_relationships} = User.mute(user, muted, false)
+      {:ok, _user_relationships} = User.mute(user, muted, %{notifications: false})
 
       {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"})
 
diff --git a/test/user_test.exs b/test/user_test.exs
index 50f72549e..b23e36be3 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -981,7 +981,7 @@ test "it mutes user without notifications" do
       refute User.mutes?(user, muted_user)
       refute User.muted_notifications?(user, muted_user)
 
-      {:ok, _user_relationships} = User.mute(user, muted_user, false)
+      {:ok, _user_relationships} = User.mute(user, muted_user, %{notifications: false})
 
       assert User.mutes?(user, muted_user)
       refute User.muted_notifications?(user, muted_user)
diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs
index 70ef0e8b5..5fd518c60 100644
--- a/test/web/mastodon_api/controllers/notification_controller_test.exs
+++ b/test/web/mastodon_api/controllers/notification_controller_test.exs
@@ -502,7 +502,7 @@ test "see notifications after muting user without notifications" do
 
     assert length(json_response_and_validate_schema(ret_conn, 200)) == 1
 
-    {:ok, _user_relationships} = User.mute(user, user2, false)
+    {:ok, _user_relationships} = User.mute(user, user2, %{notifications: false})
 
     conn = get(conn, "/api/v1/notifications")
 
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index 8f37efa3c..c34cbcfc1 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -277,7 +277,7 @@ test "represent a relationship for the following and followed user" do
       {:ok, user} = User.follow(user, other_user)
       {:ok, other_user} = User.follow(other_user, user)
       {:ok, _subscription} = User.subscribe(user, other_user)
-      {:ok, _user_relationships} = User.mute(user, other_user, true)
+      {:ok, _user_relationships} = User.mute(user, other_user, %{notifications: true})
       {:ok, _reblog_mute} = CommonAPI.hide_reblogs(user, other_user)
 
       expected =

From f6b250fb8d13f6788c1ecc6c1287e76febbfd888 Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Tue, 8 Sep 2020 14:11:00 +0300
Subject: [PATCH 010/208] Add test for expiring mutes

---
 test/user_test.exs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/user_test.exs b/test/user_test.exs
index b23e36be3..83c017ec5 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -963,6 +963,19 @@ test "it mutes people" do
       assert User.muted_notifications?(user, muted_user)
     end
 
+    test "expiring" do
+      user = insert(:user)
+      muted_user = insert(:user)
+
+      {:ok, _user_relationships} = User.mute(user, muted_user, %{expires_in: 60})
+      assert User.mutes?(user, muted_user)
+
+      assert_enqueued(
+        worker: Pleroma.Workers.MuteExpireWorker,
+        args: %{"op" => "unmute", "muter" => user.id, "mutee" => muted_user.id}
+      )
+    end
+
     test "it unmutes users" do
       user = insert(:user)
       muted_user = insert(:user)

From e3f845b24363cd867ab85b7297f2d34bfa16b13f Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Tue, 8 Sep 2020 15:13:50 +0300
Subject: [PATCH 011/208] Add expiring mutes for activities

---
 lib/pleroma/user.ex                           |  6 ++---
 .../api_spec/operations/status_operation.ex   | 22 ++++++++++++++++++-
 lib/pleroma/web/common_api/common_api.ex      | 12 +++++++++-
 lib/pleroma/workers/mute_expire_worker.ex     | 10 ++++++---
 test/user_test.exs                            | 12 ++++++++--
 test/web/common_api/common_api_test.exs       | 18 +++++++++++++++
 6 files changed, 70 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 040db8d80..46e03553c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1366,10 +1366,10 @@ def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
          {:ok, user_notification_mute} <-
            (notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
              {:ok, nil} do
-      with seconds when seconds > 0 <- expires_in do
+      if expires_in > 0 do
         Pleroma.Workers.MuteExpireWorker.enqueue(
-          "unmute",
-          %{"muter" => muter.id, "mutee" => mutee.id},
+          "unmute_user",
+          %{"muter_id" => muter.id, "mutee_id" => mutee.id},
           schedule_in: expires_in
         )
       end
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index 5bd4619d5..6589a16f3 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -223,7 +223,27 @@ def mute_conversation_operation do
       security: [%{"oAuth" => ["write:mutes"]}],
       description: "Do not receive notifications for the thread that this status is part of.",
       operationId: "StatusController.mute_conversation",
-      parameters: [id_param()],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            expires_in: %Schema{
+              type: :integer,
+              nullable: true,
+              description: "Expire the mute in `expires_in` seconds. Default 0 for infinity",
+              default: 0
+            }
+          }
+        }),
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :expires_in,
+          :query,
+          %Schema{type: :integer, default: 0},
+          "Expire the mute in `expires_in` seconds. Default 0 for infinity"
+        )
+      ],
       responses: %{
         200 => status_response(),
         400 => Operation.response("Error", "application/json", ApiError)
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 4ab533658..b217c4d10 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -451,9 +451,19 @@ def unpin(id, user) do
     end
   end
 
-  def add_mute(user, activity) do
+  def add_mute(user, activity, params \\ %{}) do
+    expires_in = Map.get(params, :expires_in, 0)
+
     with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]),
          _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do
+      if expires_in > 0 do
+        Pleroma.Workers.MuteExpireWorker.enqueue(
+          "unmute_conversation",
+          %{"user_id" => user.id, "activity_id" => activity.id},
+          schedule_in: expires_in
+        )
+      end
+
       {:ok, activity}
     else
       {:error, _} -> {:error, dgettext("errors", "conversation is already muted")}
diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex
index b8ec939a9..622fdbadd 100644
--- a/lib/pleroma/workers/mute_expire_worker.ex
+++ b/lib/pleroma/workers/mute_expire_worker.ex
@@ -8,15 +8,19 @@ defmodule Pleroma.Workers.MuteExpireWorker do
   require Logger
 
   @impl Oban.Worker
-  def perform(%Job{args: %{"op" => "unmute", "muter" => muter_id, "mutee" => mutee_id}}) do
+  def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do
     muter = Pleroma.User.get_by_id(muter_id)
     mutee = Pleroma.User.get_by_id(mutee_id)
     Pleroma.User.unmute(muter, mutee)
     :ok
   end
 
-  def perform(any) do
-    Logger.error("Got call to perform(#{inspect(any)})")
+  def perform(%Job{
+        args: %{"op" => "unmute_conversation", "user_id" => user_id, "activity_id" => activity_id}
+      }) do
+    user = Pleroma.User.get_by_id(user_id)
+    activity = Pleroma.Activity.get_by_id(activity_id)
+    Pleroma.Web.CommonAPI.remove_mute(user, activity)
     :ok
   end
 end
diff --git a/test/user_test.exs b/test/user_test.exs
index 83c017ec5..d49afb35a 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -970,10 +970,18 @@ test "expiring" do
       {:ok, _user_relationships} = User.mute(user, muted_user, %{expires_in: 60})
       assert User.mutes?(user, muted_user)
 
+      worker = Pleroma.Workers.MuteExpireWorker
+      args = %{"op" => "unmute_user", "muter_id" => user.id, "mutee_id" => muted_user.id}
+
       assert_enqueued(
-        worker: Pleroma.Workers.MuteExpireWorker,
-        args: %{"op" => "unmute", "muter" => user.id, "mutee" => muted_user.id}
+        worker: worker,
+        args: args
       )
+
+      assert :ok = perform_job(worker, args)
+
+      refute User.mutes?(user, muted_user)
+      refute User.muted_notifications?(user, muted_user)
     end
 
     test "it unmutes users" do
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 800db9a20..7ceb7ec7f 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -3,7 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPITest do
+  use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
+
   alias Pleroma.Activity
   alias Pleroma.Chat
   alias Pleroma.Conversation.Participation
@@ -878,6 +880,22 @@ test "add mute", %{user: user, activity: activity} do
       assert CommonAPI.thread_muted?(user, activity)
     end
 
+    test "add expiring mute", %{user: user, activity: activity} do
+      {:ok, _} = CommonAPI.add_mute(user, activity, %{expires_in: 60})
+      assert CommonAPI.thread_muted?(user, activity)
+
+      worker = Pleroma.Workers.MuteExpireWorker
+      args = %{"op" => "unmute_conversation", "user_id" => user.id, "activity_id" => activity.id}
+
+      assert_enqueued(
+        worker: worker,
+        args: args
+      )
+
+      assert :ok = perform_job(worker, args)
+      refute CommonAPI.thread_muted?(user, activity)
+    end
+
     test "remove mute", %{user: user, activity: activity} do
       CommonAPI.add_mute(user, activity)
       {:ok, _} = CommonAPI.remove_mute(user, activity)

From 91b9985e1c0ef1766eb1705364e1aebe69d9b9bd Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Tue, 8 Sep 2020 15:26:06 +0300
Subject: [PATCH 012/208] Pass expires_in param from status controller

---
 lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index ecfa38489..da14c0b6c 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -285,9 +285,9 @@ def unbookmark(%{assigns: %{user: user}} = conn, %{id: id}) do
   end
 
   @doc "POST /api/v1/statuses/:id/mute"
-  def mute_conversation(%{assigns: %{user: user}} = conn, %{id: id}) do
+  def mute_conversation(%{assigns: %{user: user}, body_params: params} = conn, %{id: id}) do
     with %Activity{} = activity <- Activity.get_by_id(id),
-         {:ok, activity} <- CommonAPI.add_mute(user, activity) do
+         {:ok, activity} <- CommonAPI.add_mute(user, activity, params) do
       try_render(conn, "show.json", activity: activity, for: user, as: :activity)
     end
   end

From de2499e54b33a1746e5f6a5b79f1422d31c11570 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 9 Sep 2020 09:52:07 +0300
Subject: [PATCH 013/208] don't run update in tests

---
 lib/pleroma/stats.ex | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
index e5c9c668b..48afe901e 100644
--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@@ -23,7 +23,6 @@ def start_link(_) do
 
   @impl true
   def init(_args) do
-    if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo)
     {:ok, nil, {:continue, :calculate_stats}}
   end
 
@@ -32,11 +31,6 @@ def force_update do
     GenServer.call(__MODULE__, :force_update)
   end
 
-  @doc "Performs collect stats"
-  def do_collect do
-    GenServer.cast(__MODULE__, :run_update)
-  end
-
   @doc "Returns stats data"
   @spec get_stats() :: %{
           domain_count: non_neg_integer(),
@@ -111,7 +105,11 @@ def get_status_visibility_count(instance \\ nil) do
   @impl true
   def handle_continue(:calculate_stats, _) do
     stats = calculate_stat_data()
-    Process.send_after(self(), :run_update, @interval)
+
+    unless Pleroma.Config.get(:env) == :test do
+      Process.send_after(self(), :run_update, @interval)
+    end
+
     {:noreply, stats}
   end
 
@@ -126,13 +124,6 @@ def handle_call(:get_state, _from, state) do
     {:reply, state, state}
   end
 
-  @impl true
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
   @impl true
   def handle_info(:run_update, _) do
     new_stats = calculate_stat_data()

From 527afb813af6c64337d02ddf1a2f159fe557acbc Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Sun, 13 Sep 2020 12:23:45 +0300
Subject: [PATCH 014/208] Remove unused require

---
 lib/pleroma/workers/mute_expire_worker.ex | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex
index 622fdbadd..c8b44894e 100644
--- a/lib/pleroma/workers/mute_expire_worker.ex
+++ b/lib/pleroma/workers/mute_expire_worker.ex
@@ -5,8 +5,6 @@
 defmodule Pleroma.Workers.MuteExpireWorker do
   use Pleroma.Workers.WorkerHelper, queue: "mute_expire"
 
-  require Logger
-
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do
     muter = Pleroma.User.get_by_id(muter_id)

From 28d0986f839651df7d305da8932f7b5c48a4fbfb Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Sun, 20 Sep 2020 20:51:20 +0300
Subject: [PATCH 015/208] Refactor mutes removing in CommonAPI and User

---
 lib/pleroma/user.ex                       | 14 ++++++++++++++
 lib/pleroma/web/common_api/common_api.ex  | 18 +++++++++++++++++-
 lib/pleroma/workers/mute_expire_worker.ex |  8 ++------
 test/user_test.exs                        | 11 +++++++++++
 test/web/common_api/common_api_test.exs   |  6 ++++++
 5 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 83eb4d5ff..83e89a12c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1385,6 +1385,20 @@ def unmute(%User{} = muter, %User{} = mutee) do
     end
   end
 
+  def unmute(muter_id, mutee_id) do
+    with {:muter, %User{} = muter} <- {:muter, User.get_by_id(muter_id)},
+         {:mutee, %User{} = mutee} <- {:mutee, User.get_by_id(mutee_id)} do
+      unmute(muter, mutee)
+    else
+      {who, result} = error ->
+        Logger.warn(
+          "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}"
+        )
+
+        {:error, error}
+    end
+  end
+
   def subscribe(%User{} = subscriber, %User{} = target) do
     deny_follow_blocked = Config.get([:user, :deny_follow_blocked])
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index fca9246a5..aa4c6ddab 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -472,11 +472,27 @@ def add_mute(user, activity, params \\ %{}) do
     end
   end
 
-  def remove_mute(user, activity) do
+  def remove_mute(%User{} = user, %Activity{} = activity) do
     ThreadMute.remove_mute(user.id, activity.data["context"])
     {:ok, activity}
   end
 
+  def remove_mute(user_id, activity_id) do
+    with {:user, %User{} = user} <- {:user, User.get_by_id(user_id)},
+         {:activity, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)} do
+      remove_mute(user, activity)
+    else
+      {what, result} = error ->
+        Logger.warn(
+          "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{
+            activity_id
+          }"
+        )
+
+        {:error, error}
+    end
+  end
+
   def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}})
       when is_binary(context) do
     ThreadMute.exists?(user_id, context)
diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex
index c8b44894e..32a12ba85 100644
--- a/lib/pleroma/workers/mute_expire_worker.ex
+++ b/lib/pleroma/workers/mute_expire_worker.ex
@@ -7,18 +7,14 @@ defmodule Pleroma.Workers.MuteExpireWorker do
 
   @impl Oban.Worker
   def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do
-    muter = Pleroma.User.get_by_id(muter_id)
-    mutee = Pleroma.User.get_by_id(mutee_id)
-    Pleroma.User.unmute(muter, mutee)
+    Pleroma.User.unmute(muter_id, mutee_id)
     :ok
   end
 
   def perform(%Job{
         args: %{"op" => "unmute_conversation", "user_id" => user_id, "activity_id" => activity_id}
       }) do
-    user = Pleroma.User.get_by_id(user_id)
-    activity = Pleroma.Activity.get_by_id(activity_id)
-    Pleroma.Web.CommonAPI.remove_mute(user, activity)
+    Pleroma.Web.CommonAPI.remove_mute(user_id, activity_id)
     :ok
   end
 end
diff --git a/test/user_test.exs b/test/user_test.exs
index ce0d4d38b..79c8b76b8 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1034,6 +1034,17 @@ test "it unmutes users" do
       refute User.muted_notifications?(user, muted_user)
     end
 
+    test "it unmutes users by id" do
+      user = insert(:user)
+      muted_user = insert(:user)
+
+      {:ok, _user_relationships} = User.mute(user, muted_user)
+      {:ok, _user_mute} = User.unmute(user.id, muted_user.id)
+
+      refute User.mutes?(user, muted_user)
+      refute User.muted_notifications?(user, muted_user)
+    end
+
     test "it mutes user without notifications" do
       user = insert(:user)
       muted_user = insert(:user)
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index bf4353a57..45ab1e0bd 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -913,6 +913,12 @@ test "remove mute", %{user: user, activity: activity} do
       refute CommonAPI.thread_muted?(user, activity)
     end
 
+    test "remove mute by ids", %{user: user, activity: activity} do
+      CommonAPI.add_mute(user, activity)
+      {:ok, _} = CommonAPI.remove_mute(user.id, activity.id)
+      refute CommonAPI.thread_muted?(user, activity)
+    end
+
     test "check that mutes can't be duplicate", %{user: user, activity: activity} do
       CommonAPI.add_mute(user, activity)
       {:error, _} = CommonAPI.add_mute(user, activity)

From b0bd81ef7187ddf5b4e6cfbc1780fc60b65798c6 Mon Sep 17 00:00:00 2001
From: Roman Chvanikov <chvanikoff@pm.me>
Date: Sun, 20 Sep 2020 20:58:32 +0300
Subject: [PATCH 016/208] Update CHANGELOG

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5a7e27fd3..9fd5f9efe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Experimental websocket-based federation between Pleroma instances.
+- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
 
 ### Changed
 

From 7efadc3cbd46369e960f31c33a2c555f718ca8c5 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 1 Oct 2020 21:34:45 +0300
Subject: [PATCH 017/208] No auth check in OStatusController, even on
 non-federating instances.

---
 lib/pleroma/web/ostatus/ostatus_controller.ex | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index de1b0b3f0..8646d2c1c 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -16,10 +16,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.Metadata.PlayerView
   alias Pleroma.Web.Router
 
-  plug(Pleroma.Plugs.EnsureAuthenticatedPlug,
-    unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-  )
-
   plug(
     RateLimiter,
     [name: :ap_routes, params: ["uuid"]] when action in [:object, :activity]

From 0d575735bfd280b878bdecc6d018d8cca23ad09f Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 1 Oct 2020 21:41:22 +0300
Subject: [PATCH 018/208] No auth check in UserController.feed_redirect/2, even
 on non-federating instances.

---
 lib/pleroma/web/feed/user_controller.ex | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 71eb1ea7e..09ecdedb4 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -23,12 +23,7 @@ def feed_redirect(%{assigns: %{format: "html"}} = conn, %{"nickname" => nickname
 
   def feed_redirect(%{assigns: %{format: format}} = conn, _params)
       when format in ["json", "activity+json"] do
-    with %{halted: false} = conn <-
-           Pleroma.Plugs.EnsureAuthenticatedPlug.call(conn,
-             unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-           ) do
-      ActivityPubController.call(conn, :user)
-    end
+    ActivityPubController.call(conn, :user)
   end
 
   def feed_redirect(conn, %{"nickname" => nickname}) do

From f6024252ae8601d41bea943bb3cae5c656416eb9 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 2 Oct 2020 22:18:02 +0300
Subject: [PATCH 019/208] [#3053] No auth check in StaticFEController, even on
 non-federating instances. Adjusted tests.

---
 lib/pleroma/web/feed/tag_controller.ex        |   4 +-
 lib/pleroma/web/feed/user_controller.ex       |   4 +-
 lib/pleroma/web/router.ex                     |  11 +-
 .../web/static_fe/static_fe_controller.ex     | 176 +++++++++---------
 test/support/conn_case.ex                     |  22 ---
 .../activity_pub_controller_test.exs          |  19 ++
 test/web/feed/user_controller_test.exs        |  12 +-
 test/web/ostatus/ostatus_controller_test.exs  |  24 +--
 .../static_fe/static_fe_controller_test.exs   |  24 ++-
 9 files changed, 162 insertions(+), 134 deletions(-)

diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index 93a8294b7..c348b32c2 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -10,14 +10,14 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.Feed.FeedView
 
   def feed(conn, params) do
-    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+    unless Config.restrict_unauthenticated_access?(:activities, :local) do
       render_feed(conn, params)
     else
       render_error(conn, :not_found, "Not found")
     end
   end
 
-  def render_feed(conn, %{"tag" => raw_tag} = params) do
+  defp render_feed(conn, %{"tag" => raw_tag} = params) do
     {format, tag} = parse_tag(raw_tag)
 
     activities =
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 09ecdedb4..5fbcd82d7 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -40,11 +40,11 @@ def feed(conn, params) do
     end
   end
 
-  def render_feed(conn, %{"nickname" => nickname} = params) do
+  defp render_feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
-      if format in ["rss", "atom"] do
+      if format in ["atom", "rss"] do
         format
       else
         "atom"
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 42a9db21d..e0e92549f 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -561,12 +561,17 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Plugs.StaticFEPlug)
   end
 
+  pipeline :ostatus_no_html do
+    plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"])
+  end
+
   pipeline :oembed do
     plug(:accepts, ["json", "xml"])
   end
 
   scope "/", Pleroma.Web do
-    pipe_through([:ostatus, :http_signature])
+    # Note: no authentication plugs, all endpoints below should only yield public objects
+    pipe_through(:ostatus)
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
@@ -579,6 +584,10 @@ defmodule Pleroma.Web.Router do
 
     get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
+  end
+
+  scope "/", Pleroma.Web do
+    pipe_through(:ostatus_no_html)
 
     get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed)
   end
diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index a7a891b13..b1c62f5b0 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -17,12 +17,95 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
   plug(:put_view, Pleroma.Web.StaticFE.StaticFEView)
   plug(:assign_id)
 
-  plug(Pleroma.Plugs.EnsureAuthenticatedPlug,
-    unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-  )
-
   @page_keys ["max_id", "min_id", "limit", "since_id", "order"]
 
+  @doc "Renders requested local public activity"
+  def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
+    with %Activity{local: true} = activity <-
+           Activity.get_by_id_with_object(notice_id),
+         true <- Visibility.is_public?(activity.object),
+         %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
+      meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
+
+      timeline =
+        activity.object.data["context"]
+        |> ActivityPub.fetch_activities_for_context(%{})
+        |> Enum.reverse()
+        |> Enum.map(&represent(&1, &1.object.id == activity.object.id))
+
+      render(conn, "conversation.html", %{activities: timeline, meta: meta})
+    else
+      %Activity{object: %Object{data: data}} ->
+        conn
+        |> put_status(:found)
+        |> redirect(external: data["url"] || data["external_url"] || data["id"])
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
+  @doc "Renders public activities of requested user"
+  def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
+    case User.get_cached_by_nickname_or_id(username_or_id) do
+      %User{} = user ->
+        meta = Metadata.build_tags(%{user: user})
+
+        params =
+          params
+          |> Map.take(@page_keys)
+          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
+
+        timeline =
+          user
+          |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
+          |> Enum.map(&represent/1)
+
+        prev_page_id =
+          (params["min_id"] || params["max_id"]) &&
+            List.first(timeline) && List.first(timeline).id
+
+        next_page_id = List.last(timeline) && List.last(timeline).id
+
+        render(conn, "profile.html", %{
+          user: User.sanitize_html(user),
+          timeline: timeline,
+          prev_page_id: prev_page_id,
+          next_page_id: next_page_id,
+          meta: meta
+        })
+
+      _ ->
+        not_found(conn, "User not found.")
+    end
+  end
+
+  def show(%{assigns: %{object_id: _}} = conn, _params) do
+    url = Helpers.url(conn) <> conn.request_path
+
+    case Activity.get_create_by_object_ap_id_with_object(url) do
+      %Activity{} = activity ->
+        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
+        redirect(conn, to: to)
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
+  def show(%{assigns: %{activity_id: _}} = conn, _params) do
+    url = Helpers.url(conn) <> conn.request_path
+
+    case Activity.get_by_ap_id(url) do
+      %Activity{} = activity ->
+        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
+        redirect(conn, to: to)
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
   defp get_title(%Object{data: %{"name" => name}}) when is_binary(name),
     do: name
 
@@ -81,91 +164,6 @@ defp represent(%Activity{object: %Object{data: data}} = activity, selected) do
     }
   end
 
-  def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
-    with %Activity{local: true} = activity <-
-           Activity.get_by_id_with_object(notice_id),
-         true <- Visibility.is_public?(activity.object),
-         %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
-      meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
-
-      timeline =
-        activity.object.data["context"]
-        |> ActivityPub.fetch_activities_for_context(%{})
-        |> Enum.reverse()
-        |> Enum.map(&represent(&1, &1.object.id == activity.object.id))
-
-      render(conn, "conversation.html", %{activities: timeline, meta: meta})
-    else
-      %Activity{object: %Object{data: data}} ->
-        conn
-        |> put_status(:found)
-        |> redirect(external: data["url"] || data["external_url"] || data["id"])
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
-  def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
-    case User.get_cached_by_nickname_or_id(username_or_id) do
-      %User{} = user ->
-        meta = Metadata.build_tags(%{user: user})
-
-        params =
-          params
-          |> Map.take(@page_keys)
-          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
-
-        timeline =
-          user
-          |> ActivityPub.fetch_user_activities(nil, params)
-          |> Enum.map(&represent/1)
-
-        prev_page_id =
-          (params["min_id"] || params["max_id"]) &&
-            List.first(timeline) && List.first(timeline).id
-
-        next_page_id = List.last(timeline) && List.last(timeline).id
-
-        render(conn, "profile.html", %{
-          user: User.sanitize_html(user),
-          timeline: timeline,
-          prev_page_id: prev_page_id,
-          next_page_id: next_page_id,
-          meta: meta
-        })
-
-      _ ->
-        not_found(conn, "User not found.")
-    end
-  end
-
-  def show(%{assigns: %{object_id: _}} = conn, _params) do
-    url = Helpers.url(conn) <> conn.request_path
-
-    case Activity.get_create_by_object_ap_id_with_object(url) do
-      %Activity{} = activity ->
-        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
-        redirect(conn, to: to)
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
-  def show(%{assigns: %{activity_id: _}} = conn, _params) do
-    url = Helpers.url(conn) <> conn.request_path
-
-    case Activity.get_by_ap_id(url) do
-      %Activity{} = activity ->
-        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
-        redirect(conn, to: to)
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
   defp assign_id(%{path_info: ["notice", notice_id]} = conn, _opts),
     do: assign(conn, :notice_id, notice_id)
 
diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex
index 7ef681258..7b28d70e7 100644
--- a/test/support/conn_case.ex
+++ b/test/support/conn_case.ex
@@ -111,28 +111,6 @@ defp json_response_and_validate_schema(
       defp json_response_and_validate_schema(conn, _status) do
         flunk("Response schema not found for #{conn.method} #{conn.request_path} #{conn.status}")
       end
-
-      defp ensure_federating_or_authenticated(conn, url, user) do
-        initial_setting = Config.get([:instance, :federating])
-        on_exit(fn -> Config.put([:instance, :federating], initial_setting) end)
-
-        Config.put([:instance, :federating], false)
-
-        conn
-        |> get(url)
-        |> response(403)
-
-        conn
-        |> assign(:user, user)
-        |> get(url)
-        |> response(200)
-
-        Config.put([:instance, :federating], true)
-
-        conn
-        |> get(url)
-        |> response(200)
-      end
     end
   end
 
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index 0517571f2..ab57b6523 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -33,6 +33,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
   setup do: clear_config([:instance, :federating], true)
 
+  defp ensure_federating_or_authenticated(conn, url, user) do
+    Config.put([:instance, :federating], false)
+
+    conn
+    |> get(url)
+    |> response(403)
+
+    conn
+    |> assign(:user, user)
+    |> get(url)
+    |> response(200)
+
+    Config.put([:instance, :federating], true)
+
+    conn
+    |> get(url)
+    |> response(200)
+  end
+
   describe "/relay" do
     setup do: clear_config([:instance, :allow_relay])
 
diff --git a/test/web/feed/user_controller_test.exs b/test/web/feed/user_controller_test.exs
index 9a5610baa..7383e82cc 100644
--- a/test/web/feed/user_controller_test.exs
+++ b/test/web/feed/user_controller_test.exs
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
-  setup do: clear_config([:instance, :federating], true)
+  setup do: clear_config([:static_fe, :enabled], false)
 
   describe "feed" do
     setup do: clear_config([:feed])
@@ -192,6 +192,16 @@ test "returns 404 when the user is remote", %{conn: conn} do
              |> get(user_feed_path(conn, :feed, user.nickname))
              |> response(404)
     end
+
+    test "does not require authentication on non-federating instances", %{conn: conn} do
+      clear_config([:instance, :federating], false)
+      user = insert(:user)
+
+      conn
+      |> put_req_header("accept", "application/rss+xml")
+      |> get("/users/#{user.nickname}/feed.rss")
+      |> response(200)
+    end
   end
 
   # Note: see ActivityPubControllerTest for JSON format tests
diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs
index ee498f4b5..65b2c22db 100644
--- a/test/web/ostatus/ostatus_controller_test.exs
+++ b/test/web/ostatus/ostatus_controller_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -21,7 +20,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
     :ok
   end
 
-  setup do: clear_config([:instance, :federating], true)
+  setup do: clear_config([:static_fe, :enabled], false)
 
   describe "Mastodon compatibility routes" do
     setup %{conn: conn} do
@@ -215,15 +214,16 @@ test "404s a non-existing notice", %{conn: conn} do
       assert response(conn, 404)
     end
 
-    test "it requires authentication if instance is NOT federating", %{
+    test "does not require authentication on non-federating instances", %{
       conn: conn
     } do
-      user = insert(:user)
+      clear_config([:instance, :federating], false)
       note_activity = insert(:note_activity)
 
-      conn = put_req_header(conn, "accept", "text/html")
-
-      ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}", user)
+      conn
+      |> put_req_header("accept", "text/html")
+      |> get("/notice/#{note_activity.id}")
+      |> response(200)
     end
   end
 
@@ -325,14 +325,16 @@ test "404s when attachment isn't audio or video", %{conn: conn} do
       |> response(404)
     end
 
-    test "it requires authentication if instance is NOT federating", %{
+    test "does not require authentication on non-federating instances", %{
       conn: conn,
       note_activity: note_activity
     } do
-      user = insert(:user)
-      conn = put_req_header(conn, "accept", "text/html")
+      clear_config([:instance, :federating], false)
 
-      ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}/embed_player", user)
+      conn
+      |> put_req_header("accept", "text/html")
+      |> get("/notice/#{note_activity.id}/embed_player")
+      |> response(200)
     end
   end
 end
diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs
index 1598bf675..bab0b0a7b 100644
--- a/test/web/static_fe/static_fe_controller_test.exs
+++ b/test/web/static_fe/static_fe_controller_test.exs
@@ -2,14 +2,12 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do
   use Pleroma.Web.ConnCase
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
   setup_all do: clear_config([:static_fe, :enabled], true)
-  setup do: clear_config([:instance, :federating], true)
 
   setup %{conn: conn} do
     conn = put_req_header(conn, "accept", "text/html")
@@ -70,8 +68,15 @@ test "pagination, page 2", %{conn: conn, user: user} do
       refute html =~ ">test29<"
     end
 
-    test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}", user)
+    test "does not require authentication on non-federating instances", %{
+      conn: conn,
+      user: user
+    } do
+      clear_config([:instance, :federating], false)
+
+      conn = get(conn, "/users/#{user.nickname}")
+
+      assert html_response(conn, 200) =~ user.nickname
     end
   end
 
@@ -183,10 +188,17 @@ test "302 for remote cached status", %{conn: conn, user: user} do
       assert html_response(conn, 302) =~ "redirected"
     end
 
-    test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do
+    test "does not require authentication on non-federating instances", %{
+      conn: conn,
+      user: user
+    } do
+      clear_config([:instance, :federating], false)
+
       {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"})
 
-      ensure_federating_or_authenticated(conn, "/notice/#{activity.id}", user)
+      conn = get(conn, "/notice/#{activity.id}")
+
+      assert html_response(conn, 200) =~ "testing a thing!"
     end
   end
 end

From 858dbe43607b31cba8c319755c92099cd2c9f5c1 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 2 Oct 2020 21:47:43 +0200
Subject: [PATCH 020/208] docs/ap_extensions.md: document uploadMedia

---
 docs/ap_extensions.md | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/docs/ap_extensions.md b/docs/ap_extensions.md
index c4550a1ac..7d2956d6d 100644
--- a/docs/ap_extensions.md
+++ b/docs/ap_extensions.md
@@ -1,4 +1,26 @@
-# ChatMessages
+# AP Extensions
+## Actor endpoints
+
+The following endpoints are additionally present into our actors.
+
+- `oauthRegistrationEndpoint`
+- `uploadMedia`
+
+### uploadMedia
+
+Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
+
+Content-Type: multipart/form-data
+
+Parameters:
+- (required) `file`: The file being uploaded
+- (optionnal) `description`: A plain-text description of the media, for accessibility purposes.
+
+Response: HTTP 201 Created with the object into the body, no `Location` header provided as it doesn't have an `id`
+
+The object given in the reponse should then be inserted into an Object's `attachment` field.
+
+## ChatMessages
 
 ChatMessages are the messages sent in 1-on-1 chats. They are similar to
 `Note`s, but the addresing is done by having a single AP actor in the `to`

From ac6e0f6684ecce7d46ce60ee68da6353cd949c0f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 5 Oct 2020 17:18:58 +0200
Subject: [PATCH 021/208] docs/ap_extensions.md: document
 oauthRegistrationEndpoint

---
 docs/ap_extensions.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/ap_extensions.md b/docs/ap_extensions.md
index 7d2956d6d..f37167cdc 100644
--- a/docs/ap_extensions.md
+++ b/docs/ap_extensions.md
@@ -6,6 +6,12 @@ The following endpoints are additionally present into our actors.
 - `oauthRegistrationEndpoint`
 - `uploadMedia`
 
+### oauthRegistrationEndpoint
+
+Points to MastodonAPI `/api/v1/apps` for now.
+
+See <https://docs.joinmastodon.org/methods/apps/>
+
 ### uploadMedia
 
 Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>

From 4c229d7fccc7cff028d5fe9815944847b5d3c9e7 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 5 Oct 2020 17:32:10 +0200
Subject: [PATCH 022/208] docs/ap_extensions.md: Add JSON-LD full names

---
 docs/ap_extensions.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/ap_extensions.md b/docs/ap_extensions.md
index f37167cdc..3d1caeb3e 100644
--- a/docs/ap_extensions.md
+++ b/docs/ap_extensions.md
@@ -3,8 +3,8 @@
 
 The following endpoints are additionally present into our actors.
 
-- `oauthRegistrationEndpoint`
-- `uploadMedia`
+- `oauthRegistrationEndpoint` (`http://litepub.social/ns#oauthRegistrationEndpoint`)
+- `uploadMedia` (`https://www.w3.org/ns/activitystreams#uploadMedia`)
 
 ### oauthRegistrationEndpoint
 
@@ -14,7 +14,7 @@ See <https://docs.joinmastodon.org/methods/apps/>
 
 ### uploadMedia
 
-Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
+Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>, it is part of the ActivityStreams namespace because it used to be part of the ActivityPub specification and got removed from it.
 
 Content-Type: multipart/form-data
 
@@ -28,12 +28,14 @@ The object given in the reponse should then be inserted into an Object's `attach
 
 ## ChatMessages
 
-ChatMessages are the messages sent in 1-on-1 chats. They are similar to
+`ChatMessage`s are the messages sent in 1-on-1 chats. They are similar to
 `Note`s, but the addresing is done by having a single AP actor in the `to`
 field. Addressing multiple actors is not allowed. These messages are always
 private, there is no public version of them. They are created with a `Create`
 activity.
 
+They are part of the `litepub` namespace as `http://litepub.social/ns#ChatMessage`.
+
 Example:
 
 ```json

From f497eb034df6647fef9086a6e2ef03e61e2efc47 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 5 Oct 2020 21:11:00 +0200
Subject: [PATCH 023/208] activity_pub_controller.ex: Remove unused @doc block

[ci skip]
---
 .../web/activity_pub/activity_pub_controller.ex     | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 732c44271..832155643 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -523,19 +523,6 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
     {new_user, for_user}
   end
 
-  @doc """
-  Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
-
-  Parameters:
-  - (required) `file`: data of the media
-  - (optionnal) `description`: description of the media, intended for accessibility
-
-  Response:
-  - HTTP Code: 201 Created
-  - HTTP Body: ActivityPub object to be inserted into another's `attachment` field
-
-  Note: Will not point to a URL with a `Location` header because no standalone Activity has been created.
-  """
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

From 094edde7c4ddf65f46e5d692a5ef5b859587d1e1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 5 Oct 2020 23:48:00 +0300
Subject: [PATCH 024/208] [#3053] Unauthenticated access control for
 OStatus-related controllers and ActivityPubController (base actions: :user,
 :object, :activity). Tests adjustments.

---
 .../activity_pub/activity_pub_controller.ex   | 56 +++++++++-------
 lib/pleroma/web/activity_pub/visibility.ex    | 39 ++++++++---
 lib/pleroma/web/feed/tag_controller.ex        | 15 +++--
 lib/pleroma/web/feed/user_controller.ex       | 19 +++---
 lib/pleroma/web/ostatus/ostatus_controller.ex | 26 +++----
 lib/pleroma/web/router.ex                     | 52 ++++++++++----
 .../web/static_fe/static_fe_controller.ex     | 48 ++++++-------
 .../activity_pub_controller_test.exs          | 67 -------------------
 test/web/feed/tag_controller_test.exs         | 13 ++--
 9 files changed, 159 insertions(+), 176 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 732c44271..c78edfb4c 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -32,17 +32,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers]
 
+  # Note: :following and :followers must be served even without authentication (as via :api)
+  @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media]
+
+  # Always accessible actions (must perform entity accessibility checks)
+  @no_auth_no_federation_actions [:user, :activity, :object]
+
+  @authenticated_or_federating_actions @federating_only_actions ++
+                                         @auth_only_actions ++ @no_auth_no_federation_actions
+
   plug(FederatingPlug when action in @federating_only_actions)
 
+  plug(EnsureAuthenticatedPlug when action in @auth_only_actions)
+
   plug(
     EnsureAuthenticatedPlug,
-    [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions
-  )
-
-  # Note: :following and :followers must be served even without authentication (as via :api)
-  plug(
-    EnsureAuthenticatedPlug
-    when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
+    [unless_func: &FederatingPlug.federating?/1]
+    when action not in @authenticated_or_federating_actions
   )
 
   plug(
@@ -66,21 +72,22 @@ defp relay_active?(conn, _) do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)},
          {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("user.json", %{user: user})
     else
-      nil -> {:error, :not_found}
-      %{local: false} -> {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)},
+         {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -88,25 +95,15 @@ def object(conn, _) do
       |> put_view(ObjectView)
       |> render("object.json", object: object)
     else
-      {:public?, false} ->
-        {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
-  def track_object_fetch(conn, nil), do: conn
-
-  def track_object_fetch(conn, object_id) do
-    with %{assigns: %{user: %User{id: user_id}}} <- conn do
-      Delivery.create(object_id, user_id)
-    end
-
-    conn
-  end
-
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -114,8 +111,7 @@ def activity(conn, _params) do
       |> put_view(ObjectView)
       |> render("object.json", object: activity)
     else
-      {:public?, false} -> {:error, :not_found}
-      nil -> {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
@@ -550,4 +546,14 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} =
       |> json(object.data)
     end
   end
+
+  def track_object_fetch(conn, nil), do: conn
+
+  def track_object_fetch(conn, object_id) do
+    with %{assigns: %{user: %User{id: user_id}}} <- conn do
+      Delivery.create(object_id, user_id)
+    end
+
+    conn
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
index 5c349bb7a..76bd54a42 100644
--- a/lib/pleroma/web/activity_pub/visibility.ex
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -44,29 +44,30 @@ def is_direct?(activity) do
   def is_list?(%{data: %{"listMessage" => _}}), do: true
   def is_list?(_), do: false
 
-  @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean()
-  def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
+  @spec visible_for_user?(Activity.t() | nil, User.t() | nil) :: boolean()
+  def visible_for_user?(%Activity{actor: ap_id}, %User{ap_id: ap_id}), do: true
 
   def visible_for_user?(nil, _), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
+  def visible_for_user?(%Activity{data: %{"listMessage" => _}}, nil), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
+  def visible_for_user?(
+        %Activity{data: %{"listMessage" => list_ap_id}} = activity,
+        %User{} = user
+      ) do
     user.ap_id in activity.data["to"] ||
       list_ap_id
       |> Pleroma.List.get_by_ap_id()
       |> Pleroma.List.member?(user)
   end
 
-  def visible_for_user?(%{local: local} = activity, nil) do
-    cfg_key = if local, do: :local, else: :remote
-
-    if Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key),
+  def visible_for_user?(%Activity{} = activity, nil) do
+    if restrict_unauthenticated_access?(activity),
       do: false,
       else: is_public?(activity)
   end
 
-  def visible_for_user?(activity, user) do
+  def visible_for_user?(%Activity{} = activity, user) do
     x = [user.ap_id | User.following(user)]
     y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
     is_public?(activity) || Enum.any?(x, &(&1 in y))
@@ -82,6 +83,26 @@ def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do
     result
   end
 
+  def restrict_unauthenticated_access?(%Activity{local: local}) do
+    restrict_unauthenticated_access_to_activity?(local)
+  end
+
+  def restrict_unauthenticated_access?(%Object{} = object) do
+    object
+    |> Object.local?()
+    |> restrict_unauthenticated_access_to_activity?()
+  end
+
+  def restrict_unauthenticated_access?(%User{} = user) do
+    User.visible_for(user, _reading_user = nil)
+  end
+
+  defp restrict_unauthenticated_access_to_activity?(local?) when is_boolean(local?) do
+    cfg_key = if local?, do: :local, else: :remote
+
+    Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key)
+  end
+
   def get_visibility(object) do
     to = object.data["to"] || []
     cc = object.data["cc"] || []
diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index c348b32c2..218cdbdf3 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.Feed.FeedView
 
   def feed(conn, params) do
-    unless Config.restrict_unauthenticated_access?(:activities, :local) do
+    if Config.get!([:instance, :public]) do
       render_feed(conn, params)
     else
       render_error(conn, :not_found, "Not found")
@@ -36,12 +36,13 @@ defp render_feed(conn, %{"tag" => raw_tag} = params) do
   end
 
   @spec parse_tag(binary() | any()) :: {format :: String.t(), tag :: String.t()}
-  defp parse_tag(raw_tag) when is_binary(raw_tag) do
-    case Enum.reverse(String.split(raw_tag, ".")) do
-      [format | tag] when format in ["atom", "rss"] -> {format, Enum.join(tag, ".")}
-      _ -> {"rss", raw_tag}
+  defp parse_tag(raw_tag) do
+    case is_binary(raw_tag) && Enum.reverse(String.split(raw_tag, ".")) do
+      [format | tag] when format in ["rss", "atom"] ->
+        {format, Enum.join(tag, ".")}
+
+      _ ->
+        {"atom", raw_tag}
     end
   end
-
-  defp parse_tag(raw_tag), do: {"rss", raw_tag}
 end
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 5fbcd82d7..f1d2bb7be 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -6,6 +6,8 @@ defmodule Pleroma.Web.Feed.UserController do
   use Pleroma.Web, :controller
 
   alias Fallback.RedirectController
+
+  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.ActivityPubController
@@ -32,15 +34,7 @@ def feed_redirect(conn, %{"nickname" => nickname}) do
     end
   end
 
-  def feed(conn, params) do
-    unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do
-      render_feed(conn, params)
-    else
-      errors(conn, {:error, :not_found})
-    end
-  end
-
-  defp render_feed(conn, %{"nickname" => nickname} = params) do
+  def feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
@@ -50,7 +44,8 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do
         "atom"
       end
 
-    with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do
+    with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)},
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do
       activities =
         %{
           type: ["Create"],
@@ -65,7 +60,7 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do
       |> render("user.#{format}",
         user: user,
         activities: activities,
-        feed_config: Pleroma.Config.get([:feed])
+        feed_config: Config.get([:feed])
       )
     end
   end
@@ -77,6 +72,8 @@ def errors(conn, {:error, :not_found}) do
   def errors(conn, {:fetch_user, %User{local: false}}), do: errors(conn, {:error, :not_found})
   def errors(conn, {:fetch_user, nil}), do: errors(conn, {:error, :not_found})
 
+  def errors(conn, {:visibility, _}), do: errors(conn, {:error, :not_found})
+
   def errors(conn, _) do
     render_error(conn, :internal_server_error, "Something went wrong")
   end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 8646d2c1c..b4dc2a87f 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -33,16 +33,15 @@ def object(%{assigns: %{format: format}} = conn, _params)
     ActivityPubController.call(conn, :object)
   end
 
-  def object(%{assigns: %{format: format}} = conn, _params) do
+  def object(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <-
            {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
-      case format do
-        _ -> redirect(conn, to: "/notice/#{activity.id}")
-      end
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+      redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -55,15 +54,14 @@ def activity(%{assigns: %{format: format}} = conn, _params)
     ActivityPubController.call(conn, :activity)
   end
 
-  def activity(%{assigns: %{format: format}} = conn, _params) do
+  def activity(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
-      case format do
-        _ -> redirect(conn, to: "/notice/#{activity.id}")
-      end
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+      redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -74,6 +72,7 @@ def activity(%{assigns: %{format: format}} = conn, _params) do
   def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
     with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       cond do
         format in ["json", "activity+json"] ->
@@ -101,7 +100,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
           RedirectController.redirector(conn, nil)
       end
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         conn
         |> put_status(404)
         |> RedirectController.redirector(nil, 404)
@@ -115,6 +114,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
   def notice_player(conn, %{"id" => id}) do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %Object{} = object <- Object.normalize(activity),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e0e92549f..6439a1c39 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -5,6 +5,14 @@
 defmodule Pleroma.Web.Router do
   use Pleroma.Web, :router
 
+  pipeline :accepts_html do
+    plug(:accepts, ["html"])
+  end
+
+  pipeline :accepts_xml_rss_atom do
+    plug(:accepts, ["xml", "rss", "atom"])
+  end
+
   pipeline :browser do
     plug(:accepts, ["html"])
     plug(:fetch_session)
@@ -556,39 +564,55 @@ defmodule Pleroma.Web.Router do
     )
   end
 
-  pipeline :ostatus do
+  pipeline :ostatus_html_json do
+    plug(:accepts, ["html", "activity+json", "json"])
+    plug(Pleroma.Plugs.StaticFEPlug)
+  end
+
+  pipeline :ostatus_html_xml do
+    plug(:accepts, ["html", "xml", "rss", "atom"])
+    plug(Pleroma.Plugs.StaticFEPlug)
+  end
+
+  pipeline :ostatus_html_xml_json do
     plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
     plug(Pleroma.Plugs.StaticFEPlug)
   end
 
-  pipeline :ostatus_no_html do
-    plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"])
-  end
-
-  pipeline :oembed do
-    plug(:accepts, ["json", "xml"])
-  end
-
   scope "/", Pleroma.Web do
-    # Note: no authentication plugs, all endpoints below should only yield public objects
-    pipe_through(:ostatus)
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_json)
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
     get("/notice/:id", OStatus.OStatusController, :notice)
-    get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player)
 
     # Mastodon compatibility routes
     get("/users/:nickname/statuses/:id", OStatus.OStatusController, :object)
     get("/users/:nickname/statuses/:id/activity", OStatus.OStatusController, :activity)
+  end
 
-    get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
+  scope "/", Pleroma.Web do
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_xml_json)
+
+    # Note: for json format responds with user profile (not user feed)
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
   end
 
   scope "/", Pleroma.Web do
-    pipe_through(:ostatus_no_html)
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_xml)
+    get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
+  end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:accepts_html)
+    get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player)
+  end
+
+  scope "/", Pleroma.Web do
+    pipe_through(:accepts_xml_rss_atom)
     get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed)
   end
 
diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index b1c62f5b0..76b82589f 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -24,6 +24,7 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     with %Activity{local: true} = activity <-
            Activity.get_by_id_with_object(notice_id),
          true <- Visibility.is_public?(activity.object),
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
       meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
 
@@ -47,34 +48,35 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
 
   @doc "Renders public activities of requested user"
   def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
-    case User.get_cached_by_nickname_or_id(username_or_id) do
-      %User{} = user ->
-        meta = Metadata.build_tags(%{user: user})
+    with {_, %User{local: true} = user} <-
+           {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)},
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do
+      meta = Metadata.build_tags(%{user: user})
 
-        params =
-          params
-          |> Map.take(@page_keys)
-          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
+      params =
+        params
+        |> Map.take(@page_keys)
+        |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
 
-        timeline =
-          user
-          |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
-          |> Enum.map(&represent/1)
+      timeline =
+        user
+        |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
+        |> Enum.map(&represent/1)
 
-        prev_page_id =
-          (params["min_id"] || params["max_id"]) &&
-            List.first(timeline) && List.first(timeline).id
+      prev_page_id =
+        (params["min_id"] || params["max_id"]) &&
+          List.first(timeline) && List.first(timeline).id
 
-        next_page_id = List.last(timeline) && List.last(timeline).id
-
-        render(conn, "profile.html", %{
-          user: User.sanitize_html(user),
-          timeline: timeline,
-          prev_page_id: prev_page_id,
-          next_page_id: next_page_id,
-          meta: meta
-        })
+      next_page_id = List.last(timeline) && List.last(timeline).id
 
+      render(conn, "profile.html", %{
+        user: User.sanitize_html(user),
+        timeline: timeline,
+        prev_page_id: prev_page_id,
+        next_page_id: next_page_id,
+        meta: meta
+      })
+    else
       _ ->
         not_found(conn, "User not found.")
     end
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index ab57b6523..9ec13b21f 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -33,25 +33,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
   setup do: clear_config([:instance, :federating], true)
 
-  defp ensure_federating_or_authenticated(conn, url, user) do
-    Config.put([:instance, :federating], false)
-
-    conn
-    |> get(url)
-    |> response(403)
-
-    conn
-    |> assign(:user, user)
-    |> get(url)
-    |> response(200)
-
-    Config.put([:instance, :federating], true)
-
-    conn
-    |> get(url)
-    |> response(200)
-  end
-
   describe "/relay" do
     setup do: clear_config([:instance, :allow_relay])
 
@@ -175,21 +156,6 @@ test "it returns error when user is not found", %{conn: conn} do
 
       assert response == "Not found"
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-
-      conn =
-        put_req_header(
-          conn,
-          "accept",
-          "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
-        )
-
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}.json", user)
-    end
   end
 
   describe "mastodon compatibility routes" do
@@ -357,18 +323,6 @@ test "cached purged after object deletion", %{conn: conn} do
 
       assert "Not found" == json_response(conn2, :not_found)
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      note = insert(:note)
-      uuid = String.split(note.data["id"], "/") |> List.last()
-
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/objects/#{uuid}", user)
-    end
   end
 
   describe "/activities/:uuid" do
@@ -440,18 +394,6 @@ test "cached purged after activity deletion", %{conn: conn} do
 
       assert "Not found" == json_response(conn2, :not_found)
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      activity = insert(:note_activity)
-      uuid = String.split(activity.data["id"], "/") |> List.last()
-
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/activities/#{uuid}", user)
-    end
   end
 
   describe "/inbox" do
@@ -912,15 +854,6 @@ test "it returns an announce activity in a collection", %{conn: conn} do
 
       assert response(conn, 200) =~ announce_activity.data["object"]
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}/outbox", user)
-    end
   end
 
   describe "POST /users/:nickname/outbox (C2S)" do
diff --git a/test/web/feed/tag_controller_test.exs b/test/web/feed/tag_controller_test.exs
index 868e40965..e4084b0e5 100644
--- a/test/web/feed/tag_controller_test.exs
+++ b/test/web/feed/tag_controller_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do
   import Pleroma.Factory
   import SweetXml
 
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Feed.FeedView
@@ -15,7 +16,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do
   setup do: clear_config([:feed])
 
   test "gets a feed (ATOM)", %{conn: conn} do
-    Pleroma.Config.put(
+    Config.put(
       [:feed, :post_title],
       %{max_length: 25, omission: "..."}
     )
@@ -82,7 +83,7 @@ test "gets a feed (ATOM)", %{conn: conn} do
   end
 
   test "gets a feed (RSS)", %{conn: conn} do
-    Pleroma.Config.put(
+    Config.put(
       [:feed, :post_title],
       %{max_length: 25, omission: "..."}
     )
@@ -157,7 +158,7 @@ test "gets a feed (RSS)", %{conn: conn} do
     response =
       conn
       |> put_req_header("accept", "application/rss+xml")
-      |> get(tag_feed_path(conn, :feed, "pleromaart"))
+      |> get(tag_feed_path(conn, :feed, "pleromaart.rss"))
       |> response(200)
 
     xml = parse(response)
@@ -183,14 +184,12 @@ test "gets a feed (RSS)", %{conn: conn} do
   end
 
   describe "private instance" do
-    setup do: clear_config([:instance, :public])
+    setup do: clear_config([:instance, :public], false)
 
     test "returns 404 for tags feed", %{conn: conn} do
-      Config.put([:instance, :public], false)
-
       conn
       |> put_req_header("accept", "application/rss+xml")
-      |> get(tag_feed_path(conn, :feed, "pleromaart"))
+      |> get(tag_feed_path(conn, :feed, "pleromaart.rss"))
       |> response(404)
     end
   end

From 257e059e61b89752bcde9544cb5ae645b167c96b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 19 Aug 2020 15:31:33 +0400
Subject: [PATCH 025/208] Add account export

---
 lib/pleroma/export.ex | 118 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 lib/pleroma/export.ex

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
new file mode 100644
index 000000000..82a4b7ace
--- /dev/null
+++ b/lib/pleroma/export.ex
@@ -0,0 +1,118 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Export do
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+
+  import Ecto.Query
+
+  def run(user) do
+    with {:ok, dir} <- create_dir(),
+         :ok <- actor(dir, user),
+         :ok <- statuses(dir, user),
+         :ok <- likes(dir, user),
+         :ok <- bookmarks(dir, user) do
+      IO.inspect({"DONE", dir})
+    else
+      err -> IO.inspect({"export error", err})
+    end
+  end
+
+  def actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(dir <> "/actor.json", json)
+    end
+  end
+
+  defp create_dir do
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    dir = Path.join(System.tmp_dir!(), "archive-" <> datetime)
+
+    with :ok <- File.mkdir(dir), do: {:ok, dir}
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = dir <> "/#{name}.json"
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      counter = :counters.new(1, [])
+
+      query
+      |> Pleroma.RepoStreamer.chunk_stream(100)
+      |> Stream.each(fn items ->
+        Enum.each(items, fn i ->
+          with {:ok, str} <- fun.(i),
+               :ok <- IO.write(file, str <> ",\n") do
+            :counters.add(counter, 1, 1)
+          end
+        end)
+      end)
+      |> Stream.run()
+
+      total = :counters.get(counter, 1)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  def bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
+  end
+
+  def likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
+  end
+
+  def statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:blocking_user, user)
+      |> Map.put(:muting_user, user)
+      |> Map.put(:reply_filtering_user, user)
+      |> Map.put(:announce_filtering_user, user)
+      |> Map.put(:user, user)
+
+    [[user.ap_id], User.following(user), Pleroma.List.memberships(user)]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        activity |> Map.delete("@context") |> Jason.encode()
+      end
+    end)
+  end
+end

From 9d564ffc2988f145bc9cf26477eea93b1bf01cb0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 24 Aug 2020 20:59:57 +0400
Subject: [PATCH 026/208] Zip exported files

---
 lib/pleroma/export.ex | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index 82a4b7ace..f0f1ef093 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -12,15 +12,17 @@ defmodule Pleroma.Export do
 
   import Ecto.Query
 
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+
   def run(user) do
-    with {:ok, dir} <- create_dir(),
-         :ok <- actor(dir, user),
-         :ok <- statuses(dir, user),
-         :ok <- likes(dir, user),
-         :ok <- bookmarks(dir, user) do
-      IO.inspect({"DONE", dir})
-    else
-      err -> IO.inspect({"export error", err})
+    with {:ok, path} <- create_dir(user),
+         :ok <- actor(path, user),
+         :ok <- statuses(path, user),
+         :ok <- likes(path, user),
+         :ok <- bookmarks(path, user),
+         {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
+         {:ok, _} <- File.rm_rf(path) do
+      {:ok, zip_path}
     end
   end
 
@@ -33,9 +35,9 @@ def actor(dir, user) do
     end
   end
 
-  defp create_dir do
+  defp create_dir(user) do
     datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
-    dir = Path.join(System.tmp_dir!(), "archive-" <> datetime)
+    dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
 
     with :ok <- File.mkdir(dir), do: {:ok, dir}
   end

From c01a81804835fb92c145b90e3a264c5d4cf9c886 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 25 Aug 2020 18:51:09 +0400
Subject: [PATCH 027/208] Add tests

---
 lib/pleroma/export.ex |   8 +--
 test/export_test.exs  | 111 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+), 4 deletions(-)
 create mode 100644 test/export_test.exs

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index f0f1ef093..45b8ce749 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -26,7 +26,7 @@ def run(user) do
     end
   end
 
-  def actor(dir, user) do
+  defp actor(dir, user) do
     with {:ok, json} <-
            UserView.render("user.json", %{user: user})
            |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
@@ -82,7 +82,7 @@ defp write(query, dir, name, fun) do
     end
   end
 
-  def bookmarks(dir, %{id: user_id} = _user) do
+  defp bookmarks(dir, %{id: user_id} = _user) do
     Bookmark
     |> where(user_id: ^user_id)
     |> join(:inner, [b], activity in assoc(b, :activity))
@@ -90,7 +90,7 @@ def bookmarks(dir, %{id: user_id} = _user) do
     |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
   end
 
-  def likes(dir, user) do
+  defp likes(dir, user) do
     user.ap_id
     |> Activity.Queries.by_actor()
     |> Activity.Queries.by_type("Like")
@@ -98,7 +98,7 @@ def likes(dir, user) do
     |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
   end
 
-  def statuses(dir, user) do
+  defp statuses(dir, user) do
     opts =
       %{}
       |> Map.put(:type, ["Create", "Announce"])
diff --git a/test/export_test.exs b/test/export_test.exs
new file mode 100644
index 000000000..5afd58ccc
--- /dev/null
+++ b/test/export_test.exs
@@ -0,0 +1,111 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ExportTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Bookmark
+
+  test "it exports user data" do
+    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+    {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
+      CommonAPI.post(user, %{status: "status1"})
+
+    {:ok, %{object: %{data: %{"id" => id2}}} = status2} =
+      CommonAPI.post(user, %{status: "status2"})
+
+    {:ok, %{object: %{data: %{"id" => id3}}} = status3} =
+      CommonAPI.post(user, %{status: "status3"})
+
+    CommonAPI.favorite(user, status1.id)
+    CommonAPI.favorite(user, status2.id)
+
+    Bookmark.create(user.id, status2.id)
+    Bookmark.create(user.id, status3.id)
+
+    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, zipfile} = :zip.zip_open(path, [:memory])
+    assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
+
+    assert %{
+             "@context" => [
+               "https://www.w3.org/ns/activitystreams",
+               "http://localhost:4001/schemas/litepub-0.1.jsonld",
+               %{"@language" => "und"}
+             ],
+             "bookmarks" => "bookmarks.json",
+             "followers" => "http://cofe.io/users/cofe/followers",
+             "following" => "http://cofe.io/users/cofe/following",
+             "id" => "http://cofe.io/users/cofe",
+             "inbox" => "http://cofe.io/users/cofe/inbox",
+             "likes" => "likes.json",
+             "name" => "Cofe",
+             "outbox" => "http://cofe.io/users/cofe/outbox",
+             "preferredUsername" => "cofe",
+             "publicKey" => %{
+               "id" => "http://cofe.io/users/cofe#main-key",
+               "owner" => "http://cofe.io/users/cofe"
+             },
+             "type" => "Person",
+             "url" => "http://cofe.io/users/cofe"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "outbox.json",
+             "orderedItems" => [
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status1",
+                   "type" => "Note"
+                 },
+                 "type" => "Create"
+               },
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status2"
+                 }
+               },
+               %{
+                 "actor" => "http://cofe.io/users/cofe",
+                 "object" => %{
+                   "content" => "status3"
+                 }
+               }
+             ],
+             "totalItems" => 3,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "likes.json",
+             "orderedItems" => [^id1, ^id2],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "bookmarks.json",
+             "orderedItems" => [^id2, ^id3],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    :zip.zip_close(zipfile)
+    File.rm!(path)
+  end
+end

From c82f9129592553718be4bd4712a2b1848dd0a447 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 25 Aug 2020 19:16:01 +0400
Subject: [PATCH 028/208] Fix credo warning

---
 test/export_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/export_test.exs b/test/export_test.exs
index 5afd58ccc..01ca8e7e8 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -6,8 +6,8 @@ defmodule Pleroma.ExportTest do
   use Pleroma.DataCase
   import Pleroma.Factory
 
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Bookmark
+  alias Pleroma.Web.CommonAPI
 
   test "it exports user data" do
     user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})

From be42ab70dc9538df54ac6f30ee123666223b7287 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 31 Aug 2020 20:31:21 +0400
Subject: [PATCH 029/208] Add backup upload

---
 lib/pleroma/export.ex | 20 +++++++++++++++++++-
 test/export_test.exs  | 17 ++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index 45b8ce749..b84eccd78 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -22,7 +22,25 @@ def run(user) do
          :ok <- bookmarks(path, user),
          {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
          {:ok, _} <- File.rm_rf(path) do
-      {:ok, zip_path}
+      {:ok, :binary.list_to_bin(zip_path)}
+    end
+  end
+
+  def upload(zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+    file_name = zip_path |> String.split("/") |> List.last()
+    id = Ecto.UUID.generate()
+
+    upload = %Pleroma.Upload{
+      id: id,
+      name: file_name,
+      tempfile: zip_path,
+      content_type: "application/zip",
+      path: id <> "/" <> file_name
+    }
+
+    with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do
+      {:ok, upload}
     end
   end
 
diff --git a/test/export_test.exs b/test/export_test.exs
index 01ca8e7e8..fae269974 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -28,7 +28,7 @@ test "it exports user data" do
     Bookmark.create(user.id, status3.id)
 
     assert {:ok, path} = Pleroma.Export.run(user)
-    assert {:ok, zipfile} = :zip.zip_open(path, [:memory])
+    assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
     assert %{
@@ -108,4 +108,19 @@ test "it exports user data" do
     :zip.zip_close(zipfile)
     File.rm!(path)
   end
+
+  test "it uploads an exported backup archive" do
+    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+    {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
+    {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
+    {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
+    CommonAPI.favorite(user, status1.id)
+    CommonAPI.favorite(user, status2.id)
+    Bookmark.create(user.id, status2.id)
+    Bookmark.create(user.id, status3.id)
+
+    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+  end
 end

From 75e07ba206b94155c5210151a49e29a11bce6e50 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 31 Aug 2020 23:07:14 +0400
Subject: [PATCH 030/208] Fix tests

---
 lib/pleroma/export.ex |  3 ++-
 test/export_test.exs  | 47 +++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index b84eccd78..8b1bfefe2 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -39,7 +39,8 @@ def upload(zip_path) do
       path: id <> "/" <> file_name
     }
 
-    with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
       {:ok, upload}
     end
   end
diff --git a/test/export_test.exs b/test/export_test.exs
index fae269974..d7e8f558c 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.ExportTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  import Mock
 
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
@@ -109,18 +110,42 @@ test "it exports user data" do
     File.rm!(path)
   end
 
-  test "it uploads an exported backup archive" do
-    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+  describe "it uploads an exported backup archive" do
+    setup do
+      clear_config(Pleroma.Uploaders.S3,
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com"
+      )
 
-    {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
-    {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
-    {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
-    CommonAPI.favorite(user, status1.id)
-    CommonAPI.favorite(user, status2.id)
-    Bookmark.create(user.id, status2.id)
-    Bookmark.create(user.id, status3.id)
+      clear_config([Pleroma.Upload, :uploader])
 
-    assert {:ok, path} = Pleroma.Export.run(user)
-    assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+      {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
+      {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
+      {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
+      CommonAPI.favorite(user, status1.id)
+      CommonAPI.favorite(user, status2.id)
+      Bookmark.create(user.id, status2.id)
+      Bookmark.create(user.id, status3.id)
+
+      assert {:ok, path} = Pleroma.Export.run(user)
+
+      [path: path]
+    end
+
+    test "S3", %{path: path} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
+
+      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+        assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      end
+    end
+
+    test "Local", %{path: path} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
+      assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+    end
   end
 end

From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 2 Sep 2020 20:21:33 +0400
Subject: [PATCH 031/208] Add `backups` table

---
 lib/pleroma/{export.ex => backup.ex}          | 110 ++++++++++++++----
 .../20200831192323_create_backups.exs         |  17 +++
 test/{export_test.exs => backup_test.exs}     |  48 ++++++--
 3 files changed, 141 insertions(+), 34 deletions(-)
 rename lib/pleroma/{export.ex => backup.ex} (60%)
 create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs
 rename test/{export_test.exs => backup_test.exs} (75%)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex
similarity index 60%
rename from lib/pleroma/export.ex
rename to lib/pleroma/backup.ex
index 8b1bfefe2..4580d8f92 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/backup.ex
@@ -2,41 +2,110 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Export do
+defmodule Pleroma.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
   alias Pleroma.Activity
   alias Pleroma.Bookmark
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
 
-  import Ecto.Query
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user) do
+    with :ok <- validate_limit(user),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      {:ok, backup}
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  defp validate_limit(user) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = 7
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error, "Last export was less than #{days} days ago"}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- zip(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
 
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def zip(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = Path.join(System.tmp_dir!(), name)
 
-  def run(user) do
-    with {:ok, path} <- create_dir(user),
-         :ok <- actor(path, user),
-         :ok <- statuses(path, user),
-         :ok <- likes(path, user),
-         :ok <- bookmarks(path, user),
-         {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
-         {:ok, _} <- File.rm_rf(path) do
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
       {:ok, :binary.list_to_bin(zip_path)}
     end
   end
 
-  def upload(zip_path) do
+  def upload(%__MODULE__{} = backup, zip_path) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
-    file_name = zip_path |> String.split("/") |> List.last()
-    id = Ecto.UUID.generate()
 
     upload = %Pleroma.Upload{
-      id: id,
-      name: file_name,
+      name: backup.file_name,
       tempfile: zip_path,
-      content_type: "application/zip",
-      path: id <> "/" <> file_name
+      content_type: backup.content_type,
+      path: "backups/" <> backup.file_name
     }
 
     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@@ -54,13 +123,6 @@ defp actor(dir, user) do
     end
   end
 
-  defp create_dir(user) do
-    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
-    dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
-
-    with :ok <- File.mkdir(dir), do: {:ok, dir}
-  end
-
   defp write_header(file, name) do
     IO.write(
       file,
diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs
new file mode 100644
index 000000000..3ac5889e2
--- /dev/null
+++ b/priv/repo/migrations/20200831192323_create_backups.exs
@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.CreateBackups do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:backups) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:file_name, :string, null: false)
+      add(:content_type, :string, null: false)
+      add(:processed, :boolean, null: false, default: false)
+      add(:file_size, :bigint)
+
+      timestamps()
+    end
+
+    create_if_not_exists(index(:backups, [:user_id]))
+  end
+end
diff --git a/test/export_test.exs b/test/backup_test.exs
similarity index 75%
rename from test/export_test.exs
rename to test/backup_test.exs
index d7e8f558c..27f5cb7f7 100644
--- a/test/export_test.exs
+++ b/test/backup_test.exs
@@ -2,15 +2,41 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ExportTest do
+defmodule Pleroma.BackupTest do
   use Pleroma.DataCase
   import Pleroma.Factory
   import Mock
 
+  alias Pleroma.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
 
-  test "it exports user data" do
+  test "it creates a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, backup} = Backup.create(user)
+
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+  end
+
+  test "it return an error if the export limit is over" do
+    %{id: user_id} = user = insert(:user)
+    limit_days = 7
+
+    assert {:ok, backup} = Backup.create(user)
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+
+    assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
+  end
+
+  test "it process a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
+    assert {:ok, %Backup{} = backup} = Backup.process(backup)
+    assert backup.file_size > 0
+    assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+  end
+
+  test "it creates a zip archive with user data" do
     user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
 
     {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
@@ -28,7 +54,8 @@ test "it exports user data" do
     Bookmark.create(user.id, status2.id)
     Bookmark.create(user.id, status3.id)
 
-    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+    assert {:ok, path} = Backup.zip(backup)
     assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
@@ -110,7 +137,7 @@ test "it exports user data" do
     File.rm!(path)
   end
 
-  describe "it uploads an exported backup archive" do
+  describe "it uploads a backup archive" do
     setup do
       clear_config(Pleroma.Uploaders.S3,
         bucket: "test_bucket",
@@ -129,23 +156,24 @@ test "it exports user data" do
       Bookmark.create(user.id, status2.id)
       Bookmark.create(user.id, status3.id)
 
-      assert {:ok, path} = Pleroma.Export.run(user)
+      assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+      assert {:ok, path} = Backup.zip(backup)
 
-      [path: path]
+      [path: path, backup: backup]
     end
 
-    test "S3", %{path: path} do
+    test "S3", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
 
       with_mock ExAws, request: fn _ -> {:ok, :ok} end do
-        assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+        assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
       end
     end
 
-    test "Local", %{path: path} do
+    test "Local", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
 
-      assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
     end
   end
 end

From a0ad9bd734e9af0ce912c32c7480a60ff87a4368 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 2 Sep 2020 21:45:22 +0400
Subject: [PATCH 032/208] Add BackupWorker

---
 config/config.exs                    |  1 +
 config/description.exs               |  6 ++++++
 lib/pleroma/backup.ex                | 11 ++++++++++-
 lib/pleroma/workers/backup_worker.ex | 17 +++++++++++++++++
 test/backup_test.exs                 | 20 ++++++++++++++------
 5 files changed, 48 insertions(+), 7 deletions(-)
 create mode 100644 lib/pleroma/workers/backup_worker.ex

diff --git a/config/config.exs b/config/config.exs
index 2e6b0796a..1f10167e5 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -551,6 +551,7 @@
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
diff --git a/config/description.exs b/config/description.exs
index 6fa78a5d1..13e44afe8 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -2288,6 +2288,12 @@
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 4580d8f92..9b5d2625f 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,7 +30,7 @@ defmodule Pleroma.Backup do
   def create(user) do
     with :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      {:ok, backup}
+      Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id})
     end
   end
 
@@ -71,6 +71,15 @@ def get_last(user_id) do
     |> Repo.one()
   end
 
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.delete_all()
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
   def process(%__MODULE__{} = backup) do
     with {:ok, zip_file} <- zip(backup),
          {:ok, %{size: size}} <- File.stat(zip_file),
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
new file mode 100644
index 000000000..c982ffa3a
--- /dev/null
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.BackupWorker do
+  alias Pleroma.Backup
+
+  use Pleroma.Workers.WorkerHelper, queue: "backup"
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
+    with {:ok, %Backup{} = backup} <-
+           backup_id |> Backup.get() |> Backup.process() do
+      {:ok, backup}
+    end
+  end
+end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 27f5cb7f7..5b1f76dd9 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -3,35 +3,43 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BackupTest do
+  use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
+
   import Pleroma.Factory
   import Mock
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackupWorker
 
-  test "it creates a backup record" do
+  setup do: clear_config([Pleroma.Upload, :uploader])
+
+  test "it creates a backup record and an Oban job" do
     %{id: user_id} = user = insert(:user)
-    assert {:ok, backup} = Backup.create(user)
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    assert_enqueued(worker: BackupWorker, args: args)
 
+    backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
   end
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
     limit_days = 7
-
-    assert {:ok, backup} = Backup.create(user)
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
 
     assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
   end
 
   test "it process a backup record" do
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     %{id: user_id} = user = insert(:user)
-    assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
-    assert {:ok, %Backup{} = backup} = Backup.process(backup)
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
+    assert {:ok, backup} = BackupWorker.perform(job)
     assert backup.file_size > 0
     assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
   end

From 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 18:30:39 +0400
Subject: [PATCH 033/208] Add config for Pleroma.Backup

---
 config/config.exs                |  4 ++++
 config/description.exs           | 20 ++++++++++++++++++++
 docs/configuration/cheatsheet.md |  5 +++++
 lib/pleroma/backup.ex            |  2 +-
 test/backup_test.exs             |  2 +-
 5 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 1f10167e5..09023e2c3 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -818,6 +818,10 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.Backup,
+  purge_after_days: 30,
+  limit_days: 7
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/config/description.exs b/config/description.exs
index 13e44afe8..4942e196d 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3712,5 +3712,25 @@
         ]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
   }
 ]
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 42e5fe808..cc4081f14 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1083,6 +1083,11 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Account Backup
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 9b5d2625f..e384b6b00 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -49,7 +49,7 @@ def new(user) do
   defp validate_limit(user) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
-        days = 7
+        days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
         diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
 
         if diff > days do
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 5b1f76dd9..f343b0361 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -27,7 +27,7 @@ test "it creates a backup record and an Oban job" do
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
-    limit_days = 7
+    limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
     assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
     backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup

From 739cb1463ba07513f047b2ac8f7e22a16c89ef4e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 21:48:52 +0400
Subject: [PATCH 034/208] Add backups deletion

---
 lib/pleroma/backup.ex                | 14 +++++++--
 lib/pleroma/workers/backup_worker.ex | 37 ++++++++++++++++++++--
 test/backup_test.exs                 | 47 +++++++++++++++++++++++++---
 test/support/oban_helpers.ex         |  3 ++
 4 files changed, 91 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index e384b6b00..bd50fd910 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Backup do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
 
   schema "backups" do
     field(:content_type, :string)
@@ -30,7 +31,7 @@ defmodule Pleroma.Backup do
   def create(user) do
     with :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id})
+      BackupWorker.process(backup)
     end
   end
 
@@ -46,6 +47,14 @@ def new(user) do
     }
   end
 
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
+      Repo.delete(backup)
+    end
+  end
+
   defp validate_limit(user) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
@@ -75,7 +84,8 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
     __MODULE__
     |> where(user_id: ^user_id)
     |> where([b], b.id != ^latest_id)
-    |> Repo.delete_all()
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
   end
 
   def get(id), do: Repo.get(__MODULE__, id)
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index c982ffa3a..f40020794 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -3,15 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.BackupWorker do
+  use Oban.Worker, queue: :backup, max_attempts: 1
+
+  alias Oban.Job
   alias Pleroma.Backup
 
-  use Pleroma.Workers.WorkerHelper, queue: "backup"
+  def process(backup) do
+    %{"op" => "process", "backup_id" => backup.id}
+    |> new()
+    |> Oban.insert()
+  end
+
+  def schedule_deletion(backup) do
+    days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
+    time = 60 * 60 * 24 * days
+    scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
+
+    %{"op" => "delete", "backup_id" => backup.id}
+    |> new(scheduled_at: scheduled_at)
+    |> Oban.insert()
+  end
+
+  def delete(backup) do
+    %{"op" => "delete", "backup_id" => backup.id}
+    |> new()
+    |> Oban.insert()
+  end
 
-  @impl Oban.Worker
   def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
     with {:ok, %Backup{} = backup} <-
-           backup_id |> Backup.get() |> Backup.process() do
+           backup_id |> Backup.get() |> Backup.process(),
+         {:ok, _job} <- schedule_deletion(backup),
+         :ok <- Backup.remove_outdated(backup) do
       {:ok, backup}
     end
   end
+
+  def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
+    case Backup.get(backup_id) do
+      %Backup{} = backup -> Backup.delete(backup)
+      nil -> :ok
+    end
+  end
 end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index f343b0361..59aebe360 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
+  alias Pleroma.Tests.ObanHelpers
 
-  setup do: clear_config([Pleroma.Upload, :uploader])
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Pleroma.Backup, :limit_days])
+  end
 
   test "it creates a backup record and an Oban job" do
     %{id: user_id} = user = insert(:user)
@@ -38,10 +42,34 @@ test "it return an error if the export limit is over" do
   test "it process a backup record" do
     Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     %{id: user_id} = user = insert(:user)
-    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
-    assert {:ok, backup} = BackupWorker.perform(job)
+
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+    assert {:ok, backup} = perform_job(BackupWorker, args)
     assert backup.file_size > 0
     assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+    delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
+
+    assert_enqueued(worker: BackupWorker, args: delete_job_args)
+    assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
+    refute Backup.get(backup_id)
+  end
+
+  test "it removes outdated backups after creating a fresh one" do
+    Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+    user = insert(:user)
+
+    assert {:ok, job1} = Backup.create(user)
+
+    assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
+    assert {:ok, job2} = Backup.create(user)
+    assert Pleroma.Repo.aggregate(Backup, :count) == 2
+    assert {:ok, backup2} = ObanHelpers.perform(job2)
+
+    ObanHelpers.perform_all()
+
+    assert [^backup2] = Pleroma.Repo.all(Backup)
   end
 
   test "it creates a zip archive with user data" do
@@ -145,7 +173,7 @@ test "it creates a zip archive with user data" do
     File.rm!(path)
   end
 
-  describe "it uploads a backup archive" do
+  describe "it uploads and deletes a backup archive" do
     setup do
       clear_config(Pleroma.Uploaders.S3,
         bucket: "test_bucket",
@@ -173,8 +201,16 @@ test "it creates a zip archive with user data" do
     test "S3", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
 
-      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+      with_mock ExAws,
+        request: fn
+          %{http_method: :put} -> {:ok, :ok}
+          %{http_method: :delete} -> {:ok, %{status_code: 204}}
+        end do
         assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+        assert {:ok, _backup} = Backup.delete(backup)
+      end
+
+      with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
       end
     end
 
@@ -182,6 +218,7 @@ test "Local", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
 
       assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+      assert {:ok, _backup} = Backup.delete(backup)
     end
   end
 end
diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex
index 9f90a821c..2468f66dc 100644
--- a/test/support/oban_helpers.ex
+++ b/test/support/oban_helpers.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
   Oban test helpers.
   """
 
+  require Ecto.Query
+
   alias Pleroma.Repo
 
   def wipe_all do
@@ -15,6 +17,7 @@ def wipe_all do
 
   def perform_all do
     Oban.Job
+    |> Ecto.Query.where(state: "available")
     |> Repo.all()
     |> perform()
   end

From abdffc6b8c2eec8f81ffe89f943f11d1f90d7074 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 22:00:26 +0400
Subject: [PATCH 035/208] Fix Credo warning

---
 test/backup_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/backup_test.exs b/test/backup_test.exs
index 59aebe360..5fc519eab 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -11,9 +11,9 @@ defmodule Pleroma.BackupTest do
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
+  alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
-  alias Pleroma.Tests.ObanHelpers
 
   setup do
     clear_config([Pleroma.Upload, :uploader])

From 2c73bfe1227065fa203b0b78c9eb12cf86ab3948 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 9 Sep 2020 01:04:00 +0400
Subject: [PATCH 036/208] Add API endpoints for Backups

---
 lib/pleroma/backup.ex                         |  7 ++
 .../operations/pleroma_backup_operation.ex    | 79 +++++++++++++++++
 .../controllers/backup_controller.ex          | 27 ++++++
 .../web/pleroma_api/views/backup_view.ex      | 24 ++++++
 lib/pleroma/web/router.ex                     |  3 +
 .../controllers/backup_controller_test.exs    | 84 +++++++++++++++++++
 6 files changed, 224 insertions(+)
 create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
 create mode 100644 lib/pleroma/web/pleroma_api/views/backup_view.ex
 create mode 100644 test/web/pleroma_api/controllers/backup_controller_test.exs

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index bd50fd910..348e537a8 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -80,6 +80,13 @@ def get_last(user_id) do
     |> Repo.one()
   end
 
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
   def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
     __MODULE__
     |> where(user_id: ^user_id)
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
new file mode 100644
index 000000000..f877ca31b
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "List backups",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.index",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "Create a backup",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.create",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp backup do
+    %Schema{
+      title: "Backup",
+      description: "Response schema for a backup",
+      type: :object,
+      properties: %{
+        inserted_at: %Schema{type: :string, format: :"date-time"},
+        content_type: %Schema{type: :string},
+        file_name: %Schema{type: :string},
+        file_size: %Schema{type: :integer},
+        processed: %Schema{type: :boolean}
+      },
+      example: %{
+        "content_type" => "application/zip",
+        "file_name" =>
+          "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip",
+        "file_size" => 1024,
+        "inserted_at" => "2020-09-08T19:58:20",
+        "processed" => true
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
new file mode 100644
index 000000000..e52c77ff2
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+  plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
+  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
+
+  def index(%{assigns: %{user: user}} = conn, _params) do
+    backups = Pleroma.Backup.list(user)
+    render(conn, "index.json", backups: backups)
+  end
+
+  def create(%{assigns: %{user: user}} = conn, _params) do
+    with {:ok, _} <- Pleroma.Backup.create(user) do
+      backups = Pleroma.Backup.list(user)
+      render(conn, "index.json", backups: backups)
+    end
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
new file mode 100644
index 000000000..02b94ce4f
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Backup
+  alias Pleroma.Web.CommonAPI.Utils
+
+  def render("show.json", %{backup: %Backup{} = backup}) do
+    %{
+      content_type: backup.content_type,
+      file_name: backup.file_name,
+      file_size: backup.file_size,
+      processed: backup.processed,
+      inserted_at: Utils.to_masto_date(backup.inserted_at)
+    }
+  end
+
+  def render("index.json", %{backups: backups}) do
+    render_many(backups, __MODULE__, "show.json")
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e22b31b4c..a1a5a1cb5 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -293,6 +293,9 @@ defmodule Pleroma.Web.Router do
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)
     post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm)
     delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable)
+
+    get("/backups", BackupController, :index)
+    post("/backups", BackupController, :create)
   end
 
   scope "/oauth", Pleroma.Web.OAuth do
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
new file mode 100644
index 000000000..1ad1b63c4
--- /dev/null
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -0,0 +1,84 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Backup
+
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Backup, :limit_days])
+    oauth_access(["read:accounts"])
+  end
+
+  test "GET /api/pleroma/backups", %{user: user, conn: conn} do
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
+
+    backup = Backup.get(backup_id)
+
+    response =
+      conn
+      |> get("/api/pleroma/backups")
+      |> json_response_and_validate_schema(:ok)
+
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "file_name" => file_name,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] = response
+
+    assert file_name == backup.file_name
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "file_name" => ^file_name,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+  end
+
+  test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "file_name" => file_name,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] =
+             conn
+             |> post("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "file_name" => ^file_name,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    days = Pleroma.Config.get([Backup, :limit_days])
+
+    assert %{"error" => "Last export was less than #{days} days ago"} ==
+             conn
+             |> post("/api/pleroma/backups")
+             |> json_response_and_validate_schema(400)
+  end
+end

From 86ce4afd9338d81f741fa57f962509a6f0f50aff Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 9 Sep 2020 20:02:20 +0400
Subject: [PATCH 037/208] Improve backup urls

---
 .../api_spec/operations/pleroma_backup_operation.ex   |  6 +++---
 lib/pleroma/web/pleroma_api/views/backup_view.ex      |  6 +++++-
 .../controllers/backup_controller_test.exs            | 11 ++++++-----
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
index f877ca31b..6993794db 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
@@ -69,9 +69,9 @@ defp backup do
       example: %{
         "content_type" => "application/zip",
         "file_name" =>
-          "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip",
-        "file_size" => 1024,
-        "inserted_at" => "2020-09-08T19:58:20",
+          "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
+        "file_size" => 4105,
+        "inserted_at" => "2020-09-08T16:42:07.000Z",
         "processed" => true
       }
     }
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
index 02b94ce4f..bf40a001e 100644
--- a/lib/pleroma/web/pleroma_api/views/backup_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do
   def render("show.json", %{backup: %Backup{} = backup}) do
     %{
       content_type: backup.content_type,
-      file_name: backup.file_name,
+      url: download_url(backup),
       file_size: backup.file_size,
       processed: backup.processed,
       inserted_at: Utils.to_masto_date(backup.inserted_at)
@@ -21,4 +21,8 @@ def render("show.json", %{backup: %Backup{} = backup}) do
   def render("index.json", %{backups: backups}) do
     render_many(backups, __MODULE__, "show.json")
   end
+
+  def download_url(%Backup{file_name: file_name}) do
+    Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name
+  end
 end
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
index 1ad1b63c4..5d2f1206e 100644
--- a/test/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
   use Pleroma.Web.ConnCase
 
   alias Pleroma.Backup
+  alias Pleroma.Web.PleromaAPI.BackupView
 
   setup do
     clear_config([Pleroma.Upload, :uploader])
@@ -26,20 +27,20 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
-               "file_name" => file_name,
+               "url" => url,
                "file_size" => 0,
                "processed" => false,
                "inserted_at" => _
              }
            ] = response
 
-    assert file_name == backup.file_name
+    assert url == BackupView.download_url(backup)
 
     Pleroma.Tests.ObanHelpers.perform_all()
 
     assert [
              %{
-               "file_name" => ^file_name,
+               "url" => ^url,
                "processed" => true
              }
            ] =
@@ -52,7 +53,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
-               "file_name" => file_name,
+               "url" => url,
                "file_size" => 0,
                "processed" => false,
                "inserted_at" => _
@@ -66,7 +67,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
 
     assert [
              %{
-               "file_name" => ^file_name,
+               "url" => ^url,
                "processed" => true
              }
            ] =

From cd13613db3f675b6a9171dea56fc5b03e43ae6b0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 10 Sep 2020 20:53:06 +0400
Subject: [PATCH 038/208] Fix query

---
 lib/pleroma/backup.ex | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 348e537a8..ce54a413a 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -8,6 +8,8 @@ defmodule Pleroma.Backup do
   import Ecto.Changeset
   import Ecto.Query
 
+  require Pleroma.Constants
+
   alias Pleroma.Activity
   alias Pleroma.Bookmark
   alias Pleroma.Repo
@@ -158,6 +160,7 @@ defp write_header(file, name) do
         "id": "#{name}.json",
         "type": "OrderedCollection",
         "orderedItems": [
+
       """
     )
   end
@@ -209,13 +212,13 @@ defp statuses(dir, user) do
     opts =
       %{}
       |> Map.put(:type, ["Create", "Announce"])
-      |> Map.put(:blocking_user, user)
-      |> Map.put(:muting_user, user)
-      |> Map.put(:reply_filtering_user, user)
-      |> Map.put(:announce_filtering_user, user)
-      |> Map.put(:user, user)
+      |> Map.put(:actor_id, user.ap_id)
 
-    [[user.ap_id], User.following(user), Pleroma.List.memberships(user)]
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
     |> Enum.concat()
     |> ActivityPub.fetch_activities_query(opts)
     |> write(dir, "outbox", fn a ->

From 386199063b9be9fc30ad403f6afb03bf6ca47298 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 10 Sep 2020 21:09:20 +0400
Subject: [PATCH 039/208] Document `/api/pleroma/backups` API endpoint

---
 docs/API/pleroma_api.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index 3fd141bd2..aeb266159 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

From 27bc121ec00a7b088030d6fb36c7e731f5b072b6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 15 Sep 2020 18:07:28 +0400
Subject: [PATCH 040/208] Require email

---
 docs/configuration/cheatsheet.md |  3 +++
 lib/pleroma/backup.ex            | 19 ++++++++++++++++---
 test/backup_test.exs             | 16 ++++++++++++++--
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index cc4081f14..8da8a7bd6 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1085,6 +1085,9 @@ Control favicons for instances.
 
 ## Account Backup
 
+!!! note
+    Requires enabled email
+
 * `:purge_after_days` an integer, remove backup achives after N days.
 * `:limit_days` an integer, limit user to export not more often than once per N days.
 
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index ce54a413a..3b85dd1c1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -31,7 +31,9 @@ defmodule Pleroma.Backup do
   end
 
   def create(user) do
-    with :ok <- validate_limit(user),
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
       BackupWorker.process(backup)
     end
@@ -74,6 +76,17 @@ defp validate_limit(user) do
     end
   end
 
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, "Backups require enabled email"}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"}
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
   def get_last(user_id) do
     __MODULE__
     |> where(user_id: ^user_id)
@@ -100,7 +113,7 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
   def get(id), do: Repo.get(__MODULE__, id)
 
   def process(%__MODULE__{} = backup) do
-    with {:ok, zip_file} <- zip(backup),
+    with {:ok, zip_file} <- export(backup),
          {:ok, %{size: size}} <- File.stat(zip_file),
          {:ok, _upload} <- upload(backup, zip_file) do
       backup
@@ -110,7 +123,7 @@ def process(%__MODULE__{} = backup) do
   end
 
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
-  def zip(%__MODULE__{} = backup) do
+  def export(%__MODULE__{} = backup) do
     backup = Repo.preload(backup, :user)
     name = String.trim_trailing(backup.file_name, ".zip")
     dir = Path.join(System.tmp_dir!(), name)
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 5fc519eab..318c8c419 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -18,6 +18,18 @@ defmodule Pleroma.BackupTest do
   setup do
     clear_config([Pleroma.Upload, :uploader])
     clear_config([Pleroma.Backup, :limit_days])
+    clear_config([Pleroma.Emails.Mailer, :enabled])
+  end
+
+  test "it requries enabled email" do
+    Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
+    user = insert(:user)
+    assert {:error, "Backups require enabled email"} == Backup.create(user)
+  end
+
+  test "it requries user's email" do
+    user = insert(:user, %{email: nil})
+    assert {:error, "Email is required"} == Backup.create(user)
   end
 
   test "it creates a backup record and an Oban job" do
@@ -91,7 +103,7 @@ test "it creates a zip archive with user data" do
     Bookmark.create(user.id, status3.id)
 
     assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
-    assert {:ok, path} = Backup.zip(backup)
+    assert {:ok, path} = Backup.export(backup)
     assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
@@ -193,7 +205,7 @@ test "it creates a zip archive with user data" do
       Bookmark.create(user.id, status3.id)
 
       assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
-      assert {:ok, path} = Backup.zip(backup)
+      assert {:ok, path} = Backup.export(backup)
 
       [path: path, backup: backup]
     end

From e52dd62e14a956a28a706124464f3ac4b985080d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 16 Sep 2020 23:21:13 +0400
Subject: [PATCH 041/208] Add configurable temporary directory

---
 config/config.exs                | 3 ++-
 docs/configuration/cheatsheet.md | 6 ++++++
 lib/pleroma/backup.ex            | 7 ++++++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 09023e2c3..0e12d6e15 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -820,7 +820,8 @@
 
 config :pleroma, Pleroma.Backup,
   purge_after_days: 30,
-  limit_days: 7
+  limit_days: 7,
+  dir: nil
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 8da8a7bd6..9271964f1 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1090,6 +1090,12 @@ Control favicons for instances.
 
 * `:purge_after_days` an integer, remove backup achives after N days.
 * `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
 
 ## Frontend management
 
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 3b85dd1c1..450dd5b84 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -126,7 +126,7 @@ def process(%__MODULE__{} = backup) do
   def export(%__MODULE__{} = backup) do
     backup = Repo.preload(backup, :user)
     name = String.trim_trailing(backup.file_name, ".zip")
-    dir = Path.join(System.tmp_dir!(), name)
+    dir = dir(name)
 
     with :ok <- File.mkdir(dir),
          :ok <- actor(dir, backup.user),
@@ -139,6 +139,11 @@ def export(%__MODULE__{} = backup) do
     end
   end
 
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
   def upload(%__MODULE__{} = backup, zip_path) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 

From 7fdd81d000d857cbcd5bf442f68c91b1c5b1cebb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 17 Sep 2020 18:42:24 +0400
Subject: [PATCH 042/208] Add "Your backup is ready" email

---
 lib/pleroma/emails/user_email.ex     | 16 ++++++++++++++++
 lib/pleroma/workers/backup_worker.ex |  6 +++++-
 test/backup_test.exs                 |  5 ++++-
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index 1d8c72ae9..f943dda0d 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -189,4 +189,20 @@ def unsubscribe_url(user, notifications_type) do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body = """
+    <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+    <p><a href="#{download_url}"></a></p>
+    """
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index f40020794..405d55269 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -34,7 +34,11 @@ def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
     with {:ok, %Backup{} = backup} <-
            backup_id |> Backup.get() |> Backup.process(),
          {:ok, _job} <- schedule_deletion(backup),
-         :ok <- Backup.remove_outdated(backup) do
+         :ok <- Backup.remove_outdated(backup),
+         {:ok, _} <-
+           backup
+           |> Pleroma.Emails.UserEmail.backup_is_ready_email()
+           |> Pleroma.Emails.Mailer.deliver() do
       {:ok, backup}
     end
   end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 318c8c419..0ea40e6fd 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -6,8 +6,9 @@ defmodule Pleroma.BackupTest do
   use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
 
-  import Pleroma.Factory
   import Mock
+  import Pleroma.Factory
+  import Swoosh.TestAssertions
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
@@ -65,6 +66,8 @@ test "it process a backup record" do
     assert_enqueued(worker: BackupWorker, args: delete_job_args)
     assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
     refute Backup.get(backup_id)
+
+    assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup))
   end
 
   test "it removes outdated backups after creating a fresh one" do

From 7c22c9afb410668d87dcd4a90651d62d9a1e9e4d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:18:34 +0400
Subject: [PATCH 043/208] Allow admins request user backups

---
 lib/pleroma/backup.ex                         |  4 ++--
 lib/pleroma/emails/user_email.ex              | 20 +++++++++++++-----
 .../controllers/admin_api_controller.ex       | 12 ++++++++++-
 lib/pleroma/web/router.ex                     |  2 ++
 lib/pleroma/workers/backup_worker.ex          | 10 +++++----
 .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++
 6 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 450dd5b84..d589f12f1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,12 +30,12 @@ defmodule Pleroma.Backup do
     timestamps()
   end
 
-  def create(user) do
+  def create(user, admin_user_id \\ nil) do
     with :ok <- validate_email_enabled(),
          :ok <- validate_user_email(user),
          :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      BackupWorker.process(backup)
+      BackupWorker.process(backup, admin_user_id)
     end
   end
 
diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index f943dda0d..5745794ec 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -190,14 +190,24 @@ def unsubscribe_url(user, notifications_type) do
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
 
-  def backup_is_ready_email(backup) do
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
     %{user: user} = Pleroma.Repo.preload(backup, :user)
     download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
 
-    html_body = """
-    <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
-    <p><a href="#{download_url}"></a></p>
-    """
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}"></a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}"></a></p>
+        """
+      end
 
     new()
     |> to(recipient(user))
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index d5713c3dd..f7d2fe5b1 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -23,12 +23,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
+  require Logger
+
   @users_page_size 50
 
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
+    when action in [:list_users, :user_show, :right_get, :show_user_credentials, :create_backup]
   )
 
   plug(
@@ -681,6 +683,14 @@ def stats(conn, params) do
     json(conn, %{"status_visibility" => counters})
   end
 
+  def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname),
+         {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
+      Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}")
+      json(conn, "")
+    end
+  end
+
   defp page_params(params) do
     {get_page(params["page"]), get_page_size(params["page_size"])}
   end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index a1a5a1cb5..e539eeeeb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -129,6 +129,8 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
 
+    post("/backups", AdminAPIController, :create_backup)
+
     post("/users/follow", AdminAPIController, :user_follow)
     post("/users/unfollow", AdminAPIController, :user_unfollow)
 
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 405d55269..65754b6a2 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -8,8 +8,8 @@ defmodule Pleroma.Workers.BackupWorker do
   alias Oban.Job
   alias Pleroma.Backup
 
-  def process(backup) do
-    %{"op" => "process", "backup_id" => backup.id}
+  def process(backup, admin_user_id \\ nil) do
+    %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
     |> new()
     |> Oban.insert()
   end
@@ -30,14 +30,16 @@ def delete(backup) do
     |> Oban.insert()
   end
 
-  def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
+  def perform(%Job{
+        args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id}
+      }) do
     with {:ok, %Backup{} = backup} <-
            backup_id |> Backup.get() |> Backup.process(),
          {:ok, _job} <- schedule_deletion(backup),
          :ok <- Backup.remove_outdated(backup),
          {:ok, _} <-
            backup
-           |> Pleroma.Emails.UserEmail.backup_is_ready_email()
+           |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
            |> Pleroma.Emails.Mailer.deliver() do
       {:ok, backup}
     end
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index cba6b43d3..4d331779e 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2024,6 +2024,27 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
+
+  describe "/api/pleroma/backups" do
+    test "it creates a backup", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [backup] = Repo.all(Pleroma.Backup)
+
+      ObanHelpers.perform_all()
+
+      assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
+    end
+  end
 end
 
 # Needed for testing

From 563801716a0aa54e30f680b4e985d4b8c79578fb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:01:46 +0400
Subject: [PATCH 044/208] Update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..04b49d80a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Account backup
 
 ### Changed
 

From e50314d9d342dbf9a03ca484654b07717592d4bd Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:33:12 +0400
Subject: [PATCH 045/208] Fix export

---
 lib/pleroma/backup.ex | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index d589f12f1..242773bdb 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -191,16 +191,13 @@ defp write(query, dir, name, fun) do
       counter = :counters.new(1, [])
 
       query
-      |> Pleroma.RepoStreamer.chunk_stream(100)
-      |> Stream.each(fn items ->
-        Enum.each(items, fn i ->
-          with {:ok, str} <- fun.(i),
-               :ok <- IO.write(file, str <> ",\n") do
-            :counters.add(counter, 1, 1)
-          end
-        end)
+      |> Pleroma.Repo.chunk_stream(100)
+      |> Enum.each(fn i ->
+        with {:ok, str} <- fun.(i),
+             :ok <- IO.write(file, str <> ",\n") do
+          :counters.add(counter, 1, 1)
+        end
       end)
-      |> Stream.run()
 
       total = :counters.get(counter, 1)
 

From a9efd441e242f1d8ac608b866d0cfafe4833243a Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 19:57:09 +0400
Subject: [PATCH 046/208] Use `Pleroma.Repo.chunk_stream/2` instead of
 `Pleroma.RepoStreamer.chunk_stream/2`

---
 lib/pleroma/backup.ex | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 242773bdb..f5f39431d 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -188,18 +188,17 @@ defp write(query, dir, name, fun) do
 
     with {:ok, file} <- File.open(path, [:write, :utf8]),
          :ok <- write_header(file, name) do
-      counter = :counters.new(1, [])
-
-      query
-      |> Pleroma.Repo.chunk_stream(100)
-      |> Enum.each(fn i ->
-        with {:ok, str} <- fun.(i),
-             :ok <- IO.write(file, str <> ",\n") do
-          :counters.add(counter, 1, 1)
-        end
-      end)
-
-      total = :counters.get(counter, 1)
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, str} <- fun.(i),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
 
       with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
         File.close(file)

From 17562bf4147ab03e171b1f1d365a512f2e5b3202 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 20:43:27 +0400
Subject: [PATCH 047/208] Move API endpoints to `/api/v1/pleroma/backups`

---
 docs/API/pleroma_api.md                            |  4 ++--
 lib/pleroma/web/router.ex                          |  6 +++---
 .../controllers/backup_controller_test.exs         | 14 +++++++-------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index aeb266159..fa3a9a449 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -616,7 +616,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ]
 ```
 
-## `POST /api/pleroma/backups`
+## `POST /api/v1/pleroma/backups`
 ### Create a user backup archive
 
 * Method: `POST`
@@ -635,7 +635,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 }]
 ```
 
-## `GET /api/pleroma/backups`
+## `GET /api/v1/pleroma/backups`
 ### Lists user backups
 
 * Method: `GET`
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e539eeeeb..ad7e315c7 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -295,9 +295,6 @@ defmodule Pleroma.Web.Router do
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)
     post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm)
     delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable)
-
-    get("/backups", BackupController, :index)
-    post("/backups", BackupController, :create)
   end
 
   scope "/oauth", Pleroma.Web.OAuth do
@@ -358,6 +355,9 @@ defmodule Pleroma.Web.Router do
       put("/mascot", MascotController, :update)
 
       post("/scrobble", ScrobbleController, :create)
+
+      get("/backups", BackupController, :index)
+      post("/backups", BackupController, :create)
     end
 
     scope [] do
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
index 5d2f1206e..b2ac74c7d 100644
--- a/test/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -14,14 +14,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
     oauth_access(["read:accounts"])
   end
 
-  test "GET /api/pleroma/backups", %{user: user, conn: conn} do
+  test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
     assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
 
     backup = Backup.get(backup_id)
 
     response =
       conn
-      |> get("/api/pleroma/backups")
+      |> get("/api/v1/pleroma/backups")
       |> json_response_and_validate_schema(:ok)
 
     assert [
@@ -45,11 +45,11 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do
              }
            ] =
              conn
-             |> get("/api/pleroma/backups")
+             |> get("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
   end
 
-  test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
+  test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
@@ -60,7 +60,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
              }
            ] =
              conn
-             |> post("/api/pleroma/backups")
+             |> post("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
 
     Pleroma.Tests.ObanHelpers.perform_all()
@@ -72,14 +72,14 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
              }
            ] =
              conn
-             |> get("/api/pleroma/backups")
+             |> get("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
 
     days = Pleroma.Config.get([Backup, :limit_days])
 
     assert %{"error" => "Last export was less than #{days} days ago"} ==
              conn
-             |> post("/api/pleroma/backups")
+             |> post("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(400)
   end
 end

From e4792ce76af3094d378a3a201ca429ae38203696 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 21:06:16 +0400
Subject: [PATCH 048/208] Do not limit admins

---
 lib/pleroma/backup.ex                         | 10 ++++----
 .../controllers/admin_api_controller_test.exs | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index f5f39431d..e2673db80 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,12 +30,12 @@ defmodule Pleroma.Backup do
     timestamps()
   end
 
-  def create(user, admin_user_id \\ nil) do
+  def create(user, admin_id \\ nil) do
     with :ok <- validate_email_enabled(),
          :ok <- validate_user_email(user),
-         :ok <- validate_limit(user),
+         :ok <- validate_limit(user, admin_id),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      BackupWorker.process(backup, admin_user_id)
+      BackupWorker.process(backup, admin_id)
     end
   end
 
@@ -59,7 +59,9 @@ def delete(backup) do
     end
   end
 
-  defp validate_limit(user) do
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
         days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index 4d331779e..4b3abce0d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2044,6 +2044,30 @@ test "it creates a backup", %{conn: conn} do
 
       assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
     end
+
+    test "it doesn't limit admins", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [_backup] = Repo.all(Pleroma.Backup)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert Repo.aggregate(Pleroma.Backup, :count) == 2
+    end
   end
 end
 

From 8baee855d90530def46dc62b81e6a0cb0c315914 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 21 Sep 2020 21:47:36 +0400
Subject: [PATCH 049/208] Fix emails

---
 lib/pleroma/emails/user_email.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index 5745794ec..806a61fd2 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -198,14 +198,14 @@ def backup_is_ready_email(backup, admin_user_id \\ nil) do
       if is_nil(admin_user_id) do
         """
         <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
-        <p><a href="#{download_url}"></a></p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
         """
       else
         admin = Pleroma.Repo.get(User, admin_user_id)
 
         """
         <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
-        <p><a href="#{download_url}"></a></p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
         """
       end
 

From f1e4333dd7976e8cbef44a3bcfe5c96bef177c6f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 23 Sep 2020 20:23:11 +0400
Subject: [PATCH 050/208] Fix test

---
 test/backup_test.exs                                       | 7 ++++++-
 .../admin_api/controllers/admin_api_controller_test.exs    | 5 ++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/test/backup_test.exs b/test/backup_test.exs
index 0ea40e6fd..23c08b680 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -67,7 +67,12 @@ test "it process a backup record" do
     assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
     refute Backup.get(backup_id)
 
-    assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup))
+    email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
+
+    assert_email_sent(
+      to: {user.name, user.email},
+      html_body: email.html_body
+    )
   end
 
   test "it removes outdated backups after creating a fresh one" do
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index 4b3abce0d..a6dc4f62d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2042,7 +2042,10 @@ test "it creates a backup", %{conn: conn} do
 
       ObanHelpers.perform_all()
 
-      assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
+      email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)
+
+      assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
+      assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
     end
 
     test "it doesn't limit admins", %{conn: conn} do

From 6d5f02a1da81ed7693c5ae364a25bc0b54ee1a38 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 20:34:44 +0400
Subject: [PATCH 051/208] Fix API documentation

---
 docs/API/pleroma_api.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index fa3a9a449..7a0a80dad 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -620,7 +620,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ### Create a user backup archive
 
 * Method: `POST`
-* Authentication: not required
+* Authentication: required
 * Params: none
 * Response: JSON
 * Example response:

From d7a5291b4fa3b7568674c0f7643fe287fcd21eff Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:24:35 +0400
Subject: [PATCH 052/208] Use `Jason.encode/1` for likes and bookmarks

---
 lib/pleroma/backup.ex | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index e2673db80..b43dc94d6 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -194,7 +194,8 @@ defp write(query, dir, name, fun) do
         query
         |> Pleroma.Repo.chunk_stream(100)
         |> Enum.reduce(0, fn i, acc ->
-          with {:ok, str} <- fun.(i),
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
                :ok <- IO.write(file, str <> ",\n") do
             acc + 1
           else
@@ -213,7 +214,7 @@ defp bookmarks(dir, %{id: user_id} = _user) do
     |> where(user_id: ^user_id)
     |> join(:inner, [b], activity in assoc(b, :activity))
     |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
-    |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
   end
 
   defp likes(dir, user) do
@@ -221,7 +222,7 @@ defp likes(dir, user) do
     |> Activity.Queries.by_actor()
     |> Activity.Queries.by_type("Like")
     |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
-    |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
   end
 
   defp statuses(dir, user) do
@@ -239,7 +240,7 @@ defp statuses(dir, user) do
     |> ActivityPub.fetch_activities_query(opts)
     |> write(dir, "outbox", fn a ->
       with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
-        activity |> Map.delete("@context") |> Jason.encode()
+        {:ok, Map.delete(activity, "@context")}
       end
     end)
   end

From 9af9f02f4b3c4eac859a69ab9b2f546a91110287 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:45:03 +0400
Subject: [PATCH 053/208] Use Gettext for error messages

---
 lib/pleroma/backup.ex | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index b43dc94d6..0ebaf02e5 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Backup do
 
   import Ecto.Changeset
   import Ecto.Query
+  import Pleroma.Web.Gettext
 
   require Pleroma.Constants
 
@@ -70,7 +71,14 @@ defp validate_limit(user, nil) do
         if diff > days do
           :ok
         else
-          {:error, "Last export was less than #{days} days ago"}
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
         end
 
       nil ->
@@ -82,11 +90,14 @@ defp validate_email_enabled do
     if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       :ok
     else
-      {:error, "Backups require enabled email"}
+      {:error, dgettext("errors", "Backups require enabled email")}
     end
   end
 
-  defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"}
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
   defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
 
   def get_last(user_id) do

From 08972dd135c200073f5de0c8731b886cc2e72eeb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:50:31 +0400
Subject: [PATCH 054/208] Use Path.join/2

---
 lib/pleroma/backup.ex | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 0ebaf02e5..cee51d7c1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -55,7 +55,7 @@ def new(user) do
   def delete(backup) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
-    with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
       Repo.delete(backup)
     end
   end
@@ -164,7 +164,7 @@ def upload(%__MODULE__{} = backup, zip_path) do
       name: backup.file_name,
       tempfile: zip_path,
       content_type: backup.content_type,
-      path: "backups/" <> backup.file_name
+      path: Path.join("backups", backup.file_name)
     }
 
     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@@ -178,7 +178,7 @@ defp actor(dir, user) do
            UserView.render("user.json", %{user: user})
            |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
            |> Jason.encode() do
-      File.write(dir <> "/actor.json", json)
+      File.write(Path.join(dir, "actor.json"), json)
     end
   end
 
@@ -197,7 +197,7 @@ defp write_header(file, name) do
   end
 
   defp write(query, dir, name, fun) do
-    path = dir <> "/#{name}.json"
+    path = Path.join(dir, "#{name}.json")
 
     with {:ok, file} <- File.open(path, [:write, :utf8]),
          :ok <- write_header(file, name) do

From 8545d533ddee2978e9bf7f3284cc7dcb822a77e6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:53:04 +0400
Subject: [PATCH 055/208] Use to_string/1 instead of :binary.list_to_bin/1

---
 lib/pleroma/backup.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index cee51d7c1..629e879a7 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -148,7 +148,7 @@ def export(%__MODULE__{} = backup) do
          :ok <- bookmarks(dir, backup.user),
          {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
          {:ok, _} <- File.rm_rf(dir) do
-      {:ok, :binary.list_to_bin(zip_path)}
+      {:ok, to_string(zip_path)}
     end
   end
 

From bc3db724030707e9903d161a70b10fe217a83212 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 23:16:56 +0400
Subject: [PATCH 056/208] Use ModerationLog instead of Logger

---
 lib/pleroma/moderation_log.ex                 | 10 ++++++++
 .../controllers/admin_api_controller.ex       |  3 ++-
 .../controllers/admin_api_controller_test.exs | 23 +++++++++++++++++--
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index 47036a6f6..be1e81467 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -651,6 +651,16 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index f7d2fe5b1..8b5310d80 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -686,7 +686,8 @@ def stats(conn, params) do
   def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_by_nickname(nickname),
          {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
-      Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}")
+      ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
+
       json(conn, "")
     end
   end
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index a6dc4f62d..34d48c2c1 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2027,9 +2027,9 @@ test "by instance", %{conn: conn} do
 
   describe "/api/pleroma/backups" do
     test "it creates a backup", %{conn: conn} do
-      admin = insert(:user, is_admin: true)
+      admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true)
       token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user)
+      user = %{id: user_id, nickname: user_nickname} = insert(:user)
 
       assert "" ==
                conn
@@ -2046,6 +2046,25 @@ test "it creates a backup", %{conn: conn} do
 
       assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
       assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
+
+      log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}"
+
+      assert [
+               %{
+                 data: %{
+                   "action" => "create_backup",
+                   "actor" => %{
+                     "id" => ^admin_id,
+                     "nickname" => ^admin_nickname
+                   },
+                   "message" => ^log_message,
+                   "subject" => %{
+                     "id" => ^user_id,
+                     "nickname" => ^user_nickname
+                   }
+                 }
+               }
+             ] = Pleroma.ModerationLog |> Repo.all()
     end
 
     test "it doesn't limit admins", %{conn: conn} do

From 268f7f492ab4d423c89e36f42ad126ad4200e874 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 09:50:51 -0500
Subject: [PATCH 057/208] Update Phoenix and pubsub to solve all the
 System.stacktrace/0 deprecation warnings

---
 mix.exs  | 4 ++--
 mix.lock | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/mix.exs b/mix.exs
index b9ce8c500..6ec3eab8f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -114,10 +114,10 @@ defp oauth_deps do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
-      {:phoenix, "~> 1.4.17"},
+      {:phoenix, "~> 1.5.5"},
       {:tzdata, "~> 1.0.3"},
       {:plug_cowboy, "~> 2.3"},
-      {:phoenix_pubsub, "~> 1.1"},
+      {:phoenix_pubsub, "~> 2.0"},
       {:phoenix_ecto, "~> 4.0"},
       {:ecto_enum, "~> 1.4"},
       {:ecto_sql, "~> 3.4.4"},
diff --git a/mix.lock b/mix.lock
index 2603f70c0..13b14449d 100644
--- a/mix.lock
+++ b/mix.lock
@@ -84,14 +84,14 @@
   "p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
-  "phoenix": {:hex, :phoenix, "1.4.17", "1b1bd4cff7cfc87c94deaa7d60dd8c22e04368ab95499483c50640ef3bd838d8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a8e5d7a3d76d452bb5fb86e8b7bd115f737e4f8efe202a463d4aeb4a5809611"},
+  "phoenix": {:hex, :phoenix, "1.5.5", "9a5a197edc1828c5f138a8ef10524dfecc43e36ab435c14578b1e9b4bd98858c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b10eaf86ad026eafad2ee3dd336f0fb1c95a3711789855d913244e270bde463b"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
   "phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
-  "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm", "1f13f9f0f3e769a667a6b6828d29dec37497a082d195cc52dbef401a9b69bf38"},
+  "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"},
   "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.3.0", "2acfa0db038a7649e0a4614eee970e6ed9a39d191ccd79a03583b51d0da98165", [:mix], [{:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.0", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "b8bbae4b59a676de6b8bd8675eda37bc8b4424812ae429d6fdcb2b039e00003b"},
   "plug": {:hex, :plug, "1.10.4", "41eba7d1a2d671faaf531fa867645bd5a3dce0957d8e2a3f398ccff7d2ef017f", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad1e233fe73d2eec56616568d260777b67f53148a999dc2d048f4eb9778fe4a0"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"},
-  "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.0", "1cb20793aa63a6c619dd18bb33d7a3aa94818e5fd39ad357051a67f26dfa2df6", [:mix], [], "hexpm", "a48b538ae8bf381ffac344520755f3007cc10bd8e90b240af98ea29b69683fc2"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},

From 636c00037d797161c4ecd654345a436452f99415 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 09:58:45 -0500
Subject: [PATCH 058/208] Fix duplicate @doc entries

---
 lib/pleroma/web/activity_pub/publisher.ex             |  4 +---
 lib/pleroma/web/common_api/utils.ex                   | 11 +----------
 .../web/mastodon_api/controllers/auth_controller.ex   |  4 ++--
 .../web/mastodon_api/controllers/status_controller.ex |  8 ++------
 4 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex
index 9c3956683..a2930c1cd 100644
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@@ -242,9 +242,7 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
     end)
   end
 
-  @doc """
-  Publishes an activity to all relevant peers.
-  """
+  # Publishes an activity to all relevant peers.
   def publish(%User{} = actor, %Activity{} = activity) do
     public = is_public?(activity)
 
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 9d7b24eb2..85dcd89dc 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -274,7 +274,7 @@ defp build_attachment_link(_), do: ""
   def format_input(text, format, options \\ [])
 
   @doc """
-  Formatting text to plain text.
+  Formatting text to plain text, BBCode, HTML, or Markdown
   """
   def format_input(text, "text/plain", options) do
     text
@@ -285,9 +285,6 @@ def format_input(text, "text/plain", options) do
         end).()
   end
 
-  @doc """
-  Formatting text as BBCode.
-  """
   def format_input(text, "text/bbcode", options) do
     text
     |> String.replace(~r/\r/, "")
@@ -297,18 +294,12 @@ def format_input(text, "text/bbcode", options) do
     |> Formatter.linkify(options)
   end
 
-  @doc """
-  Formatting text to html.
-  """
   def format_input(text, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
     |> Formatter.linkify(options)
   end
 
-  @doc """
-  Formatting text to markdown.
-  """
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
index 57c0be5fe..a278ca622 100644
--- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
@@ -24,7 +24,7 @@ def login(%{assigns: %{user: %User{}}} = conn, _params) do
     redirect(conn, to: local_mastodon_root_path(conn))
   end
 
-  @doc "Local Mastodon FE login init action"
+  # Local Mastodon FE login init action
   def login(conn, %{"code" => auth_token}) do
     with {:ok, app} <- get_or_make_app(),
          {:ok, auth} <- Authorization.get_by_token(app, auth_token),
@@ -35,7 +35,7 @@ def login(conn, %{"code" => auth_token}) do
     end
   end
 
-  @doc "Local Mastodon FE callback action"
+  # Local Mastodon FE callback action
   def login(conn, _) do
     with {:ok, app} <- get_or_make_app() do
       path =
diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index ecfa38489..c85757f26 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -123,9 +123,8 @@ def index(%{assigns: %{user: user}} = conn, %{ids: ids} = _params) do
 
   @doc """
   POST /api/v1/statuses
-
-  Creates a scheduled status when `scheduled_at` param is present and it's far enough
   """
+  # Creates a scheduled status when `scheduled_at` param is present and it's far enough
   def create(
         %{
           assigns: %{user: user},
@@ -156,11 +155,8 @@ def create(
     end
   end
 
-  @doc """
-  POST /api/v1/statuses
 
-  Creates a regular status
-  """
+  # Creates a regular status
   def create(%{assigns: %{user: user}, body_params: %{status: _} = params} = conn, _) do
     params = Map.put(params, :in_reply_to_status_id, params[:in_reply_to_id])
 

From d3106c69c80af58244faa0373b01c618371f84e0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 10:02:20 -0500
Subject: [PATCH 059/208] Fix incompatible type (Elixir 1.11)

---
 lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index bb81d8888..694eb559b 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -30,7 +30,7 @@ defp do_purge(method, url, headers, options) do
       {:ok, %{status: status} = env} when 400 <= status and status < 500 ->
         {:error, env}
 
-      {:error, error} = error ->
+      {:error, error} ->
         error
 
       _ ->

From 218a3e61e1692058aaf5f16506cbf276d2260722 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 10:04:25 -0500
Subject: [PATCH 060/208] Fix incompatible types warning (Elixir 1.11)

---
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 732c44271..8916aba5f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -412,7 +412,7 @@ defp handle_user_activity(
       object =
         object
         |> Map.merge(Map.take(params, ["to", "cc"]))
-        |> Map.put("attributedTo", user.ap_id())
+        |> Map.put("attributedTo", user.ap_id)
         |> Transmogrifier.fix_object()
 
       ActivityPub.create(%{
@@ -456,7 +456,7 @@ def update_outbox(
         %{assigns: %{user: %User{nickname: nickname} = user}} = conn,
         %{"nickname" => nickname} = params
       ) do
-    actor = user.ap_id()
+    actor = user.ap_id
 
     params =
       params

From f3bc076f09dc6df97e3b4ff1704b41e68040354f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 10:23:20 -0500
Subject: [PATCH 061/208] Temporarily soft-fork prometheus_ex

---
 mix.exs  | 5 ++++-
 mix.lock | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index 6ec3eab8f..185734f43 100644
--- a/mix.exs
+++ b/mix.exs
@@ -165,7 +165,10 @@ defp deps do
       {:telemetry, "~> 0.3"},
       {:poolboy, "~> 1.5"},
       {:prometheus, "~> 4.6"},
-      {:prometheus_ex, "~> 3.0"},
+      {:prometheus_ex,
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/prometheus.ex.git",
+       ref: "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5",
+       override: true},
       {:prometheus_plugs, "~> 1.1"},
       {:prometheus_phoenix, "~> 1.3"},
       {:prometheus_ecto, "~> 1.4"},
diff --git a/mix.lock b/mix.lock
index 13b14449d..c23f4f843 100644
--- a/mix.lock
+++ b/mix.lock
@@ -99,7 +99,7 @@
   "pot": {:hex, :pot, "0.11.0", "61bad869a94534739dd4614a25a619bc5c47b9970e9a0ea5bef4628036fc7a16", [:rebar3], [], "hexpm", "57ee6ee6bdeb639661ffafb9acefe3c8f966e45394de6a766813bb9e1be4e54b"},
   "prometheus": {:hex, :prometheus, "4.6.0", "20510f381db1ccab818b4cf2fac5fa6ab5cc91bc364a154399901c001465f46f", [:mix, :rebar3], [], "hexpm", "4905fd2992f8038eccd7aa0cd22f40637ed618c0bed1f75c05aacec15b7545de"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"},
-  "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"},
+  "prometheus_ex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/prometheus.ex.git", "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5", [ref: "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5"]},
   "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"},
   "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"},
   "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"},

From 570a406b7a307000a985c792ccffeac2142eeec4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 10:31:08 -0500
Subject: [PATCH 062/208] use Phoenix.ConnTest is deprecated

---
 test/support/conn_case.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex
index 7ef681258..9316a82e4 100644
--- a/test/support/conn_case.ex
+++ b/test/support/conn_case.ex
@@ -22,7 +22,8 @@ defmodule Pleroma.Web.ConnCase do
   using do
     quote do
       # Import conveniences for testing with connections
-      use Phoenix.ConnTest
+      import Plug.Conn
+      import Phoenix.ConnTest
       use Pleroma.Tests.Helpers
       import Pleroma.Web.Router.Helpers
 

From fddea9e3ceb11ce2f749d23e7f3c2fa6d6c8df83 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 10:34:24 -0500
Subject: [PATCH 063/208] :pubsub is deprecated and replaced wit :pubsub_server

---
 config/config.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 2e6b0796a..0c73ecb59 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -143,7 +143,7 @@
   secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
   signing_salt: "CqaoopA2",
   render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
-  pubsub: [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2],
+  pubsub_server: Pleroma.PubSub,
   secure_cookie_flag: true,
   extra_cookie_attrs: [
     "SameSite=Lax"

From 6d1666a080ad97ba1233d50ba36d3b8a136f75a7 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Wed, 7 Oct 2020 16:44:52 +0000
Subject: [PATCH 064/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index 694eb559b..0b0cde68c 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -30,7 +30,7 @@ defp do_purge(method, url, headers, options) do
       {:ok, %{status: status} = env} when 400 <= status and status < 500 ->
         {:error, env}
 
-      {:error, error} ->
+      {:error, _} = error ->
         error
 
       _ ->

From 8caa6cf91d0d354ce64c1d923112d358295222a2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 11:47:10 -0500
Subject: [PATCH 065/208] Transport.connect/7 is deprecated

---
 lib/transports.ex | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/transports.ex b/lib/transports.ex
index aab7fad99..1ed3a942d 100644
--- a/lib/transports.ex
+++ b/lib/transports.ex
@@ -31,7 +31,12 @@ def init(%Plug.Conn{method: "GET"} = conn, {endpoint, handler, transport}) do
 
     case conn do
       %{halted: false} = conn ->
-        case Transport.connect(endpoint, handler, transport, __MODULE__, nil, conn.params) do
+        case Transport.connect(%{
+               endpoint: endpoint,
+               transport: transport,
+               options: [serializer: nil],
+               params: conn.params
+             }) do
           {:ok, socket} ->
             {:ok, conn, {__MODULE__, {socket, opts}}}
 

From 87fc5a40f4cf5aca7dab9ee5ac11bea35d1281b6 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 11:52:23 -0500
Subject: [PATCH 066/208] instrumenters has no effect in Endpoint anymore

---
 config/config.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 0c73ecb59..47fc7957b 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -123,7 +123,6 @@
 
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
-  instrumenters: [Pleroma.Web.Endpoint.Instrumenter],
   url: [host: "localhost"],
   http: [
     ip: {127, 0, 0, 1},

From 42e78a08b28f3faee21b803c1e621e96bbf5c731 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 12:30:55 -0500
Subject: [PATCH 067/208] Fix rendering of reports

---
 lib/pleroma/web/admin_api/views/report_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex
index 773f798fe..535556370 100644
--- a/lib/pleroma/web/admin_api/views/report_view.ex
+++ b/lib/pleroma/web/admin_api/views/report_view.ex
@@ -52,7 +52,7 @@ def render("show.json", %{report: report, user: user, account: account, statuses
   end
 
   def render("index_notes.json", %{notes: notes}) when is_list(notes) do
-    Enum.map(notes, &render(__MODULE__, "show_note.json", &1))
+    Enum.map(notes, &render(__MODULE__, "show_note.json", Map.from_struct(&1)))
   end
 
   def render("index_notes.json", _), do: []

From 70880d54f85a96d07b6c72adfbf3f1a7c50f95a2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 12:55:16 -0500
Subject: [PATCH 068/208] @env is not used

---
 lib/pleroma/application.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index e73d89350..02dd39939 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -99,7 +99,7 @@ def start(_type, _args) do
         ] ++
         task_children(@env) ++
         dont_run_in_test(@env) ++
-        chat_child(@env, chat_enabled?()) ++
+        chat_child(chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
           Pleroma.Gopher.Server
@@ -201,11 +201,11 @@ defp dont_run_in_test(_) do
     ]
   end
 
-  defp chat_child(_env, true) do
+  defp chat_child(true) do
     [Pleroma.Web.ChatChannel.ChatChannelState]
   end
 
-  defp chat_child(_, _), do: []
+  defp chat_child(_), do: []
 
   defp task_children(:test) do
     [

From 8156940a49df17c00c05bfe60223b165f9dc034b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 13:28:39 -0500
Subject: [PATCH 069/208] Compatibility with phoenix_pubsub 2.0.0

---
 lib/pleroma/application.ex   | 5 ++++-
 lib/pleroma/web/web.ex       | 2 +-
 test/support/channel_case.ex | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 02dd39939..fe94b56f4 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -202,7 +202,10 @@ defp dont_run_in_test(_) do
   end
 
   defp chat_child(true) do
-    [Pleroma.Web.ChatChannel.ChatChannelState]
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
   end
 
   defp chat_child(_), do: []
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index 4f9281851..c319d223c 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -177,7 +177,7 @@ def router do
   def channel do
     quote do
       # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
-      use Phoenix.Channel
+      import Phoenix.Channel
       import Pleroma.Web.Gettext
     end
   end
diff --git a/test/support/channel_case.ex b/test/support/channel_case.ex
index d63a0f06b..114184a9f 100644
--- a/test/support/channel_case.ex
+++ b/test/support/channel_case.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.ChannelCase do
   using do
     quote do
       # Import conveniences for testing with channels
-      use Phoenix.ChannelTest
+      import Phoenix.ChannelTest
       use Pleroma.Tests.Helpers
 
       # The default endpoint for testing

From 822e4472f3de1492aed6948514bc357c9adb934c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 15:06:16 -0500
Subject: [PATCH 070/208] Fix incorrect use of connect/1

Hint from Phoenix 1.4.17, which has a connect/7 shim:

lib/phoenix/socket/transport.ex:

  def connect(endpoint, handler, _transport_name, transport, serializers, params, _pid \\ self()) do
    IO.warn "Phoenix.Socket.Transport.connect/7 is deprecated"

    handler.connect(%{
      endpoint: endpoint,
      transport: transport,
      options: [serializer: serializers],
      params: params
    })
  end
---
 lib/transports.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/transports.ex b/lib/transports.ex
index 1ed3a942d..c3665bebe 100644
--- a/lib/transports.ex
+++ b/lib/transports.ex
@@ -31,7 +31,7 @@ def init(%Plug.Conn{method: "GET"} = conn, {endpoint, handler, transport}) do
 
     case conn do
       %{halted: false} = conn ->
-        case Transport.connect(%{
+        case handler.connect(%{
                endpoint: endpoint,
                transport: transport,
                options: [serializer: nil],

From ed6511a086694fc163b488d807f17d246f80ad5b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 7 Oct 2020 15:28:29 -0500
Subject: [PATCH 071/208] Lint

---
 lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index c85757f26..a47a7af95 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -155,7 +155,6 @@ def create(
     end
   end
 
-
   # Creates a regular status
   def create(%{assigns: %{user: user}, body_params: %{status: _} = params} = conn, _) do
     params = Map.put(params, :in_reply_to_status_id, params[:in_reply_to_id])

From 9c672ecbb5d4477cd16d2139a2cb66d3923ac5c8 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 8 Oct 2020 20:01:48 -0500
Subject: [PATCH 072/208] Remote Timeline: add Streaming support

---
 CHANGELOG.md                                  |  1 +
 docs/API/differences_in_mastoapi_responses.md |  6 ++++++
 lib/pleroma/activity/ir/topics.ex             | 13 ++++++++++++-
 lib/pleroma/web/streamer/streamer.ex          |  9 +++++++++
 test/activity/ir/topics_test.exs              | 14 ++++++++++++++
 test/integration/mastodon_websocket_test.exs  |  1 +
 test/web/streamer/streamer_test.exs           |  8 ++++++++
 7 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..0eeffb72f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`
 
 ### Changed
 
diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md
index 38865dc68..bb1000b0b 100644
--- a/docs/API/differences_in_mastoapi_responses.md
+++ b/docs/API/differences_in_mastoapi_responses.md
@@ -9,9 +9,13 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
+Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
+
 ## Statuses
 
 - `visibility`: has an additional possible value `list`
@@ -249,6 +253,8 @@ Has these additional fields under the `pleroma` object:
 
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.
diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index 9e65bedad..fe2e8cb5c 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -40,7 +40,8 @@ defp visibility_tags(object, activity) do
   end
 
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
-    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+    tags ++
+      remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
   defp item_creation_tags(tags, _, _) do
@@ -55,9 +56,19 @@ defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
 
   defp hashtags_to_topics(_), do: []
 
+  defp remote_topics(%{local: true}), do: []
+
+  defp remote_topics(%{actor: actor}) when is_binary(actor),
+    do: ["public:remote:" <> URI.parse(actor).host]
+
+  defp remote_topics(_), do: []
+
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
   defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
 
+  defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
+    do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]
+
   defp attachment_topics(_object, _act), do: ["public:media"]
 end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 5475f18a6..d774f0dd9 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -57,6 +57,15 @@ def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
     {:ok, "hashtag:" <> tag}
   end
 
+  # Allow remote instance streams.
+  def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do
+    {:ok, "public:remote:" <> instance}
+  end
+
+  def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do
+    {:ok, "public:remote:media:" <> instance}
+  end
+
   # Expand user streams.
   def get_topic(
         stream,
diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs
index 14a6e6b71..c8dcb28cc 100644
--- a/test/activity/ir/topics_test.exs
+++ b/test/activity/ir/topics_test.exs
@@ -93,6 +93,13 @@ test "only converts strings to hash tags", %{
 
       refute Enum.member?(topics, "hashtag:2")
     end
+
+    test "non-local action produces public:remote topic", %{activity: activity} do
+      activity = %{activity | local: false, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      assert Enum.member?(topics, "public:remote:lain.com")
+    end
   end
 
   describe "public visibility create events with attachments" do
@@ -124,6 +131,13 @@ test "non-local doesn't produce public:local:media topics", %{activity: activity
 
       refute Enum.member?(topics, "public:local:media")
     end
+
+    test "non-local action produces public:remote:media topic", %{activity: activity} do
+      activity = %{activity | local: false, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      assert Enum.member?(topics, "public:remote:media:lain.com")
+    end
   end
 
   describe "non-public visibility" do
diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs
index 0f2e6cc2b..bb8e795b7 100644
--- a/test/integration/mastodon_websocket_test.exs
+++ b/test/integration/mastodon_websocket_test.exs
@@ -49,6 +49,7 @@ test "requires authentication and a valid token for protected streams" do
   test "allows public streams without authentication" do
     assert {:ok, _} = start_socket("?stream=public")
     assert {:ok, _} = start_socket("?stream=public:local")
+    assert {:ok, _} = start_socket("?stream=public:remote&instance=lain.com")
     assert {:ok, _} = start_socket("?stream=hashtag&tag=lain")
   end
 
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index 185724a9f..1495ed124 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -29,6 +29,14 @@ test "allows public" do
       assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil)
     end
 
+    test "allows instance streams" do
+      assert {:ok, "public:remote:lain.com"} =
+               Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"})
+
+      assert {:ok, "public:remote:media:lain.com"} =
+               Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"})
+    end
+
     test "allows hashtag streams" do
       assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"})
     end

From e1eb54d3899883b5af6a43687a2345543d69ad4a Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 11 Oct 2020 13:37:19 +0300
Subject: [PATCH 073/208] [#3053] Rollback of access control changes in
 ActivityPubController (base actions: :user, :object, :activity).

---
 .../activity_pub/activity_pub_controller.ex   | 56 +++++++++----------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index c78edfb4c..732c44271 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -32,23 +32,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers]
 
-  # Note: :following and :followers must be served even without authentication (as via :api)
-  @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media]
-
-  # Always accessible actions (must perform entity accessibility checks)
-  @no_auth_no_federation_actions [:user, :activity, :object]
-
-  @authenticated_or_federating_actions @federating_only_actions ++
-                                         @auth_only_actions ++ @no_auth_no_federation_actions
-
   plug(FederatingPlug when action in @federating_only_actions)
 
-  plug(EnsureAuthenticatedPlug when action in @auth_only_actions)
-
   plug(
     EnsureAuthenticatedPlug,
-    [unless_func: &FederatingPlug.federating?/1]
-    when action not in @authenticated_or_federating_actions
+    [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions
+  )
+
+  # Note: :following and :followers must be served even without authentication (as via :api)
+  plug(
+    EnsureAuthenticatedPlug
+    when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
   plug(
@@ -72,22 +66,21 @@ defp relay_active?(conn, _) do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
-         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)},
          {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("user.json", %{user: user})
     else
-      _ -> {:error, :not_found}
+      nil -> {:error, :not_found}
+      %{local: false} -> {:error, :not_found}
     end
   end
 
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)},
-         {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -95,15 +88,25 @@ def object(conn, _) do
       |> put_view(ObjectView)
       |> render("object.json", object: object)
     else
-      _ -> {:error, :not_found}
+      {:public?, false} ->
+        {:error, :not_found}
     end
   end
 
+  def track_object_fetch(conn, nil), do: conn
+
+  def track_object_fetch(conn, object_id) do
+    with %{assigns: %{user: %User{id: user_id}}} <- conn do
+      Delivery.create(object_id, user_id)
+    end
+
+    conn
+  end
+
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -111,7 +114,8 @@ def activity(conn, _params) do
       |> put_view(ObjectView)
       |> render("object.json", object: activity)
     else
-      _ -> {:error, :not_found}
+      {:public?, false} -> {:error, :not_found}
+      nil -> {:error, :not_found}
     end
   end
 
@@ -546,14 +550,4 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} =
       |> json(object.data)
     end
   end
-
-  def track_object_fetch(conn, nil), do: conn
-
-  def track_object_fetch(conn, object_id) do
-    with %{assigns: %{user: %User{id: user_id}}} <- conn do
-      Delivery.create(object_id, user_id)
-    end
-
-    conn
-  end
 end

From 89c595b772eaaa8809f5339d708d7dc22e51b662 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 11 Oct 2020 22:34:28 +0300
Subject: [PATCH 074/208] [#3053] Removed target accessibility checks for
 OStatus endpoints delegating to RedirectController. Added tests.

---
 CHANGELOG.md                                  |  1 +
 lib/pleroma/web/ostatus/ostatus_controller.ex | 13 +++----
 lib/pleroma/web/router.ex                     | 38 +++++++++----------
 .../static_fe/static_fe_controller_test.exs   | 23 +++++++++++
 4 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9ae5d0eda..1e7bcca08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
 
 ## Unreleased (Patch)
 
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index b4dc2a87f..e03ca8c0a 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -37,11 +37,10 @@ def object(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <-
            {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -57,11 +56,10 @@ def activity(%{assigns: %{format: format}} = conn, _params)
   def activity(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -72,7 +70,6 @@ def activity(conn, _params) do
   def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
     with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       cond do
         format in ["json", "activity+json"] ->
@@ -100,7 +97,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
           RedirectController.redirector(conn, nil)
       end
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         conn
         |> put_status(404)
         |> RedirectController.redirector(nil, 404)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 97fcaafd5..ef56360ed 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -9,6 +9,18 @@ defmodule Pleroma.Web.Router do
     plug(:accepts, ["html"])
   end
 
+  pipeline :accepts_html_xml do
+    plug(:accepts, ["html", "xml", "rss", "atom"])
+  end
+
+  pipeline :accepts_html_json do
+    plug(:accepts, ["html", "activity+json", "json"])
+  end
+
+  pipeline :accepts_html_xml_json do
+    plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
+  end
+
   pipeline :accepts_xml_rss_atom do
     plug(:accepts, ["xml", "rss", "atom"])
   end
@@ -574,24 +586,10 @@ defmodule Pleroma.Web.Router do
     )
   end
 
-  pipeline :ostatus_html_json do
-    plug(:accepts, ["html", "activity+json", "json"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
-  pipeline :ostatus_html_xml do
-    plug(:accepts, ["html", "xml", "rss", "atom"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
-  pipeline :ostatus_html_xml_json do
-    plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_json)
+    # Note: http signature is only considered for json requests (no auth for non-json requests)
+    pipe_through([:accepts_html_json, :http_signature, Pleroma.Plugs.StaticFEPlug])
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
@@ -604,15 +602,17 @@ defmodule Pleroma.Web.Router do
 
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_xml_json)
+    # Note: http signature is only considered for json requests (no auth for non-json requests)
+    pipe_through([:accepts_html_xml_json, :http_signature, Pleroma.Plugs.StaticFEPlug])
 
-    # Note: for json format responds with user profile (not user feed)
+    # Note: returns user _profile_ for json requests, redirects to user _feed_ for non-json ones
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
   end
 
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_xml)
+    pipe_through([:accepts_html_xml, Pleroma.Plugs.StaticFEPlug])
+
     get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
   end
 
diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs
index bab0b0a7b..8baf5b1ce 100644
--- a/test/web/static_fe/static_fe_controller_test.exs
+++ b/test/web/static_fe/static_fe_controller_test.exs
@@ -78,6 +78,18 @@ test "does not require authentication on non-federating instances", %{
 
       assert html_response(conn, 200) =~ user.nickname
     end
+
+    test "returns 404 for local user with `restrict_unauthenticated/profiles/local` setting", %{
+      conn: conn
+    } do
+      clear_config([:restrict_unauthenticated, :profiles, :local], true)
+
+      local_user = insert(:user, local: true)
+
+      conn
+      |> get("/users/#{local_user.nickname}")
+      |> html_response(404)
+    end
   end
 
   describe "notice html" do
@@ -200,5 +212,16 @@ test "does not require authentication on non-federating instances", %{
 
       assert html_response(conn, 200) =~ "testing a thing!"
     end
+
+    test "returns 404 for local public activity with `restrict_unauthenticated/activities/local` setting",
+         %{conn: conn, user: user} do
+      clear_config([:restrict_unauthenticated, :activities, :local], true)
+
+      {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"})
+
+      conn
+      |> get("/notice/#{activity.id}")
+      |> html_response(404)
+    end
   end
 end

From c4fae2611ff4da7c418de236fa643021ddc778c8 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 9 Oct 2020 11:26:51 -0500
Subject: [PATCH 075/208] Switch Majic to a copy hosted in our GitLab

---
 mix.exs  | 3 ++-
 mix.lock | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/mix.exs b/mix.exs
index c6e94aff7..427329d38 100644
--- a/mix.exs
+++ b/mix.exs
@@ -188,7 +188,8 @@ defp deps do
        git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git",
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
       {:restarter, path: "./restarter"},
-      {:majic, git: "https://github.com/hrefhref/majic", branch: "develop"},
+      {:majic,
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"},
       {:open_api_spex,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
        ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"},
diff --git a/mix.lock b/mix.lock
index 73d84d7cc..1f2885440 100644
--- a/mix.lock
+++ b/mix.lock
@@ -32,7 +32,7 @@
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
   "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
-  "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"},
+  "elixir_make": {:hex, :elixir_make, "0.6.1", "8faa29a5597faba999aeeb72bbb9c91694ef8068f0131192fb199f98d32994ef", [:mix], [], "hexpm", "35d33270680f8d839a4003c3e9f43afb595310a592405a00afc12de4c7f55a18"},
   "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"},
   "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
@@ -65,7 +65,7 @@
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
   "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"},
-  "majic": {:git, "https://github.com/hrefhref/majic", "51ecea6458c4b2fbc98272ce5ab2951f2c0d46cc", [branch: "develop"]},
+  "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
   "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},

From 06934b820e739f8a5a598bb807235b621ca6e2ba Mon Sep 17 00:00:00 2001
From: Ali Riza Keles <ali@sifirbir.xyz>
Date: Mon, 12 Oct 2020 23:20:10 +0100
Subject: [PATCH 076/208] Add ejabberd auth document

---
 docs/configuration/howto_ejabberd.md | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 docs/configuration/howto_ejabberd.md

diff --git a/docs/configuration/howto_ejabberd.md b/docs/configuration/howto_ejabberd.md
new file mode 100644
index 000000000..6940acbda
--- /dev/null
+++ b/docs/configuration/howto_ejabberd.md
@@ -0,0 +1,9 @@
+# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. 
+
+To get the external script please go to [https://github.com/alirizakeles/ejabberd-pleroma-auth](https://github.com/alirizakeles/ejabberd-pleroma-auth) and follow the steps described in README.
+
+After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.

From 9968b7efedc64d0239db5578de7fc66ff4ce894d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 13 Oct 2020 09:31:13 -0500
Subject: [PATCH 077/208] Change user.locked field to user.is_locked

---
 lib/mix/tasks/pleroma/user.ex                 |  6 ++--
 lib/pleroma/user.ex                           | 12 +++----
 lib/pleroma/web/activity_pub/activity_pub.ex  |  4 +--
 lib/pleroma/web/activity_pub/side_effects.ex  |  2 +-
 .../web/activity_pub/views/user_view.ex       |  2 +-
 .../web/admin_api/views/account_view.ex       |  2 +-
 .../api_spec/operations/account_operation.ex  |  2 +-
 .../web/api_spec/operations/chat_operation.ex |  2 +-
 lib/pleroma/web/api_spec/schemas/account.ex   |  4 +--
 lib/pleroma/web/api_spec/schemas/chat.ex      |  2 +-
 lib/pleroma/web/api_spec/schemas/status.ex    |  2 +-
 .../controllers/account_controller.ex         |  2 +-
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 ...01013141127_refactor_locked_user_field.exs | 15 ++++++++
 test/notification_test.exs                    |  8 ++---
 test/tasks/user_test.exs                      |  6 ++--
 test/user_test.exs                            | 18 +++++-----
 test/web/activity_pub/activity_pub_test.exs   |  2 +-
 .../transmogrifier/accept_handling_test.exs   |  4 +--
 .../transmogrifier/follow_handling_test.exs   |  4 +--
 .../transmogrifier/reject_handling_test.exs   |  4 +--
 .../user_update_handling_test.exs             |  2 +-
 test/web/activity_pub/utils_test.exs          |  4 +--
 test/web/common_api/common_api_test.exs       | 10 +++---
 .../update_credentials_test.exs               |  2 +-
 .../controllers/account_controller_test.exs   |  2 +-
 .../follow_request_controller_test.exs        |  2 +-
 .../mastodon_api/views/account_view_test.exs  | 36 +++++++++----------
 28 files changed, 89 insertions(+), 74 deletions(-)
 create mode 100644 priv/repo/migrations/20201013141127_refactor_locked_user_field.exs

diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index e06262804..a8d251411 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -419,7 +419,7 @@ def run(["list"]) do
       |> Enum.each(fn user ->
         shell_info(
           "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
-            user.locked
+            user.is_locked
           }, deactivated: #{user.deactivated}"
         )
       end)
@@ -447,10 +447,10 @@ defp set_admin(user, value) do
   defp set_locked(user, value) do
     {:ok, user} =
       user
-      |> Changeset.change(%{locked: value})
+      |> Changeset.change(%{is_locked: value})
       |> User.update_and_set_cache()
 
-    shell_info("Locked status of #{user.nickname}: #{user.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.is_locked}")
     user
   end
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 09ea80793..87c8bfbd1 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -107,7 +107,7 @@ defmodule Pleroma.User do
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
     field(:following_count, :integer, default: 0)
-    field(:locked, :boolean, default: false)
+    field(:is_locked, :boolean, default: false)
     field(:confirmation_pending, :boolean, default: false)
     field(:password_reset_pending, :boolean, default: false)
     field(:approval_pending, :boolean, default: false)
@@ -436,7 +436,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :avatar,
         :ap_enabled,
         :banner,
-        :locked,
+        :is_locked,
         :last_refreshed_at,
         :uri,
         :follower_address,
@@ -479,7 +479,7 @@ def update_changeset(struct, params \\ %{}) do
         :public_key,
         :inbox,
         :shared_inbox,
-        :locked,
+        :is_locked,
         :no_rich_text,
         :default_scope,
         :banner,
@@ -847,7 +847,7 @@ def needs_update?(_), do: true
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
 
   # "Locked" (self-locked) users demand explicit authorization of follow requests
-  def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
+  def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do
     follow(follower, followed, :follow_pending)
   end
 
@@ -954,7 +954,7 @@ def get_follow_state(
   end
 
   def locked?(%User{} = user) do
-    user.locked || false
+    user.is_locked || false
   end
 
   def get_by_id(id) do
@@ -1601,7 +1601,7 @@ def purge_user_changeset(user) do
       note_count: 0,
       follower_count: 0,
       following_count: 0,
-      locked: false,
+      is_locked: false,
       confirmation_pending: false,
       password_reset_pending: false,
       approval_pending: false,
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eb44cffec..8022f0402 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1228,7 +1228,7 @@ defp object_to_user_data(data) do
         {String.trim(name, ":"), url}
       end)
 
-    locked = data["manuallyApprovesFollowers"] || false
+    is_locked = data["manuallyApprovesFollowers"] || false
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
@@ -1257,7 +1257,7 @@ defp object_to_user_data(data) do
       banner: banner,
       fields: fields,
       emoji: emojis,
-      locked: locked,
+      is_locked: is_locked,
       discoverable: discoverable,
       invisible: invisible,
       avatar: avatar,
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 2eec0ce86..d421ca7af 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -102,7 +102,7 @@ def handle(
          %User{} = followed <- User.get_cached_by_ap_id(followed_user),
          {_, {:ok, _}, _, _} <-
            {:following, User.follow(follower, followed, :follow_pending), follower, followed} do
-      if followed.local && !followed.locked do
+      if followed.local && !followed.is_locked do
         {:ok, accept_data, _} = Builder.accept(followed, object)
         {:ok, _activity, _} = Pipeline.common_pipeline(accept_data, local: true)
       end
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 3a4564912..c6dee61db 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -101,7 +101,7 @@ def render("user.json", %{user: user}) do
       "name" => user.name,
       "summary" => user.bio,
       "url" => user.ap_id,
-      "manuallyApprovesFollowers" => user.locked,
+      "manuallyApprovesFollowers" => user.is_locked,
       "publicKey" => %{
         "id" => "#{user.ap_id}#main-key",
         "owner" => user.ap_id,
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 9c477feab..bda7ea19c 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -39,7 +39,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :fields,
       :name,
       :nickname,
-      :locked,
+      :is_locked,
       :no_rich_text,
       :default_scope,
       :hide_follows,
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index d90ddb787..57f25e1d8 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -519,7 +519,7 @@ defp update_credentials_request do
           description: "Header image encoded using multipart/form-data",
           format: :binary
         },
-        locked: %Schema{
+        is_locked: %Schema{
           allOf: [BooleanLike],
           nullable: true,
           description: "Whether manual approval of follow requests is required."
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 0dcfdb354..df786e480 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -259,7 +259,7 @@ def chats_response do
               "fields" => []
             },
             "statuses_count" => 1,
-            "locked" => false,
+            "is_locked" => false,
             "created_at" => "2020-04-16T13:40:15.000Z",
             "display_name" => "lain",
             "fields" => [],
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index ca79f0747..b8ddf4ab8 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
       header_static: %Schema{type: :string, format: :uri},
       header: %Schema{type: :string, format: :uri},
       id: FlakeID,
-      locked: %Schema{type: :boolean},
+      is_locked: %Schema{type: :boolean},
       note: %Schema{type: :string, format: :html},
       statuses_count: %Schema{type: :integer},
       url: %Schema{type: :string, format: :uri},
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
       "header" => "https://mypleroma.com/images/banner.png",
       "header_static" => "https://mypleroma.com/images/banner.png",
       "id" => "9tKi3esbG7OQgZ2920",
-      "locked" => false,
+      "is_locked" => false,
       "note" => "cofe",
       "pleroma" => %{
         "allow_following_move" => true,
diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index b4986b734..65f908e33 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
           "fields" => []
         },
         "statuses_count" => 1,
-        "locked" => false,
+        "is_locked" => false,
         "created_at" => "2020-04-16T13:40:15.000Z",
         "display_name" => "lain",
         "fields" => [],
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index 947e42890..e6890df2d 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -252,7 +252,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
         "header" => "http://localhost:4001/images/banner.png",
         "header_static" => "http://localhost:4001/images/banner.png",
         "id" => "9toJCsKN7SmSf3aj5c",
-        "locked" => false,
+        "is_locked" => false,
         "note" => "Tester Number 6",
         "pleroma" => %{
           "background_image" => nil,
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 95d8452df..145b46ea1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -177,7 +177,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
     user_params =
       [
         :no_rich_text,
-        :locked,
+        :is_locked,
         :hide_followers_count,
         :hide_follows_count,
         :hide_followers,
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 121ba1693..f382c05a3 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -242,7 +242,7 @@ defp do_render("show.json", %{user: user} = opts) do
       username: username_from_nickname(user.nickname),
       acct: user.nickname,
       display_name: display_name,
-      locked: user.locked,
+      is_locked: user.is_locked,
       created_at: Utils.to_masto_date(user.inserted_at),
       followers_count: followers_count,
       following_count: following_count,
diff --git a/priv/repo/migrations/20201013141127_refactor_locked_user_field.exs b/priv/repo/migrations/20201013141127_refactor_locked_user_field.exs
new file mode 100644
index 000000000..6cd23dbac
--- /dev/null
+++ b/priv/repo/migrations/20201013141127_refactor_locked_user_field.exs
@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.RefactorLockedUserField do
+  use Ecto.Migration
+
+  def up do
+    execute("ALTER TABLE users RENAME COLUMN locked TO is_locked;")
+  end
+
+  def down do
+    execute("ALTER TABLE users RENAME COLUMN is_locked TO locked;")
+  end
+end
diff --git a/test/notification_test.exs b/test/notification_test.exs
index f2e0f0b0d..0e9630f28 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -346,7 +346,7 @@ test "it creates notifications when someone likes user's status with a filtered
   describe "follow / follow_request notifications" do
     test "it creates `follow` notification for approved Follow activity" do
       user = insert(:user)
-      followed_user = insert(:user, locked: false)
+      followed_user = insert(:user, is_locked: false)
 
       {:ok, _, _, _activity} = CommonAPI.follow(user, followed_user)
       assert FollowingRelationship.following?(user, followed_user)
@@ -361,7 +361,7 @@ test "it creates `follow` notification for approved Follow activity" do
 
     test "it creates `follow_request` notification for pending Follow activity" do
       user = insert(:user)
-      followed_user = insert(:user, locked: true)
+      followed_user = insert(:user, is_locked: true)
 
       {:ok, _, _, _activity} = CommonAPI.follow(user, followed_user)
       refute FollowingRelationship.following?(user, followed_user)
@@ -383,7 +383,7 @@ test "it creates `follow_request` notification for pending Follow activity" do
 
     test "it doesn't create a notification for follow-unfollow-follow chains" do
       user = insert(:user)
-      followed_user = insert(:user, locked: false)
+      followed_user = insert(:user, is_locked: false)
 
       {:ok, _, _, _activity} = CommonAPI.follow(user, followed_user)
       assert FollowingRelationship.following?(user, followed_user)
@@ -397,7 +397,7 @@ test "it doesn't create a notification for follow-unfollow-follow chains" do
     end
 
     test "dismisses the notification on follow request rejection" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       follower = insert(:user)
       {:ok, _, _, _follow_activity} = CommonAPI.follow(follower, user)
       assert [notification] = Notification.for_user(user)
diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs
index b8c423c48..09e1f472e 100644
--- a/test/tasks/user_test.exs
+++ b/test/tasks/user_test.exs
@@ -248,14 +248,14 @@ test "All statuses set" do
 
       user = User.get_cached_by_nickname(user.nickname)
       assert user.is_moderator
-      assert user.locked
+      assert user.is_locked
       assert user.is_admin
       refute user.confirmation_pending
     end
 
     test "All statuses unset" do
       user =
-        insert(:user, locked: true, is_moderator: true, is_admin: true, confirmation_pending: true)
+        insert(:user, is_locked: true, is_moderator: true, is_admin: true, confirmation_pending: true)
 
       Mix.Tasks.Pleroma.User.run([
         "set",
@@ -280,7 +280,7 @@ test "All statuses unset" do
 
       user = User.get_cached_by_nickname(user.nickname)
       refute user.is_moderator
-      refute user.locked
+      refute user.is_locked
       refute user.is_admin
       assert user.confirmation_pending
     end
diff --git a/test/user_test.exs b/test/user_test.exs
index d506f7047..d8ac652af 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -174,7 +174,7 @@ test "ap_following returns the following collection for the user" do
 
   test "returns all pending follow requests" do
     unlocked = insert(:user)
-    locked = insert(:user, locked: true)
+    locked = insert(:user, is_locked: true)
     follower = insert(:user)
 
     CommonAPI.follow(follower, unlocked)
@@ -187,7 +187,7 @@ test "returns all pending follow requests" do
   end
 
   test "doesn't return already accepted or duplicate follow requests" do
-    locked = insert(:user, locked: true)
+    locked = insert(:user, is_locked: true)
     pending_follower = insert(:user)
     accepted_follower = insert(:user)
 
@@ -201,7 +201,7 @@ test "doesn't return already accepted or duplicate follow requests" do
   end
 
   test "doesn't return follow requests for deactivated accounts" do
-    locked = insert(:user, locked: true)
+    locked = insert(:user, is_locked: true)
     pending_follower = insert(:user, %{deactivated: true})
 
     CommonAPI.follow(pending_follower, locked)
@@ -211,7 +211,7 @@ test "doesn't return follow requests for deactivated accounts" do
   end
 
   test "clears follow requests when requester is blocked" do
-    followed = insert(:user, locked: true)
+    followed = insert(:user, is_locked: true)
     follower = insert(:user)
 
     CommonAPI.follow(follower, followed)
@@ -299,8 +299,8 @@ test "can't subscribe to a user who blocked us" do
   end
 
   test "local users do not automatically follow local locked accounts" do
-    follower = insert(:user, locked: true)
-    followed = insert(:user, locked: true)
+    follower = insert(:user, is_locked: true)
+    followed = insert(:user, is_locked: true)
 
     {:ok, follower} = User.maybe_direct_follow(follower, followed)
 
@@ -1360,7 +1360,7 @@ test "it deactivates a user, all follow relationships and all activities", %{use
       follower = insert(:user)
       {:ok, follower} = User.follow(follower, user)
 
-      locked_user = insert(:user, name: "locked", locked: true)
+      locked_user = insert(:user, name: "locked", is_locked: true)
       {:ok, _} = User.follow(user, locked_user, :follow_pending)
 
       object = insert(:note, user: user)
@@ -1450,7 +1450,7 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
         note_count: 9,
         follower_count: 9,
         following_count: 9001,
-        locked: true,
+        is_locked: true,
         confirmation_pending: true,
         password_reset_pending: true,
         approval_pending: true,
@@ -1492,7 +1492,7 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
              note_count: 0,
              follower_count: 0,
              following_count: 0,
-             locked: false,
+             is_locked: false,
              confirmation_pending: false,
              password_reset_pending: false,
              approval_pending: false,
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 804305a13..3efc0d1b4 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1120,7 +1120,7 @@ test "creates an undo activity for the last follow" do
 
     test "creates an undo activity for a pending follow request" do
       follower = insert(:user)
-      followed = insert(:user, %{locked: true})
+      followed = insert(:user, %{is_locked: true})
 
       {:ok, _, _, follow_activity} = CommonAPI.follow(follower, followed)
       {:ok, activity} = ActivityPub.unfollow(follower, followed)
diff --git a/test/web/activity_pub/transmogrifier/accept_handling_test.exs b/test/web/activity_pub/transmogrifier/accept_handling_test.exs
index 77d468f5c..c6ff96f08 100644
--- a/test/web/activity_pub/transmogrifier/accept_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/accept_handling_test.exs
@@ -46,7 +46,7 @@ test "it works for incoming accepts which were pre-accepted" do
 
   test "it works for incoming accepts which are referenced by IRI only" do
     follower = insert(:user)
-    followed = insert(:user, locked: true)
+    followed = insert(:user, is_locked: true)
 
     {:ok, _, _, follow_activity} = CommonAPI.follow(follower, followed)
 
@@ -72,7 +72,7 @@ test "it works for incoming accepts which are referenced by IRI only" do
 
   test "it fails for incoming accepts which cannot be correlated" do
     follower = insert(:user)
-    followed = insert(:user, locked: true)
+    followed = insert(:user, is_locked: true)
 
     accept_data =
       File.read!("test/fixtures/mastodon-accept-activity.json")
diff --git a/test/web/activity_pub/transmogrifier/follow_handling_test.exs b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
index 757d90941..4ef8210ad 100644
--- a/test/web/activity_pub/transmogrifier/follow_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
@@ -65,7 +65,7 @@ test "it works for incoming follow requests" do
     end
 
     test "with locked accounts, it does create a Follow, but not an Accept" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
       data =
         File.read!("test/fixtures/mastodon-follow-activity.json")
@@ -188,7 +188,7 @@ test "it works for incoming follow requests from hubzilla" do
 
     test "it works for incoming follows to locked account" do
       pending_follower = insert(:user, ap_id: "http://mastodon.example.org/users/admin")
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
       data =
         File.read!("test/fixtures/mastodon-follow-activity.json")
diff --git a/test/web/activity_pub/transmogrifier/reject_handling_test.exs b/test/web/activity_pub/transmogrifier/reject_handling_test.exs
index 7592fbe1c..5c1451def 100644
--- a/test/web/activity_pub/transmogrifier/reject_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/reject_handling_test.exs
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.RejectHandlingTest do
 
   test "it fails for incoming rejects which cannot be correlated" do
     follower = insert(:user)
-    followed = insert(:user, locked: true)
+    followed = insert(:user, is_locked: true)
 
     accept_data =
       File.read!("test/fixtures/mastodon-reject-activity.json")
@@ -33,7 +33,7 @@ test "it fails for incoming rejects which cannot be correlated" do
 
   test "it works for incoming rejects which are referenced by IRI only" do
     follower = insert(:user)
-    followed = insert(:user, locked: true)
+    followed = insert(:user, is_locked: true)
 
     {:ok, follower} = User.follow(follower, followed)
     {:ok, _, _, follow_activity} = CommonAPI.follow(follower, followed)
diff --git a/test/web/activity_pub/transmogrifier/user_update_handling_test.exs b/test/web/activity_pub/transmogrifier/user_update_handling_test.exs
index 64636656c..7c4d16db7 100644
--- a/test/web/activity_pub/transmogrifier/user_update_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/user_update_handling_test.exs
@@ -154,6 +154,6 @@ test "it works for incoming update activities which lock the account" do
     {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(update_data)
 
     user = User.get_cached_by_ap_id(user.ap_id)
-    assert user.locked == true
+    assert user.is_locked == true
   end
 end
diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs
index d50213545..be9cd7d13 100644
--- a/test/web/activity_pub/utils_test.exs
+++ b/test/web/activity_pub/utils_test.exs
@@ -193,7 +193,7 @@ test "fetches only Create activities" do
 
   describe "update_follow_state_for_all/2" do
     test "updates the state of all Follow activities with the same actor and object" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       follower = insert(:user)
 
       {:ok, _, _, follow_activity} = CommonAPI.follow(follower, user)
@@ -217,7 +217,7 @@ test "updates the state of all Follow activities with the same actor and object"
 
   describe "update_follow_state/2" do
     test "updates the state of the given follow activity" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       follower = insert(:user)
 
       {:ok, _, _, follow_activity} = CommonAPI.follow(follower, user)
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index e34f5a49b..a8e22d44f 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -1071,7 +1071,7 @@ test "also unsubscribes a user" do
 
     test "cancels a pending follow for a local user" do
       follower = insert(:user)
-      followed = insert(:user, locked: true)
+      followed = insert(:user, is_locked: true)
 
       assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} =
                CommonAPI.follow(follower, followed)
@@ -1093,7 +1093,7 @@ test "cancels a pending follow for a local user" do
 
     test "cancels a pending follow for a remote user" do
       follower = insert(:user)
-      followed = insert(:user, locked: true, local: false, ap_enabled: true)
+      followed = insert(:user, is_locked: true, local: false, ap_enabled: true)
 
       assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} =
                CommonAPI.follow(follower, followed)
@@ -1116,7 +1116,7 @@ test "cancels a pending follow for a remote user" do
 
   describe "accept_follow_request/2" do
     test "after acceptance, it sets all existing pending follow request states to 'accept'" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       follower = insert(:user)
       follower_two = insert(:user)
 
@@ -1136,7 +1136,7 @@ test "after acceptance, it sets all existing pending follow request states to 'a
     end
 
     test "after rejection, it sets all existing pending follow request states to 'reject'" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       follower = insert(:user)
       follower_two = insert(:user)
 
@@ -1156,7 +1156,7 @@ test "after rejection, it sets all existing pending follow request states to 're
     end
 
     test "doesn't create a following relationship if the corresponding follow request doesn't exist" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       not_follower = insert(:user)
       CommonAPI.accept_follow_request(not_follower, user)
 
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index 2e6704726..f49dbdf4d 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -102,7 +102,7 @@ test "updates the user's bio", %{conn: conn} do
     end
 
     test "updates the user's locking status", %{conn: conn} do
-      conn = patch(conn, "/api/v1/accounts/update_credentials", %{locked: "true"})
+      conn = patch(conn, "/api/v1/accounts/update_credentials", %{is_locked: "true"})
 
       assert user_data = json_response_and_validate_schema(conn, 200)
       assert user_data["locked"] == true
diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..0b803b7ab 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -706,7 +706,7 @@ test "following / unfollowing a user", %{conn: conn} do
     end
 
     test "cancelling follow request", %{conn: conn} do
-      %{id: other_user_id} = insert(:user, %{locked: true})
+      %{id: other_user_id} = insert(:user, %{is_locked: true})
 
       assert %{"id" => ^other_user_id, "following" => false, "requested" => true} =
                conn
diff --git a/test/web/mastodon_api/controllers/follow_request_controller_test.exs b/test/web/mastodon_api/controllers/follow_request_controller_test.exs
index 6749e0e83..a9dd7cd30 100644
--- a/test/web/mastodon_api/controllers/follow_request_controller_test.exs
+++ b/test/web/mastodon_api/controllers/follow_request_controller_test.exs
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.FollowRequestControllerTest do
 
   describe "locked accounts" do
     setup do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
       %{conn: conn} = oauth_access(["follow"], user: user)
       %{user: user, conn: conn}
     end
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index a5f39b215..90b436376 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -43,7 +43,7 @@ test "Represent a user account" do
       username: "shp",
       acct: user.nickname,
       display_name: user.name,
-      locked: false,
+      is_locked: false,
       created_at: "2017-08-15T15:47:06.000Z",
       followers_count: 3,
       following_count: 0,
@@ -148,7 +148,7 @@ test "Represent a Service(bot) account" do
       username: "shp",
       acct: user.nickname,
       display_name: user.name,
-      locked: false,
+      is_locked: false,
       created_at: "2017-08-15T15:47:06.000Z",
       followers_count: 3,
       following_count: 0,
@@ -332,7 +332,7 @@ test "represent a relationship for the user blocking a domain" do
 
     test "represent a relationship for the user with a pending follow request" do
       user = insert(:user)
-      other_user = insert(:user, locked: true)
+      other_user = insert(:user, is_locked: true)
 
       {:ok, user, other_user, _} = CommonAPI.follow(user, other_user)
       user = User.get_cached_by_id(user.id)
@@ -481,62 +481,62 @@ test "shows zero when no follow requests are pending" do
     end
 
     test "shows non-zero when follow requests are pending" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
-      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{locked: true, follow_requests_count: 1} =
+      assert %{is_locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "decreases when accepting a follow request" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
-      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{locked: true, follow_requests_count: 1} =
+      assert %{is_locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
 
       {:ok, _other_user} = CommonAPI.accept_follow_request(other_user, user)
 
-      assert %{locked: true, follow_requests_count: 0} =
+      assert %{is_locked: true, follow_requests_count: 0} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "decreases when rejecting a follow request" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
-      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{locked: true, follow_requests_count: 1} =
+      assert %{is_locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
 
       {:ok, _other_user} = CommonAPI.reject_follow_request(other_user, user)
 
-      assert %{locked: true, follow_requests_count: 0} =
+      assert %{is_locked: true, follow_requests_count: 0} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "shows non-zero when historical unapproved requests are present" do
-      user = insert(:user, locked: true)
+      user = insert(:user, is_locked: true)
 
-      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      {:ok, user} = User.update_and_set_cache(user, %{locked: false})
+      {:ok, user} = User.update_and_set_cache(user, %{is_locked: false})
 
-      assert %{locked: false, follow_requests_count: 1} =
+      assert %{is_locked: false, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
     end
   end

From 8bacdc36806efd01a7897359ff0fd2c8e24730d2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 13 Oct 2020 09:45:08 -0500
Subject: [PATCH 078/208] Change user.discoverable field to
 user.is_discoverable

---
 lib/pleroma/user.ex                               |  8 ++++----
 lib/pleroma/user/search.ex                        |  2 +-
 lib/pleroma/web/activity_pub/activity_pub.ex      |  4 ++--
 lib/pleroma/web/activity_pub/views/user_view.ex   |  2 +-
 lib/pleroma/web/admin_api/views/account_view.ex   |  2 +-
 .../web/api_spec/operations/account_operation.ex  |  4 ++--
 .../web/api_spec/operations/chat_operation.ex     |  2 +-
 lib/pleroma/web/api_spec/schemas/account.ex       |  4 ++--
 lib/pleroma/web/api_spec/schemas/chat.ex          |  2 +-
 lib/pleroma/web/api_spec/schemas/status.ex        |  2 +-
 .../controllers/account_controller.ex             |  2 +-
 .../web/mastodon_api/views/account_view.ex        |  2 +-
 lib/pleroma/web/metadata/restrict_indexing.ex     |  2 +-
 ...013144052_refactor_discoverable_user_field.exs | 15 +++++++++++++++
 test/support/factory.ex                           |  2 +-
 test/user_search_test.exs                         |  2 +-
 test/user_test.exs                                |  4 ++--
 test/web/admin_api/search_test.exs                |  2 +-
 .../update_credentials_test.exs                   |  8 ++++----
 .../controllers/account_controller_test.exs       |  2 +-
 test/web/mastodon_api/views/account_view_test.exs |  4 ++--
 test/web/metadata/metadata_test.exs               |  8 ++++----
 test/web/metadata/restrict_indexing_test.exs      |  4 ++--
 23 files changed, 52 insertions(+), 37 deletions(-)
 create mode 100644 priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 09ea80793..c4640f928 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -136,7 +136,7 @@ defmodule Pleroma.User do
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
+    field(:is_discoverable, :boolean, default: false)
     field(:invisible, :boolean, default: false)
     field(:allow_following_move, :boolean, default: true)
     field(:skip_thread_containment, :boolean, default: false)
@@ -448,7 +448,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :follower_count,
         :fields,
         :following_count,
-        :discoverable,
+        :is_discoverable,
         :invisible,
         :actor_type,
         :also_known_as,
@@ -495,7 +495,7 @@ def update_changeset(struct, params \\ %{}) do
         :fields,
         :raw_fields,
         :pleroma_settings_store,
-        :discoverable,
+        :is_discoverable,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
@@ -1618,7 +1618,7 @@ def purge_user_changeset(user) do
       pleroma_settings_store: %{},
       fields: [],
       raw_fields: [],
-      discoverable: false,
+      is_discoverable: false,
       also_known_as: []
     })
   end
diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index 35a828008..2dab67211 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -164,7 +164,7 @@ defp filter_invisible_users(query) do
   end
 
   defp filter_discoverable_users(query) do
-    from(q in query, where: q.discoverable == true)
+    from(q in query, where: q.is_discoverable == true)
   end
 
   defp filter_internal_users(query) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eb44cffec..236fefbb6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1232,7 +1232,7 @@ defp object_to_user_data(data) do
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    discoverable = data["discoverable"] || false
+    is_discoverable = data["is_discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
@@ -1258,7 +1258,7 @@ defp object_to_user_data(data) do
       fields: fields,
       emoji: emojis,
       locked: locked,
-      discoverable: discoverable,
+      is_discoverable: is_discoverable,
       invisible: invisible,
       avatar: avatar,
       name: data["name"],
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 3a4564912..81cd7e81d 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -110,7 +110,7 @@ def render("user.json", %{user: user}) do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
-      "discoverable" => user.discoverable,
+      "is_discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 9c477feab..5f3a78a0f 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :skip_thread_containment,
       :pleroma_settings_store,
       :raw_fields,
-      :discoverable,
+      :is_discoverable,
       :actor_type
     ])
     |> Map.merge(%{
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index d90ddb787..1696b19a5 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -606,7 +606,7 @@ defp update_credentials_request do
           description: "Sets the background image of the user.",
           format: :binary
         },
-        discoverable: %Schema{
+        is_discoverable: %Schema{
           allOf: [BooleanLike],
           nullable: true,
           description:
@@ -630,7 +630,7 @@ defp update_credentials_request do
         pleroma_settings_store: %{"pleroma-fe" => %{"key" => "val"}},
         skip_thread_containment: false,
         allow_following_move: false,
-        discoverable: false,
+        is_discoverable: false,
         actor_type: "Person"
       }
     }
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 0dcfdb354..18693a5d7 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -253,7 +253,7 @@ def chats_response do
               "sensitive" => false,
               "note" => "lain",
               "pleroma" => %{
-                "discoverable" => false,
+                "is_discoverable" => false,
                 "actor_type" => "Person"
               },
               "fields" => []
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index ca79f0747..459e7dba6 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -124,7 +124,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
             type: :object,
             properties: %{
               actor_type: ActorType,
-              discoverable: %Schema{
+              is_discoverable: %Schema{
                 type: :boolean,
                 description:
                   "whether the user allows discovery of the account in search results and other services."
@@ -205,7 +205,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
         "note" => "foobar",
         "pleroma" => %{
           "actor_type" => "Person",
-          "discoverable" => false,
+          "is_discoverable" => false,
           "no_rich_text" => false,
           "show_role" => true
         },
diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index b4986b734..da245d0de 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -44,7 +44,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
           "sensitive" => false,
           "note" => "lain",
           "pleroma" => %{
-            "discoverable" => false,
+            "is_discoverable" => false,
             "actor_type" => "Person"
           },
           "fields" => []
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index 947e42890..52b870d63 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -284,7 +284,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
         "source" => %{
           "fields" => [],
           "note" => "Tester Number 6",
-          "pleroma" => %{"actor_type" => "Person", "discoverable" => false},
+          "pleroma" => %{"actor_type" => "Person", "is_discoverable" => false},
           "sensitive" => false
         },
         "statuses_count" => 1,
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 95d8452df..e06ff9307 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -186,7 +186,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
         :show_role,
         :skip_thread_containment,
         :allow_following_move,
-        :discoverable,
+        :is_discoverable,
         :accepts_chat_messages
       ]
       |> Enum.reduce(%{}, fn key, acc ->
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 121ba1693..0636d9cc1 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -261,7 +261,7 @@ defp do_render("show.json", %{user: user} = opts) do
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
-          discoverable: user.discoverable,
+          is_discoverable: user.is_discoverable,
           actor_type: user.actor_type
         }
       },
diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex
index a1dcb6e15..900c2434d 100644
--- a/lib/pleroma/web/metadata/restrict_indexing.ex
+++ b/lib/pleroma/web/metadata/restrict_indexing.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   """
 
   @impl true
-  def build_tags(%{user: %{local: true, discoverable: true}}), do: []
+  def build_tags(%{user: %{local: true, is_discoverable: true}}), do: []
 
   def build_tags(_) do
     [
diff --git a/priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs b/priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs
new file mode 100644
index 000000000..3fdc190cc
--- /dev/null
+++ b/priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs
@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.RefactorDiscoverableUserField do
+  use Ecto.Migration
+
+  def up do
+    execute("ALTER TABLE users RENAME COLUMN discoverable TO is_discoverable;")
+  end
+
+  def down do
+    execute("ALTER TABLE users RENAME COLUMN is_discoverable TO discoverable;")
+  end
+end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index fb82be0c4..80b882ee4 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -31,7 +31,7 @@ def user_factory do
       nickname: sequence(:nickname, &"nick#{&1}"),
       password_hash: Pbkdf2.hash_pwd_salt("test"),
       bio: sequence(:bio, &"Tester Number #{&1}"),
-      discoverable: true,
+      is_discoverable: true,
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},
diff --git a/test/user_search_test.exs b/test/user_search_test.exs
index c4b805005..31d787ffa 100644
--- a/test/user_search_test.exs
+++ b/test/user_search_test.exs
@@ -66,7 +66,7 @@ test "excludes invisible users from results" do
     end
 
     test "excludes users when discoverable is false" do
-      insert(:user, %{nickname: "john 3000", discoverable: false})
+      insert(:user, %{nickname: "john 3000", is_discoverable: false})
       insert(:user, %{nickname: "john 3001"})
 
       users = User.search("john")
diff --git a/test/user_test.exs b/test/user_test.exs
index d506f7047..7d7bf4b78 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1467,7 +1467,7 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
         pleroma_settings_store: %{"q" => "x"},
         fields: [%{"gg" => "qq"}],
         raw_fields: [%{"gg" => "qq"}],
-        discoverable: true,
+        is_discoverable: true,
         also_known_as: ["https://lol.olo/users/loll"]
       })
 
@@ -1509,7 +1509,7 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
              pleroma_settings_store: %{},
              fields: [],
              raw_fields: [],
-             discoverable: false,
+             is_discoverable: false,
              also_known_as: []
            } = user
   end
diff --git a/test/web/admin_api/search_test.exs b/test/web/admin_api/search_test.exs
index d88867c52..ceec64f1e 100644
--- a/test/web/admin_api/search_test.exs
+++ b/test/web/admin_api/search_test.exs
@@ -180,7 +180,7 @@ test "it returns unapproved user" do
 
     test "it returns non-discoverable users" do
       insert(:user)
-      insert(:user, discoverable: false)
+      insert(:user, is_discoverable: false)
 
       {:ok, _results, total} = Search.user()
 
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index 2e6704726..d31cc27cc 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -147,14 +147,14 @@ test "updates the user's hide_followers status", %{conn: conn} do
     end
 
     test "updates the user's discoverable status", %{conn: conn} do
-      assert %{"source" => %{"pleroma" => %{"discoverable" => true}}} =
+      assert %{"source" => %{"pleroma" => %{"is_discoverable" => true}}} =
                conn
-               |> patch("/api/v1/accounts/update_credentials", %{discoverable: "true"})
+               |> patch("/api/v1/accounts/update_credentials", %{is_discoverable: "true"})
                |> json_response_and_validate_schema(:ok)
 
-      assert %{"source" => %{"pleroma" => %{"discoverable" => false}}} =
+      assert %{"source" => %{"pleroma" => %{"is_discoverable" => false}}} =
                conn
-               |> patch("/api/v1/accounts/update_credentials", %{discoverable: "false"})
+               |> patch("/api/v1/accounts/update_credentials", %{is_discoverable: "false"})
                |> json_response_and_validate_schema(:ok)
     end
 
diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..f7eb97dbb 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -1278,7 +1278,7 @@ test "registration from trusted app" do
                  "note" => "",
                  "pleroma" => %{
                    "actor_type" => "Person",
-                   "discoverable" => false,
+                   "is_discoverable" => false,
                    "no_rich_text" => false,
                    "show_role" => true
                  },
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index a5f39b215..3b0454df2 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -69,7 +69,7 @@ test "Represent a user account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Person",
-          discoverable: true
+          is_discoverable: true
         },
         fields: []
       },
@@ -167,7 +167,7 @@ test "Represent a Service(bot) account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Service",
-          discoverable: true
+          is_discoverable: true
         },
         fields: []
       },
diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs
index ca6cbe67f..8fb946540 100644
--- a/test/web/metadata/metadata_test.exs
+++ b/test/web/metadata/metadata_test.exs
@@ -16,14 +16,14 @@ test "for remote user" do
     end
 
     test "for local user" do
-      user = insert(:user, discoverable: false)
+      user = insert(:user, is_discoverable: false)
 
       assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
     end
 
     test "for local user set to discoverable" do
-      user = insert(:user, discoverable: true)
+      user = insert(:user, is_discoverable: true)
 
       refute Pleroma.Web.Metadata.build_tags(%{user: user}) =~
                "<meta content=\"noindex, noarchive\" name=\"robots\">"
@@ -33,14 +33,14 @@ test "for local user set to discoverable" do
   describe "no metadata for private instances" do
     test "for local user set to discoverable" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: true)
+      user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: true)
 
       assert "" = Pleroma.Web.Metadata.build_tags(%{user: user})
     end
 
     test "search exclusion metadata is included" do
       clear_config([:instance, :public], false)
-      user = insert(:user, bio: "This is my secret fedi account bio", discoverable: false)
+      user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: false)
 
       assert ~s(<meta content="noindex, noarchive" name="robots">) ==
                Pleroma.Web.Metadata.build_tags(%{user: user})
diff --git a/test/web/metadata/restrict_indexing_test.exs b/test/web/metadata/restrict_indexing_test.exs
index 6b3a65372..282d132c8 100644
--- a/test/web/metadata/restrict_indexing_test.exs
+++ b/test/web/metadata/restrict_indexing_test.exs
@@ -14,13 +14,13 @@ test "for remote user" do
 
     test "for local user" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true, discoverable: true}
+               user: %Pleroma.User{local: true, is_discoverable: true}
              }) == []
     end
 
     test "for local user when discoverable is false" do
       assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{
-               user: %Pleroma.User{local: true, discoverable: false}
+               user: %Pleroma.User{local: true, is_discoverable: false}
              }) == [{:meta, [name: "robots", content: "noindex, noarchive"], []}]
     end
   end

From 40f3cdc030536d49c80fe7318ff43cd5ba011df4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 13 Oct 2020 10:37:24 -0500
Subject: [PATCH 079/208] JPEG content_type must be image/jpeg

---
 test/pleroma/object_test.exs                           | 10 +++++-----
 test/pleroma/upload/filter/anonymize_filename_test.exs |  2 +-
 test/pleroma/upload/filter/dedupe_test.exs             |  2 +-
 test/pleroma/upload/filter/exiftool_test.exs           |  2 +-
 test/pleroma/upload/filter/mogrifun_test.exs           |  2 +-
 test/pleroma/upload/filter/mogrify_test.exs            |  2 +-
 test/pleroma/upload/filter_test.exs                    |  2 +-
 test/pleroma/upload_test.exs                           |  2 +-
 test/pleroma/uploaders/local_test.exs                  |  4 ++--
 test/pleroma/uploaders/s3_test.exs                     |  2 +-
 test/pleroma/web/activity_pub/activity_pub_test.exs    |  2 +-
 .../object_validators/attachment_validator_test.exs    |  2 +-
 .../object_validators/chat_validation_test.exs         |  6 +++---
 test/pleroma/web/common_api_test.exs                   |  2 +-
 .../controllers/account_controller_test.exs            |  2 +-
 .../mastodon_api/controllers/media_controller_test.exs |  6 +++---
 .../controllers/status_controller_test.exs             |  4 ++--
 .../web/mastodon_api/update_credentials_test.exs       |  6 +++---
 .../views/scheduled_activity_view_test.exs             |  2 +-
 .../pleroma_api/controllers/chat_controller_test.exs   |  2 +-
 .../pleroma_api/controllers/mascot_controller_test.exs |  4 ++--
 .../views/chat_message_reference_view_test.exs         |  2 +-
 test/pleroma/web/plugs/uploaded_media_plug_test.exs    |  2 +-
 test/pleroma/web/push/impl_test.exs                    |  2 +-
 24 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs
index 198d3b1cf..99caba336 100644
--- a/test/pleroma/object_test.exs
+++ b/test/pleroma/object_test.exs
@@ -82,7 +82,7 @@ test "Disabled via config" do
       Pleroma.Config.put([:instance, :cleanup_attachments], false)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -116,7 +116,7 @@ test "in subdirectories" do
       Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -155,7 +155,7 @@ test "with dedupe enabled" do
       File.mkdir_p!(uploads_dir)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -188,7 +188,7 @@ test "with objects that have legacy data.url attribute" do
       Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -225,7 +225,7 @@ test "With custom base_url" do
       Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/upload/filter/anonymize_filename_test.exs b/test/pleroma/upload/filter/anonymize_filename_test.exs
index 19b915cc8..7ef01ce91 100644
--- a/test/pleroma/upload/filter/anonymize_filename_test.exs
+++ b/test/pleroma/upload/filter/anonymize_filename_test.exs
@@ -13,7 +13,7 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do
 
     upload_file = %Upload{
       name: "an… image.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg")
     }
 
diff --git a/test/pleroma/upload/filter/dedupe_test.exs b/test/pleroma/upload/filter/dedupe_test.exs
index 75c7198e1..92a3d7df3 100644
--- a/test/pleroma/upload/filter/dedupe_test.exs
+++ b/test/pleroma/upload/filter/dedupe_test.exs
@@ -18,7 +18,7 @@ test "adds shasum" do
 
     upload = %Upload{
       name: "an… image.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
       tempfile: Path.absname("test/fixtures/image_tmp.jpg")
     }
diff --git a/test/pleroma/upload/filter/exiftool_test.exs b/test/pleroma/upload/filter/exiftool_test.exs
index d4cd4ba11..6b978b64c 100644
--- a/test/pleroma/upload/filter/exiftool_test.exs
+++ b/test/pleroma/upload/filter/exiftool_test.exs
@@ -16,7 +16,7 @@ test "apply exiftool filter" do
 
     upload = %Pleroma.Upload{
       name: "image_with_GPS_data.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/DSCN0010.jpg"),
       tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
     }
diff --git a/test/pleroma/upload/filter/mogrifun_test.exs b/test/pleroma/upload/filter/mogrifun_test.exs
index dc1e9e78f..fc2f68276 100644
--- a/test/pleroma/upload/filter/mogrifun_test.exs
+++ b/test/pleroma/upload/filter/mogrifun_test.exs
@@ -17,7 +17,7 @@ test "apply mogrify filter" do
 
     upload = %Upload{
       name: "an… image.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
       tempfile: Path.absname("test/fixtures/image_tmp.jpg")
     }
diff --git a/test/pleroma/upload/filter/mogrify_test.exs b/test/pleroma/upload/filter/mogrify_test.exs
index bf64b96b3..6dee02e8b 100644
--- a/test/pleroma/upload/filter/mogrify_test.exs
+++ b/test/pleroma/upload/filter/mogrify_test.exs
@@ -18,7 +18,7 @@ test "apply mogrify filter" do
 
     upload = %Pleroma.Upload{
       name: "an… image.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
       tempfile: Path.absname("test/fixtures/image_tmp.jpg")
     }
diff --git a/test/pleroma/upload/filter_test.exs b/test/pleroma/upload/filter_test.exs
index 352b66402..09394929c 100644
--- a/test/pleroma/upload/filter_test.exs
+++ b/test/pleroma/upload/filter_test.exs
@@ -20,7 +20,7 @@ test "applies filters" do
 
     upload = %Pleroma.Upload{
       name: "an… image.jpg",
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
       tempfile: Path.absname("test/fixtures/image_tmp.jpg")
     }
diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs
index 4280bfcac..f52d4dff6 100644
--- a/test/pleroma/upload_test.exs
+++ b/test/pleroma/upload_test.exs
@@ -112,7 +112,7 @@ test "does not allow descriptions longer than the post limit" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "image.jpg"
       }
diff --git a/test/pleroma/uploaders/local_test.exs b/test/pleroma/uploaders/local_test.exs
index 18122ff6c..1ce7be485 100644
--- a/test/pleroma/uploaders/local_test.exs
+++ b/test/pleroma/uploaders/local_test.exs
@@ -19,7 +19,7 @@ test "put file to local folder" do
 
       file = %Pleroma.Upload{
         name: "image.jpg",
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: file_path,
         tempfile: Path.absname("test/fixtures/image_tmp.jpg")
       }
@@ -38,7 +38,7 @@ test "deletes local file" do
 
       file = %Pleroma.Upload{
         name: "image.jpg",
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: file_path,
         tempfile: Path.absname("test/fixtures/image_tmp.jpg")
       }
diff --git a/test/pleroma/uploaders/s3_test.exs b/test/pleroma/uploaders/s3_test.exs
index d949c90a5..e7a013dd8 100644
--- a/test/pleroma/uploaders/s3_test.exs
+++ b/test/pleroma/uploaders/s3_test.exs
@@ -56,7 +56,7 @@ test "it returns path with bucket namespace when namespace is set" do
     setup do
       file_upload = %Pleroma.Upload{
         name: "image-tet.jpg",
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: "test_folder/image-tet.jpg",
         tempfile: Path.absname("test/instance_static/add/shortcode.png")
       }
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 804305a13..8f53d0599 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -1029,7 +1029,7 @@ test "returns reblogs for users for whom reblogs have not been muted" do
   describe "uploading files" do
     setup do
       test_file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
index 558bb3131..760388e80 100644
--- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
@@ -56,7 +56,7 @@ test "it handles our own uploads" do
       user = insert(:user)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/activity_pub/object_validators/chat_validation_test.exs b/test/pleroma/web/activity_pub/object_validators/chat_validation_test.exs
index 16e4808e5..d7e299224 100644
--- a/test/pleroma/web/activity_pub/object_validators/chat_validation_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/chat_validation_test.exs
@@ -77,7 +77,7 @@ test "validates for a basic object with an attachment", %{
       user: user
     } do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -98,7 +98,7 @@ test "validates for a basic object with an attachment in an array", %{
       user: user
     } do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -119,7 +119,7 @@ test "validates for a basic object with an attachment but without content", %{
       user: user
     } do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index e34f5a49b..d19d484b7 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -100,7 +100,7 @@ test "it posts a chat message without content but with an attachment" do
       recipient = insert(:user)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..4ef404541 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -380,7 +380,7 @@ test "gets an users media, excludes reblogs", %{conn: conn} do
       other_user = insert(:user)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs
index 906fd940f..d2bd57515 100644
--- a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.MediaControllerTest do
 
     setup do
       image = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -74,7 +74,7 @@ test "/api/v2/media", %{conn: conn, user: user, image: image} do
 
     setup %{user: actor} do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -106,7 +106,7 @@ test "/api/v1/media/:id good request", %{conn: conn, object: object} do
 
     setup %{user: actor} do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 633a25e50..61359214a 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -167,7 +167,7 @@ test "Get MRF reason when posting a status is rejected by one", %{conn: conn} do
 
     test "posting an undefined status with an attachment", %{user: user, conn: conn} do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -408,7 +408,7 @@ test "creates a scheduled activity with a media attachment", %{user: user, conn:
         |> Kernel.<>("Z")
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs
index fe462caa3..ed1921c91 100644
--- a/test/pleroma/web/mastodon_api/update_credentials_test.exs
+++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs
@@ -222,7 +222,7 @@ test "updates the user's name", %{conn: conn} do
 
     test "updates the user's avatar", %{user: user, conn: conn} do
       new_avatar = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -246,7 +246,7 @@ test "updates the user's avatar", %{user: user, conn: conn} do
 
     test "updates the user's banner", %{user: user, conn: conn} do
       new_header = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
@@ -265,7 +265,7 @@ test "updates the user's banner", %{user: user, conn: conn} do
 
     test "updates the user's background", %{conn: conn, user: user} do
       new_header = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
index fbfd873ef..04f73f5a0 100644
--- a/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/scheduled_activity_view_test.exs
@@ -22,7 +22,7 @@ test "A scheduled activity with a media attachment" do
       |> NaiveDateTime.to_iso8601()
 
     file = %Plug.Upload{
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image.jpg"),
       filename: "an_image.jpg"
     }
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index 11d5ba373..6381f9757 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -105,7 +105,7 @@ test "it fails if there is no content", %{conn: conn, user: user} do
 
     test "it works with an attachment", %{conn: conn, user: user} do
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }
diff --git a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
index e2ead6e15..d6be92869 100644
--- a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
@@ -24,7 +24,7 @@ test "mascot upload" do
     assert json_response_and_validate_schema(ret_conn, 415)
 
     file = %Plug.Upload{
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image.jpg"),
       filename: "an_image.jpg"
     }
@@ -48,7 +48,7 @@ test "mascot retrieving" do
 
     # When a user sets their mascot, we should get that back
     file = %Plug.Upload{
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image.jpg"),
       filename: "an_image.jpg"
     }
diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
index 26272c125..f171a1e55 100644
--- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -19,7 +19,7 @@ test "it displays a chat message" do
     recipient = insert(:user)
 
     file = %Plug.Upload{
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image.jpg"),
       filename: "an_image.jpg"
     }
diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs
index 07f52c8cd..7c8313121 100644
--- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs
+++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs
@@ -11,7 +11,7 @@ defp upload_file(context) do
     File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
     file = %Plug.Upload{
-      content_type: "image/jpg",
+      content_type: "image/jpeg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
       filename: "nice_tf.jpg"
     }
diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs
index 6cab46696..7d8cc999a 100644
--- a/test/pleroma/web/push/impl_test.exs
+++ b/test/pleroma/web/push/impl_test.exs
@@ -219,7 +219,7 @@ test "builds content for chat messages with no content" do
       recipient = insert(:user)
 
       file = %Plug.Upload{
-        content_type: "image/jpg",
+        content_type: "image/jpeg",
         path: Path.absname("test/fixtures/image.jpg"),
         filename: "an_image.jpg"
       }

From 33f4f39b1cf3a6d8ce350da194696b19ca6f3a05 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:39:41 +0400
Subject: [PATCH 080/208] Add pagination for Blocks

---
 .../api_spec/operations/account_operation.ex  |  1 +
 .../controllers/account_controller.ex         | 12 +++--
 .../controllers/account_controller_test.exs   | 46 +++++++++++++++++--
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index d90ddb787..9cd516f05 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -348,6 +348,7 @@ def blocks_operation do
       operationId: "AccountController.blocks",
       description: "View your blocks. See also accounts/:id/{block,unblock}",
       security: [%{"oAuth" => ["read:blocks"]}],
+      parameters: pagination_params(),
       responses: %{
         200 => Operation.response("Accounts", "application/json", array_of_accounts())
       }
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 4f9696d52..1b221e3a1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -448,9 +448,15 @@ def mutes(%{assigns: %{user: user}} = conn, _) do
   end
 
   @doc "GET /api/v1/blocks"
-  def blocks(%{assigns: %{user: user}} = conn, _) do
-    users = User.blocked_users(user, _restrict_deactivated = true)
-    render(conn, "index.json", users: users, for: user, as: :user)
+  def blocks(%{assigns: %{user: user}} = conn, params) do
+    users =
+      user
+      |> User.blocked_users_relation(_restrict_deactivated = true)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true))
+
+    conn
+    |> add_link_headers(users)
+    |> render("index.json", users: users, for: user, as: :user)
   end
 
   @doc "GET /api/v1/endorsements"
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..6ad9dfc39 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1521,16 +1521,52 @@ test "getting a list of mutes" do
 
   test "getting a list of blocks" do
     %{user: user, conn: conn} = oauth_access(["read:blocks"])
-    other_user = insert(:user)
+    %{id: id1} = other_user1 = insert(:user)
+    %{id: id2} = other_user2 = insert(:user)
+    %{id: id3} = other_user3 = insert(:user)
 
-    {:ok, _user_relationship} = User.block(user, other_user)
+    {:ok, _user_relationship} = User.block(user, other_user1)
+    {:ok, _user_relationship} = User.block(user, other_user3)
+    {:ok, _user_relationship} = User.block(user, other_user2)
 
-    conn =
+    result =
       conn
       |> assign(:user, user)
       |> get("/api/v1/blocks")
+      |> json_response_and_validate_schema(200)
 
-    other_user_id = to_string(other_user.id)
-    assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200)
+    assert [id1, id2, id3] == Enum.map(result, & &1["id"])
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id1}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}, %{"id" => ^id3}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}&max_id=#{id3}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}&limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
   end
 end

From 09be8cb33663e1e8a469e8b228e4e9c8ee769e32 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 13 Oct 2020 12:49:43 -0500
Subject: [PATCH 081/208] Credo

---
 test/mix/tasks/pleroma/user_test.exs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/mix/tasks/pleroma/user_test.exs b/test/mix/tasks/pleroma/user_test.exs
index 09e1f472e..ce819f815 100644
--- a/test/mix/tasks/pleroma/user_test.exs
+++ b/test/mix/tasks/pleroma/user_test.exs
@@ -255,7 +255,12 @@ test "All statuses set" do
 
     test "All statuses unset" do
       user =
-        insert(:user, is_locked: true, is_moderator: true, is_admin: true, confirmation_pending: true)
+        insert(:user,
+          is_locked: true,
+          is_moderator: true,
+          is_admin: true,
+          confirmation_pending: true
+        )
 
       Mix.Tasks.Pleroma.User.run([
         "set",

From 6734abcbd448b92d57ec376b796fea1fad18b792 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:58:18 +0400
Subject: [PATCH 082/208] Add pagination for Mutes

---
 .../api_spec/operations/account_operation.ex  |  1 +
 .../controllers/account_controller.ex         | 12 +++--
 .../controllers/account_controller_test.exs   | 49 +++++++++++++++++--
 3 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 9cd516f05..4934b7788 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -335,6 +335,7 @@ def mutes_operation do
       operationId: "AccountController.mutes",
       description: "Accounts the user has muted.",
       security: [%{"oAuth" => ["follow", "read:mutes"]}],
+      parameters: pagination_params(),
       responses: %{
         200 => Operation.response("Accounts", "application/json", array_of_accounts())
       }
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 1b221e3a1..c8606e5d6 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -442,9 +442,15 @@ def follow_by_uri(%{body_params: %{uri: uri}} = conn, _) do
   end
 
   @doc "GET /api/v1/mutes"
-  def mutes(%{assigns: %{user: user}} = conn, _) do
-    users = User.muted_users(user, _restrict_deactivated = true)
-    render(conn, "index.json", users: users, for: user, as: :user)
+  def mutes(%{assigns: %{user: user}} = conn, params) do
+    users =
+      user
+      |> User.muted_users_relation(_restrict_deactivated = true)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true))
+
+    conn
+    |> add_link_headers(users)
+    |> render("index.json", users: users, for: user, as: :user)
   end
 
   @doc "GET /api/v1/blocks"
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index 6ad9dfc39..69f2b6f4a 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1509,14 +1509,53 @@ test "returns an empty list on a bad request", %{conn: conn} do
 
   test "getting a list of mutes" do
     %{user: user, conn: conn} = oauth_access(["read:mutes"])
-    other_user = insert(:user)
+    %{id: id1} = other_user1 = insert(:user)
+    %{id: id2} = other_user2 = insert(:user)
+    %{id: id3} = other_user3 = insert(:user)
 
-    {:ok, _user_relationships} = User.mute(user, other_user)
+    {:ok, _user_relationships} = User.mute(user, other_user1)
+    {:ok, _user_relationships} = User.mute(user, other_user2)
+    {:ok, _user_relationships} = User.mute(user, other_user3)
 
-    conn = get(conn, "/api/v1/mutes")
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes")
+      |> json_response_and_validate_schema(200)
 
-    other_user_id = to_string(other_user.id)
-    assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200)
+    assert [id1, id2, id3] == Enum.map(result, & &1["id"])
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id1}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}, %{"id" => ^id3}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}&max_id=#{id3}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}&limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
   end
 
   test "getting a list of blocks" do

From 2b58b0dbce36042a8acbb41a8ddc8696d2e00d0e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:58:26 +0400
Subject: [PATCH 083/208] Update CHANGELOG

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..216d7bb32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Support pagination of blocks and mutes
 
 ### Changed
 

From 943f65c7399a03075cb6392ce89ae5403ab0c1a0 Mon Sep 17 00:00:00 2001
From: Ali Riza Keles <ali@sifirbir.xyz>
Date: Tue, 13 Oct 2020 19:58:38 +0100
Subject: [PATCH 084/208] Include python script and description

---
 docs/configuration/howto_ejabberd.md | 129 ++++++++++++++++++++++++++-
 1 file changed, 128 insertions(+), 1 deletion(-)

diff --git a/docs/configuration/howto_ejabberd.md b/docs/configuration/howto_ejabberd.md
index 6940acbda..520a0acbc 100644
--- a/docs/configuration/howto_ejabberd.md
+++ b/docs/configuration/howto_ejabberd.md
@@ -4,6 +4,133 @@ If you want to give your Pleroma users an XMPP (chat) account, you can configure
 
 In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. 
 
-To get the external script please go to [https://github.com/alirizakeles/ejabberd-pleroma-auth](https://github.com/alirizakeles/ejabberd-pleroma-auth) and follow the steps described in README.
+Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary.
+
+```bash
+cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py
+chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py
+chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py
+```
+
+Set external auth params in ejabberd.yaml file:
+
+```bash
+auth_method: [external]
+extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py"
+extauth_instances: 3
+auth_use_cache: false
+```
+
+Restart / reload your ejabberd service.
 
 After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.
+
+
+```python
+import sys
+import struct
+import http.client
+from base64 import b64encode
+import logging
+
+
+PLEROMA_HOST = "127.0.0.1"
+PLEROMA_PORT = "4000"
+AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials"
+USER_ENDPOINT = "/api/v1/accounts"
+LOGFILE = "/var/log/ejabberd/pleroma_auth.log"
+
+logging.basicConfig(filename=LOGFILE, level=logging.INFO)
+
+
+# Pleroma functions
+def create_connection():
+    return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT)
+
+
+def verify_credentials(user: str, password: str) -> bool:
+    user_pass_b64 = b64encode("{}:{}".format(
+        user, password).encode('utf-8')).decode("ascii")
+    params = {}
+    headers = {
+        "Authorization": "Basic {}".format(user_pass_b64)
+    }
+
+    try:
+        conn = create_connection()
+        conn.request("GET", AUTH_ENDPOINT, params, headers)
+
+        response = conn.getresponse()
+        if response.status == 200:
+            return True
+
+        return False
+    except Exception as e:
+        logging.info("Can not connect: %s", str(e))
+        return False
+
+
+def does_user_exist(user: str) -> bool:
+    conn = create_connection()
+    conn.request("GET", "{}/{}".format(USER_ENDPOINT, user))
+
+    response = conn.getresponse()
+    if response.status == 200:
+        return True
+
+    return False
+
+
+def auth(username: str, server: str, password: str) -> bool:
+    return verify_credentials(username, password)
+
+
+def isuser(username, server):
+    return does_user_exist(username)
+
+
+def read():
+    (pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8'))
+    pkt = sys.stdin.read(pkt_size)
+    cmd = pkt.split(':')[0]
+    if cmd == 'auth':
+        username, server, password = pkt.split(':', 3)[1:]
+        write(auth(username, server, password))
+    elif cmd == 'isuser':
+        username, server = pkt.split(':', 2)[1:]
+        write(isuser(username, server))
+    elif cmd == 'setpass':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'tryregister':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'removeuser':
+        # u, s = pkt.split(':', 2)[1:]
+        write(False)
+    elif cmd == 'removeuser3':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    else:
+        write(False)
+
+
+def write(result):
+    if result:
+        sys.stdout.write('\x00\x02\x00\x01')
+    else:
+        sys.stdout.write('\x00\x02\x00\x00')
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    logging.info("Starting pleroma ejabberd auth daemon...")
+    while True:
+        try:
+            read()
+        except Exception as e:
+            logging.info(
+                "Error while processing data from ejabberd %s", str(e))
+            pass
+
+```
\ No newline at end of file

From 8b20c4d2752d620cbcd8b5d122abe7d143878a97 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 13 Oct 2020 16:15:28 -0500
Subject: [PATCH 085/208] Missed tests

---
 .../web/mastodon_api/controllers/account_controller_test.exs    | 2 +-
 test/pleroma/web/mastodon_api/update_credentials_test.exs       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index 0b803b7ab..5164063be 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1271,7 +1271,7 @@ test "registration from trusted app" do
                "follow_requests_count" => 0,
                "followers_count" => 0,
                "following_count" => 0,
-               "locked" => false,
+               "is_locked" => false,
                "note" => "",
                "source" => %{
                  "fields" => [],
diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs
index 79d41d1bf..9340b3ae0 100644
--- a/test/pleroma/web/mastodon_api/update_credentials_test.exs
+++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs
@@ -105,7 +105,7 @@ test "updates the user's locking status", %{conn: conn} do
       conn = patch(conn, "/api/v1/accounts/update_credentials", %{is_locked: "true"})
 
       assert user_data = json_response_and_validate_schema(conn, 200)
-      assert user_data["locked"] == true
+      assert user_data["is_locked"] == true
     end
 
     test "updates the user's chat acceptance status", %{conn: conn} do

From 3242cfef20ea05bae53761ffaf54347c4c6f7bda Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 14 Oct 2020 09:54:07 -0500
Subject: [PATCH 086/208] Undo API breaking changes

---
 lib/pleroma/web/api_spec/operations/account_operation.ex  | 4 ++--
 lib/pleroma/web/api_spec/operations/chat_operation.ex     | 2 +-
 lib/pleroma/web/api_spec/schemas/account.ex               | 4 ++--
 test/pleroma/web/mastodon_api/update_credentials_test.exs | 8 ++++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 1696b19a5..d90ddb787 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -606,7 +606,7 @@ defp update_credentials_request do
           description: "Sets the background image of the user.",
           format: :binary
         },
-        is_discoverable: %Schema{
+        discoverable: %Schema{
           allOf: [BooleanLike],
           nullable: true,
           description:
@@ -630,7 +630,7 @@ defp update_credentials_request do
         pleroma_settings_store: %{"pleroma-fe" => %{"key" => "val"}},
         skip_thread_containment: false,
         allow_following_move: false,
-        is_discoverable: false,
+        discoverable: false,
         actor_type: "Person"
       }
     }
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 18693a5d7..0dcfdb354 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -253,7 +253,7 @@ def chats_response do
               "sensitive" => false,
               "note" => "lain",
               "pleroma" => %{
-                "is_discoverable" => false,
+                "discoverable" => false,
                 "actor_type" => "Person"
               },
               "fields" => []
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index 459e7dba6..ca79f0747 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -124,7 +124,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
             type: :object,
             properties: %{
               actor_type: ActorType,
-              is_discoverable: %Schema{
+              discoverable: %Schema{
                 type: :boolean,
                 description:
                   "whether the user allows discovery of the account in search results and other services."
@@ -205,7 +205,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
         "note" => "foobar",
         "pleroma" => %{
           "actor_type" => "Person",
-          "is_discoverable" => false,
+          "discoverable" => false,
           "no_rich_text" => false,
           "show_role" => true
         },
diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs
index 383d351cf..fe462caa3 100644
--- a/test/pleroma/web/mastodon_api/update_credentials_test.exs
+++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs
@@ -147,14 +147,14 @@ test "updates the user's hide_followers status", %{conn: conn} do
     end
 
     test "updates the user's discoverable status", %{conn: conn} do
-      assert %{"source" => %{"pleroma" => %{"is_discoverable" => true}}} =
+      assert %{"source" => %{"pleroma" => %{"discoverable" => true}}} =
                conn
-               |> patch("/api/v1/accounts/update_credentials", %{is_discoverable: "true"})
+               |> patch("/api/v1/accounts/update_credentials", %{discoverable: "true"})
                |> json_response_and_validate_schema(:ok)
 
-      assert %{"source" => %{"pleroma" => %{"is_discoverable" => false}}} =
+      assert %{"source" => %{"pleroma" => %{"discoverable" => false}}} =
                conn
-               |> patch("/api/v1/accounts/update_credentials", %{is_discoverable: "false"})
+               |> patch("/api/v1/accounts/update_credentials", %{discoverable: "false"})
                |> json_response_and_validate_schema(:ok)
     end
 

From b001237b79d50eef1f0636951cddc7b6da20260d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 14 Oct 2020 10:44:18 -0500
Subject: [PATCH 087/208] Finish undoing API breakage

---
 lib/pleroma/web/activity_pub/views/user_view.ex               | 2 +-
 .../web/mastodon_api/controllers/account_controller.ex        | 2 +-
 lib/pleroma/web/mastodon_api/views/account_view.ex            | 2 +-
 .../web/mastodon_api/controllers/account_controller_test.exs  | 2 +-
 test/pleroma/web/mastodon_api/views/account_view_test.exs     | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 81cd7e81d..e57c91149 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -110,7 +110,7 @@ def render("user.json", %{user: user}) do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
-      "is_discoverable" => user.is_discoverable,
+      "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 6453880dc..9d2f42da9 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -186,7 +186,6 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
         :show_role,
         :skip_thread_containment,
         :allow_following_move,
-        :is_discoverable,
         :accepts_chat_messages
       ]
       |> Enum.reduce(%{}, fn key, acc ->
@@ -210,6 +209,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
         if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
       end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
+      |> Maps.put_if_present(:is_discoverable, params[:discoverable])
 
     # What happens here:
     #
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 0636d9cc1..d57537ee3 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -261,7 +261,7 @@ defp do_render("show.json", %{user: user} = opts) do
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
-          is_discoverable: user.is_discoverable,
+          discoverable: user.is_discoverable,
           actor_type: user.actor_type
         }
       },
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index f7eb97dbb..f7f1369e4 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1278,7 +1278,7 @@ test "registration from trusted app" do
                  "note" => "",
                  "pleroma" => %{
                    "actor_type" => "Person",
-                   "is_discoverable" => false,
+                   "discoverable" => false,
                    "no_rich_text" => false,
                    "show_role" => true
                  },
diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs
index 3b0454df2..a5f39b215 100644
--- a/test/pleroma/web/mastodon_api/views/account_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs
@@ -69,7 +69,7 @@ test "Represent a user account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Person",
-          is_discoverable: true
+          discoverable: true
         },
         fields: []
       },
@@ -167,7 +167,7 @@ test "Represent a Service(bot) account" do
         sensitive: false,
         pleroma: %{
           actor_type: "Service",
-          is_discoverable: true
+          discoverable: true
         },
         fields: []
       },

From ed61002815a65962cd4eba8522f090c2998741d4 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 14 Oct 2020 11:03:17 -0500
Subject: [PATCH 088/208] Undo API breaking changes

---
 .../api_spec/operations/account_operation.ex  |  2 +-
 .../web/api_spec/operations/chat_operation.ex |  2 +-
 lib/pleroma/web/api_spec/schemas/account.ex   |  4 ++--
 .../controllers/account_controller.ex         |  2 +-
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 .../controllers/account_controller_test.exs   |  2 +-
 .../mastodon_api/update_credentials_test.exs  |  4 ++--
 .../mastodon_api/views/account_view_test.exs  | 24 +++++++++----------
 8 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 57f25e1d8..d90ddb787 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -519,7 +519,7 @@ defp update_credentials_request do
           description: "Header image encoded using multipart/form-data",
           format: :binary
         },
-        is_locked: %Schema{
+        locked: %Schema{
           allOf: [BooleanLike],
           nullable: true,
           description: "Whether manual approval of follow requests is required."
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index df786e480..0dcfdb354 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -259,7 +259,7 @@ def chats_response do
               "fields" => []
             },
             "statuses_count" => 1,
-            "is_locked" => false,
+            "locked" => false,
             "created_at" => "2020-04-16T13:40:15.000Z",
             "display_name" => "lain",
             "fields" => [],
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index b8ddf4ab8..ca79f0747 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
       header_static: %Schema{type: :string, format: :uri},
       header: %Schema{type: :string, format: :uri},
       id: FlakeID,
-      is_locked: %Schema{type: :boolean},
+      locked: %Schema{type: :boolean},
       note: %Schema{type: :string, format: :html},
       statuses_count: %Schema{type: :integer},
       url: %Schema{type: :string, format: :uri},
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
       "header" => "https://mypleroma.com/images/banner.png",
       "header_static" => "https://mypleroma.com/images/banner.png",
       "id" => "9tKi3esbG7OQgZ2920",
-      "is_locked" => false,
+      "locked" => false,
       "note" => "cofe",
       "pleroma" => %{
         "allow_following_move" => true,
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 5b3980c71..97858a93c 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -177,7 +177,6 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
     user_params =
       [
         :no_rich_text,
-        :is_locked,
         :hide_followers_count,
         :hide_follows_count,
         :hide_followers,
@@ -210,6 +209,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
         if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
       end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
+      |> Maps.put_if_present(:is_locked, params[:locked])
 
     # What happens here:
     #
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index f382c05a3..d54cae732 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -242,7 +242,7 @@ defp do_render("show.json", %{user: user} = opts) do
       username: username_from_nickname(user.nickname),
       acct: user.nickname,
       display_name: display_name,
-      is_locked: user.is_locked,
+      locked: user.is_locked,
       created_at: Utils.to_masto_date(user.inserted_at),
       followers_count: followers_count,
       following_count: following_count,
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index 5164063be..0b803b7ab 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1271,7 +1271,7 @@ test "registration from trusted app" do
                "follow_requests_count" => 0,
                "followers_count" => 0,
                "following_count" => 0,
-               "is_locked" => false,
+               "locked" => false,
                "note" => "",
                "source" => %{
                  "fields" => [],
diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs
index 9340b3ae0..fe462caa3 100644
--- a/test/pleroma/web/mastodon_api/update_credentials_test.exs
+++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs
@@ -102,10 +102,10 @@ test "updates the user's bio", %{conn: conn} do
     end
 
     test "updates the user's locking status", %{conn: conn} do
-      conn = patch(conn, "/api/v1/accounts/update_credentials", %{is_locked: "true"})
+      conn = patch(conn, "/api/v1/accounts/update_credentials", %{locked: "true"})
 
       assert user_data = json_response_and_validate_schema(conn, 200)
-      assert user_data["is_locked"] == true
+      assert user_data["locked"] == true
     end
 
     test "updates the user's chat acceptance status", %{conn: conn} do
diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs
index 90b436376..203e61c71 100644
--- a/test/pleroma/web/mastodon_api/views/account_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs
@@ -43,7 +43,7 @@ test "Represent a user account" do
       username: "shp",
       acct: user.nickname,
       display_name: user.name,
-      is_locked: false,
+      locked: false,
       created_at: "2017-08-15T15:47:06.000Z",
       followers_count: 3,
       following_count: 0,
@@ -148,7 +148,7 @@ test "Represent a Service(bot) account" do
       username: "shp",
       acct: user.nickname,
       display_name: user.name,
-      is_locked: false,
+      locked: false,
       created_at: "2017-08-15T15:47:06.000Z",
       followers_count: 3,
       following_count: 0,
@@ -483,60 +483,60 @@ test "shows zero when no follow requests are pending" do
     test "shows non-zero when follow requests are pending" do
       user = insert(:user, is_locked: true)
 
-      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{is_locked: true, follow_requests_count: 1} =
+      assert %{locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "decreases when accepting a follow request" do
       user = insert(:user, is_locked: true)
 
-      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{is_locked: true, follow_requests_count: 1} =
+      assert %{locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
 
       {:ok, _other_user} = CommonAPI.accept_follow_request(other_user, user)
 
-      assert %{is_locked: true, follow_requests_count: 0} =
+      assert %{locked: true, follow_requests_count: 0} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "decreases when rejecting a follow request" do
       user = insert(:user, is_locked: true)
 
-      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
-      assert %{is_locked: true, follow_requests_count: 1} =
+      assert %{locked: true, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
 
       {:ok, _other_user} = CommonAPI.reject_follow_request(other_user, user)
 
-      assert %{is_locked: true, follow_requests_count: 0} =
+      assert %{locked: true, follow_requests_count: 0} =
                AccountView.render("show.json", %{user: user, for: user})
     end
 
     test "shows non-zero when historical unapproved requests are present" do
       user = insert(:user, is_locked: true)
 
-      assert %{is_locked: true} = AccountView.render("show.json", %{user: user, for: user})
+      assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user})
 
       other_user = insert(:user)
       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
 
       {:ok, user} = User.update_and_set_cache(user, %{is_locked: false})
 
-      assert %{is_locked: false, follow_requests_count: 1} =
+      assert %{locked: false, follow_requests_count: 1} =
                AccountView.render("show.json", %{user: user, for: user})
     end
   end

From 42112b842ce80571e83f6ea35c4ba9ff817264cb Mon Sep 17 00:00:00 2001
From: swentel <swentel@realize.be>
Date: Wed, 14 Oct 2020 21:32:18 +0200
Subject: [PATCH 089/208] Change handle and url of contact person for
 Indigenous client

---
 docs/clients.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/clients.md b/docs/clients.md
index f84295b1f..1e2c14f1b 100644
--- a/docs/clients.md
+++ b/docs/clients.md
@@ -71,7 +71,7 @@ Feel free to contact us to be added to this list!
 ### Indigenous
 - Homepage: <https://indigenous.realize.be/>
 - Source Code: <https://github.com/swentel/indigenous-android/>
-- Contact: [@realize.be@realize.be](@realize.be@realize.be)
+- Contact: [@swentel@realize.be](https://realize.be)
 - Platforms: Android
 - Features: No Streaming
 

From 77bca415951be516f1b48dc6346600856c96ef5c Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Wed, 14 Oct 2020 19:33:54 +0000
Subject: [PATCH 090/208] Apply 3 suggestion(s) to 3 file(s)

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 lib/pleroma/web/api_spec/schemas/chat.ex     | 2 +-
 lib/pleroma/web/api_spec/schemas/status.ex   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 236fefbb6..b04b72106 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1232,7 +1232,7 @@ defp object_to_user_data(data) do
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    is_discoverable = data["is_discoverable"] || false
+    is_discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index da245d0de..b4986b734 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -44,7 +44,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
           "sensitive" => false,
           "note" => "lain",
           "pleroma" => %{
-            "is_discoverable" => false,
+            "discoverable" => false,
             "actor_type" => "Person"
           },
           "fields" => []
diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex
index 52b870d63..947e42890 100644
--- a/lib/pleroma/web/api_spec/schemas/status.ex
+++ b/lib/pleroma/web/api_spec/schemas/status.ex
@@ -284,7 +284,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
         "source" => %{
           "fields" => [],
           "note" => "Tester Number 6",
-          "pleroma" => %{"actor_type" => "Person", "is_discoverable" => false},
+          "pleroma" => %{"actor_type" => "Person", "discoverable" => false},
           "sensitive" => false
         },
         "statuses_count" => 1,

From 7a2f100061913ec59db9957893eb92bb2d150dc4 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 15 Oct 2020 12:28:25 +0200
Subject: [PATCH 091/208] ActivityPub: Show own replies to muted users.

Aligns mute with block behavior.
---
 lib/pleroma/web/activity_pub/activity_pub.ex     |  9 ++++++++-
 .../web/activity_pub/activity_pub_test.exs       | 16 ++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 8022f0402..ff7b9e778 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -827,7 +827,14 @@ defp restrict_muted(query, %{muting_user: %User{} = user} = opts) do
     query =
       from([activity] in query,
         where: fragment("not (? = ANY(?))", activity.actor, ^mutes),
-        where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
+        where:
+          fragment(
+            "not (?->'to' \\?| ?) or ? = ?",
+            activity.data,
+            ^mutes,
+            activity.actor,
+            ^user.ap_id
+          )
       )
 
     unless opts[:skip_preload] do
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 1a8a844ca..e6b6086e6 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -752,6 +752,22 @@ test "does return activities from followed users on blocked domains" do
     refute repeat_activity in activities
   end
 
+  test "returns your own posts regardless of mute" do
+    user = insert(:user)
+    muted = insert(:user)
+
+    {:ok, muted_post} = CommonAPI.post(muted, %{status: "Im stupid"})
+
+    {:ok, reply} =
+      CommonAPI.post(user, %{status: "I'm muting you", in_reply_to_status_id: muted_post.id})
+
+    {:ok, _} = User.mute(user, muted)
+
+    [activity] = ActivityPub.fetch_activities([], %{muting_user: user, skip_preload: true})
+
+    assert activity.id == reply.id
+  end
+
   test "doesn't return muted activities" do
     activity_one = insert(:note_activity)
     activity_two = insert(:note_activity)

From 3985c1b4501010b72b94072951f2ff64c0fd6310 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 15 Oct 2020 16:54:59 +0400
Subject: [PATCH 092/208] Fix warnings

---
 lib/pleroma/web/templates/layout/app.html.eex      |  2 +-
 .../web/templates/layout/email_styled.html.eex     |  2 +-
 .../web/templates/layout/metadata_player.html.eex  |  2 +-
 .../web/templates/layout/static_fe.html.eex        |  2 +-
 test/pleroma/notification_test.exs                 |  2 +-
 .../pleroma/web/activity_pub/activity_pub_test.exs |  8 ++++----
 .../activity_pub/mrf/reject_non_public_test.exs    |  8 ++++----
 .../web/activity_pub/mrf/tag_policy_test.exs       |  2 +-
 .../transmogrifier/announce_handling_test.exs      |  2 +-
 .../web/activity_pub/transmogrifier_test.exs       |  2 +-
 test/pleroma/web/common_api_test.exs               |  2 +-
 test/pleroma/web/fed_sockets/fed_registry_test.exs |  4 ++--
 .../controllers/account_controller_test.exs        |  8 ++++----
 .../controllers/status_controller_test.exs         |  2 +-
 test/pleroma/web/o_auth/o_auth_controller_test.exs | 14 +++++++-------
 .../controllers/emoji_pack_controller_test.exs     |  2 +-
 .../controllers/mascot_controller_test.exs         |  2 +-
 17 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex
index 51603fe0c..3f28f1920 100644
--- a/lib/pleroma/web/templates/layout/app.html.eex
+++ b/lib/pleroma/web/templates/layout/app.html.eex
@@ -228,7 +228,7 @@
   <body>
     <div class="container">
       <h1><%= Pleroma.Config.get([:instance, :name]) %></h1>
-      <%= render @view_module, @view_template, assigns %>
+      <%= @inner_content %>
     </div>
   </body>
 </html>
diff --git a/lib/pleroma/web/templates/layout/email_styled.html.eex b/lib/pleroma/web/templates/layout/email_styled.html.eex
index ca2caaf4d..82cabd889 100644
--- a/lib/pleroma/web/templates/layout/email_styled.html.eex
+++ b/lib/pleroma/web/templates/layout/email_styled.html.eex
@@ -181,7 +181,7 @@
 							</div>
 						</div>
 					<% end %>
-					<%= render @view_module, @view_template, assigns %>
+					<%= @inner_content %>
 
 				</td>
 			</tr>
diff --git a/lib/pleroma/web/templates/layout/metadata_player.html.eex b/lib/pleroma/web/templates/layout/metadata_player.html.eex
index 460f28094..c00f6fa21 100644
--- a/lib/pleroma/web/templates/layout/metadata_player.html.eex
+++ b/lib/pleroma/web/templates/layout/metadata_player.html.eex
@@ -10,7 +10,7 @@ video, audio {
 }
 </style>
 
-<%= render @view_module, @view_template, assigns %>
+<%= @inner_content %>
 
 </body>
 </html>
diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex
index dc0ee2a5c..e6adb526b 100644
--- a/lib/pleroma/web/templates/layout/static_fe.html.eex
+++ b/lib/pleroma/web/templates/layout/static_fe.html.eex
@@ -9,7 +9,7 @@
   </head>
   <body>
     <div class="container">
-      <%= render @view_module, @view_template, assigns %>
+      <%= @inner_content %>
     </div>
   </body>
 </html>
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index f2e0f0b0d..ac43096fb 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -400,7 +400,7 @@ test "dismisses the notification on follow request rejection" do
       user = insert(:user, locked: true)
       follower = insert(:user)
       {:ok, _, _, _follow_activity} = CommonAPI.follow(follower, user)
-      assert [notification] = Notification.for_user(user)
+      assert [_notification] = Notification.for_user(user)
       {:ok, _follower} = CommonAPI.reject_follow_request(follower, user)
       assert [] = Notification.for_user(user)
     end
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 804305a13..9f641ae1c 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -505,22 +505,22 @@ test "increases replies count", %{user: user} do
 
       # public
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, :visibility, "public"))
-      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert %{data: _data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
       assert object.data["repliesCount"] == 1
 
       # unlisted
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, :visibility, "unlisted"))
-      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert %{data: _data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
       assert object.data["repliesCount"] == 2
 
       # private
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, :visibility, "private"))
-      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert %{data: _data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
       assert object.data["repliesCount"] == 2
 
       # direct
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, :visibility, "direct"))
-      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert %{data: _data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
       assert object.data["repliesCount"] == 2
     end
   end
diff --git a/test/pleroma/web/activity_pub/mrf/reject_non_public_test.exs b/test/pleroma/web/activity_pub/mrf/reject_non_public_test.exs
index 58b46b9a2..e08eb3ba6 100644
--- a/test/pleroma/web/activity_pub/mrf/reject_non_public_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/reject_non_public_test.exs
@@ -21,7 +21,7 @@ test "it's allowed when address is public" do
         "type" => "Create"
       }
 
-      assert {:ok, message} = RejectNonPublic.filter(message)
+      assert {:ok, _message} = RejectNonPublic.filter(message)
     end
 
     test "it's allowed when cc address contain public address" do
@@ -34,7 +34,7 @@ test "it's allowed when cc address contain public address" do
         "type" => "Create"
       }
 
-      assert {:ok, message} = RejectNonPublic.filter(message)
+      assert {:ok, _message} = RejectNonPublic.filter(message)
     end
   end
 
@@ -50,7 +50,7 @@ test "it's allowed when addrer of message in the follower addresses of user and
       }
 
       Pleroma.Config.put([:mrf_rejectnonpublic, :allow_followersonly], true)
-      assert {:ok, message} = RejectNonPublic.filter(message)
+      assert {:ok, _message} = RejectNonPublic.filter(message)
     end
 
     test "it's rejected when addrer of message in the follower addresses of user and it disabled in config" do
@@ -80,7 +80,7 @@ test "it's allows when direct messages are allow" do
       }
 
       Pleroma.Config.put([:mrf_rejectnonpublic, :allow_direct], true)
-      assert {:ok, message} = RejectNonPublic.filter(message)
+      assert {:ok, _message} = RejectNonPublic.filter(message)
     end
 
     test "it's reject when direct messages aren't allow" do
diff --git a/test/pleroma/web/activity_pub/mrf/tag_policy_test.exs b/test/pleroma/web/activity_pub/mrf/tag_policy_test.exs
index 6ff71d640..ffc30ba62 100644
--- a/test/pleroma/web/activity_pub/mrf/tag_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/tag_policy_test.exs
@@ -29,7 +29,7 @@ test "allows non-local follow requests" do
       actor = insert(:user, tags: ["mrf_tag:disable-remote-subscription"])
       follower = insert(:user, tags: ["mrf_tag:disable-remote-subscription"], local: true)
       message = %{"object" => actor.ap_id, "type" => "Follow", "actor" => follower.ap_id}
-      assert {:ok, message} = TagPolicy.filter(message)
+      assert {:ok, _message} = TagPolicy.filter(message)
     end
   end
 
diff --git a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
index e895636b5..54335acdb 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
@@ -144,7 +144,7 @@ test "it rejects incoming announces with an inlined activity from another origin
 
     _user = insert(:user, local: false, ap_id: data["actor"])
 
-    assert {:error, e} = Transmogrifier.handle_incoming(data)
+    assert {:error, _e} = Transmogrifier.handle_incoming(data)
   end
 
   test "it does not clobber the addressing on announce activities" do
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 561674f01..4547c84b7 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -101,7 +101,7 @@ test "it fetches reply-to activities if we don't have them" do
       {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
       returned_object = Object.normalize(returned_activity, false)
 
-      assert activity =
+      assert %Activity{} =
                Activity.get_create_by_object_ap_id(
                  "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
                )
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index e34f5a49b..aabfc6db8 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -622,7 +622,7 @@ test "it validates character limits are correctly enforced" do
       assert {:error, "The status is over the character limit"} =
                CommonAPI.post(user, %{status: "foobar"})
 
-      assert {:ok, activity} = CommonAPI.post(user, %{status: "12345"})
+      assert {:ok, _activity} = CommonAPI.post(user, %{status: "12345"})
     end
 
     test "it can handle activities that expire" do
diff --git a/test/pleroma/web/fed_sockets/fed_registry_test.exs b/test/pleroma/web/fed_sockets/fed_registry_test.exs
index 19ac874d6..73aaced46 100644
--- a/test/pleroma/web/fed_sockets/fed_registry_test.exs
+++ b/test/pleroma/web/fed_sockets/fed_registry_test.exs
@@ -52,7 +52,7 @@ test "multiple origins can be added" do
     end
 
     test "will be ignored" do
-      assert {:ok, %SocketInfo{origin: origin, pid: pid_one}} =
+      assert {:ok, %SocketInfo{origin: origin, pid: _pid_one}} =
                FedRegistry.get_fed_socket(@good_domain_origin)
 
       assert origin == "good.domain:80"
@@ -63,7 +63,7 @@ test "will be ignored" do
     test "the newer process will be closed" do
       pid_two = build_test_socket(@good_domain)
 
-      assert {:ok, %SocketInfo{origin: origin, pid: pid_one}} =
+      assert {:ok, %SocketInfo{origin: origin, pid: _pid_one}} =
                FedRegistry.get_fed_socket(@good_domain_origin)
 
       assert origin == "good.domain:80"
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..fbce5665a 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -32,7 +32,7 @@ test "works by id" do
     test "works by nickname" do
       user = insert(:user)
 
-      assert %{"id" => user_id} =
+      assert %{"id" => _user_id} =
                build_conn()
                |> get("/api/v1/accounts/#{user.nickname}")
                |> json_response_and_validate_schema(200)
@@ -43,7 +43,7 @@ test "works by nickname for remote users" do
 
       user = insert(:user, nickname: "user@example.com", local: false)
 
-      assert %{"id" => user_id} =
+      assert %{"id" => _user_id} =
                build_conn()
                |> get("/api/v1/accounts/#{user.nickname}")
                |> json_response_and_validate_schema(200)
@@ -1429,10 +1429,10 @@ test "returns an error if captcha is invalid", %{conn: conn} do
     test "returns lists to which the account belongs" do
       %{user: user, conn: conn} = oauth_access(["read:lists"])
       other_user = insert(:user)
-      assert {:ok, %Pleroma.List{id: list_id} = list} = Pleroma.List.create("Test List", user)
+      assert {:ok, %Pleroma.List{id: _list_id} = list} = Pleroma.List.create("Test List", user)
       {:ok, %{following: _following}} = Pleroma.List.follow(list, other_user)
 
-      assert [%{"id" => list_id, "title" => "Test List"}] =
+      assert [%{"id" => _list_id, "title" => "Test List"}] =
                conn
                |> get("/api/v1/accounts/#{other_user.id}/lists")
                |> json_response_and_validate_schema(200)
diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
index 633a25e50..497a6bb12 100644
--- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
@@ -937,7 +937,7 @@ test "reblogged status for another user" do
         |> get("/api/v1/statuses/#{reblog_activity1.id}")
 
       assert %{
-               "reblog" => %{"id" => id, "reblogged" => false, "reblogs_count" => 2},
+               "reblog" => %{"id" => _id, "reblogged" => false, "reblogs_count" => 2},
                "reblogged" => false,
                "favourited" => false,
                "bookmarked" => false
diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs
index 1200126b8..a00df8cc7 100644
--- a/test/pleroma/web/o_auth/o_auth_controller_test.exs
+++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs
@@ -77,7 +77,7 @@ test "GET /oauth/prepare_request encodes parameters as `state` and redirects", %
           }
         )
 
-      assert response = html_response(conn, 302)
+      assert html_response(conn, 302)
 
       redirect_query = URI.parse(redirected_to(conn)).query
       assert %{"state" => state_param} = URI.decode_query(redirect_query)
@@ -119,7 +119,7 @@ test "with user-bound registration, GET /oauth/<provider>/callback redirects to
           }
         )
 
-      assert response = html_response(conn, 302)
+      assert html_response(conn, 302)
       assert redirected_to(conn) =~ ~r/#{redirect_uri}\?code=.+/
     end
 
@@ -182,7 +182,7 @@ test "on authentication error, GET /oauth/<provider>/callback redirects to `redi
           }
         )
 
-      assert response = html_response(conn, 302)
+      assert html_response(conn, 302)
       assert redirected_to(conn) == app.redirect_uris
       assert get_flash(conn, :error) == "Failed to authenticate: (error description)."
     end
@@ -238,7 +238,7 @@ test "with valid params, POST /oauth/register?op=register redirects to `redirect
           }
         )
 
-      assert response = html_response(conn, 302)
+      assert html_response(conn, 302)
       assert redirected_to(conn) =~ ~r/#{redirect_uri}\?code=.+/
     end
 
@@ -268,7 +268,7 @@ test "with unlisted `redirect_uri`, POST /oauth/register?op=register results in
           }
         )
 
-      assert response = html_response(conn, 401)
+      assert html_response(conn, 401)
     end
 
     test "with invalid params, POST /oauth/register?op=register renders registration_details page",
@@ -336,7 +336,7 @@ test "with valid params, POST /oauth/register?op=connect redirects to `redirect_
           }
         )
 
-      assert response = html_response(conn, 302)
+      assert html_response(conn, 302)
       assert redirected_to(conn) =~ ~r/#{redirect_uri}\?code=.+/
     end
 
@@ -367,7 +367,7 @@ test "with unlisted `redirect_uri`, POST /oauth/register?op=connect results in H
           }
         )
 
-      assert response = html_response(conn, 401)
+      assert html_response(conn, 401)
     end
 
     test "with invalid params, POST /oauth/register?op=connect renders registration_details page",
diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_test.exs
index 386ad8634..3445f0ca0 100644
--- a/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/emoji_pack_controller_test.exs
@@ -569,7 +569,7 @@ test "shows pack.json", %{conn: conn} do
 
     test "for pack name with special chars", %{conn: conn} do
       assert %{
-               "files" => files,
+               "files" => _files,
                "files_count" => 1,
                "pack" => %{
                  "can-download" => true,
diff --git a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
index e2ead6e15..2f509410e 100644
--- a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs
@@ -34,7 +34,7 @@ test "mascot upload" do
       |> put_req_header("content-type", "multipart/form-data")
       |> put("/api/v1/pleroma/mascot", %{"file" => file})
 
-    assert %{"id" => _, "type" => image} = json_response_and_validate_schema(conn, 200)
+    assert %{"id" => _, "type" => _image} = json_response_and_validate_schema(conn, 200)
   end
 
   test "mascot retrieving" do

From a859d9bc15cd1e4ff15bb63d6c114f71783118ef Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 15 Oct 2020 18:05:54 +0400
Subject: [PATCH 093/208] Update dependencies

---
 mix.lock | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/mix.lock b/mix.lock
index c23f4f843..af4a0cdbb 100644
--- a/mix.lock
+++ b/mix.lock
@@ -18,8 +18,9 @@
   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"},
   "cors_plug": {:hex, :cors_plug, "2.0.2", "2b46083af45e4bc79632bd951550509395935d3e7973275b2b743bd63cc942ce", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f0d0e13f71c51fd4ef8b2c7e051388e4dfb267522a83a22392c856de7e46465f"},
   "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
+  "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.0", "69fdb5cf92df6373e15675eb4018cf629f5d8e35e74841bb637d6596cb797bbc", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "42868c229d9a2900a1501c5d0355bfd46e24c862c322b0b4f5a6f14fe0216753"},
   "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},
-  "credo": {:hex, :credo, "1.4.0", "92339d4cbadd1e88b5ee43d427b639b68a11071b6f73854e33638e30a0ea11f5", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1fd3b70dce216574ce3c18bdf510b57e7c4c85c2ec9cad4bff854abaf7e58658"},
+  "credo": {:hex, :credo, "1.4.1", "16392f1edd2cdb1de9fe4004f5ab0ae612c92e230433968eab00aafd976282fc", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "155f8a2989ad77504de5d8291fa0d41320fdcaa6a1030472e9967f285f8c7692"},
   "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt.git", "f63a705f92c26955977ee62a313012e309a4d77a", [ref: "f63a705f92c26955977ee62a313012e309a4d77a"]},
   "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
@@ -84,13 +85,13 @@
   "p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
-  "phoenix": {:hex, :phoenix, "1.5.5", "9a5a197edc1828c5f138a8ef10524dfecc43e36ab435c14578b1e9b4bd98858c", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b10eaf86ad026eafad2ee3dd336f0fb1c95a3711789855d913244e270bde463b"},
-  "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
+  "phoenix": {:hex, :phoenix, "1.5.6", "8298cdb4e0f943242ba8410780a6a69cbbe972fef199b341a36898dd751bdd66", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0dc4d39af1306b6aa5122729b0a95ca779e42c708c6fe7abbb3d336d5379e956"},
+  "phoenix_ecto": {:hex, :phoenix_ecto, "4.2.1", "13f124cf0a3ce0f1948cf24654c7b9f2347169ff75c1123f44674afee6af3b03", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "478a1bae899cac0a6e02be1deec7e2944b7754c04e7d4107fc5a517f877743c0"},
   "phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"},
-  "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.3.0", "2acfa0db038a7649e0a4614eee970e6ed9a39d191ccd79a03583b51d0da98165", [:mix], [{:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.0", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "b8bbae4b59a676de6b8bd8675eda37bc8b4424812ae429d6fdcb2b039e00003b"},
+  "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.3.2", "43d3518349a22b8b1910ea28b4dd5119926d5017b3187db3fbd1a1e05769a851", [:mix], [{:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.0", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "3e2ac4e883db7af0702d75ba00c19901760e8342b91f8f66e13941de552e777f"},
   "plug": {:hex, :plug, "1.10.4", "41eba7d1a2d671faaf531fa867645bd5a3dce0957d8e2a3f398ccff7d2ef017f", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad1e233fe73d2eec56616568d260777b67f53148a999dc2d048f4eb9778fe4a0"},
-  "plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.4.0", "e936ef151751f386804c51f87f7300f5aaae6893cdad726559c3930c6c032948", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e25ddcfc06b1b76e55af79d078b03cbc86bbcb99ce4e5e0a5e4a8114ee039be6"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.0", "1cb20793aa63a6c619dd18bb33d7a3aa94818e5fd39ad357051a67f26dfa2df6", [:mix], [], "hexpm", "a48b538ae8bf381ffac344520755f3007cc10bd8e90b240af98ea29b69683fc2"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
@@ -109,13 +110,13 @@
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"},
   "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"},
-  "swoosh": {:hex, :swoosh, "1.0.0", "c547cfc83f30e12d5d1fdcb623d7de2c2e29a5becfc68bf8f42ba4d23d2c2756", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "b3b08e463f876cb6167f7168e9ad99a069a724e124bcee61847e0e1ed13f4a0d"},
+  "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"},
   "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"},
   "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
   "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
   "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
-  "tzdata": {:hex, :tzdata, "1.0.3", "73470ad29dde46e350c60a66e6b360d3b99d2d18b74c4c349dbebbc27a09a3eb", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a6e1ee7003c4d04ecbd21dd3ec690d4c6662db5d3bbdd7262d53cdf5e7c746c1"},
+  "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"},
   "ueberauth": {:hex, :ueberauth, "0.6.3", "d42ace28b870e8072cf30e32e385579c57b9cc96ec74fa1f30f30da9c14f3cc0", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "afc293d8a1140d6591b53e3eaf415ca92842cb1d32fad3c450c6f045f7f91b60"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"},
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},

From 20e68b30f08e0e0eae691dcf541968e344efdaae Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 14 Jul 2020 07:31:21 +0300
Subject: [PATCH 094/208] added generated  `pleroma.env`

---
 .gitignore                                    |  2 +
 .../CLI_tasks/release_environments.md         |  9 +++
 docs/installation/otp_en.md                   |  6 +-
 installation/pleroma.service                  |  2 +
 lib/mix/tasks/pleroma/release_env.ex          | 64 +++++++++++++++++++
 test/mix/tasks/pleroma/release_env_test.exs   | 30 +++++++++
 6 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100644 docs/administration/CLI_tasks/release_environments.md
 create mode 100644 lib/mix/tasks/pleroma/release_env.ex
 create mode 100644 test/mix/tasks/pleroma/release_env_test.exs

diff --git a/.gitignore b/.gitignore
index 599b52b9e..6ae21e914 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,8 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/*.env
+
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql
diff --git a/docs/administration/CLI_tasks/release_environments.md b/docs/administration/CLI_tasks/release_environments.md
new file mode 100644
index 000000000..36ab43864
--- /dev/null
+++ b/docs/administration/CLI_tasks/release_environments.md
@@ -0,0 +1,9 @@
+# Generate release environment file
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl release_env gen
+```
+
+```sh tab="From Source"
+mix pleroma.release_env gen
+```
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 62d4c8a72..07fabfc0d 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -149,6 +149,9 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
+# Run the environment file generator.
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
+
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -159,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $( cat /opt/pleroma/config/pleroma.env | xargs); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -311,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
-
diff --git a/installation/pleroma.service b/installation/pleroma.service
index 5dcbc1387..ee00a3b7a 100644
--- a/installation/pleroma.service
+++ b/installation/pleroma.service
@@ -17,6 +17,8 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
+; Path to the environment file. the file contains RELEASE_COOKIE and etc 
+EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 
diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
new file mode 100644
index 000000000..cbbbdeff6
--- /dev/null
+++ b/lib/mix/tasks/pleroma/release_env.ex
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnv do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Generate Pleroma environment file."
+  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
+
+  def run(["gen" | rest]) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [
+          force: :boolean,
+          path: :string
+        ],
+        aliases: [
+          p: :path,
+          f: :force
+        ]
+      )
+
+    env_path =
+      get_option(
+        options,
+        :path,
+        "Environment file path",
+        "config/pleroma.env"
+      )
+      |> Path.expand()
+
+    proceed? =
+      if File.exists?(env_path) do
+        get_option(
+          options,
+          :force,
+          "Environment file is exist. Do you want overwritten the #{env_path} file? (y/n)",
+          "n"
+        ) === "y"
+      else
+        true
+      end
+
+    if proceed? do
+      do_generate(env_path)
+
+      shell_info(
+        "The file generated: #{env_path}.\nTo use the enviroment file need to add the line ';EnvironmentFile=#{
+          env_path
+        }' in service file (/installation/pleroma.service)."
+      )
+    end
+  end
+
+  def do_generate(path) do
+    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
+
+    File.mkdir_p!(Path.dirname(path))
+    File.write!(path, content)
+  end
+end
diff --git a/test/mix/tasks/pleroma/release_env_test.exs b/test/mix/tasks/pleroma/release_env_test.exs
new file mode 100644
index 000000000..519f1eba9
--- /dev/null
+++ b/test/mix/tasks/pleroma/release_env_test.exs
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do
+  use ExUnit.Case
+  import ExUnit.CaptureIO, only: [capture_io: 1]
+
+  @path "config/pleroma.test.env"
+
+  def do_clean do
+    if File.exists?(@path) do
+      File.rm_rf(@path)
+    end
+  end
+
+  setup do
+    do_clean()
+    on_exit(fn -> do_clean() end)
+    :ok
+  end
+
+  test "generate pleroma.env" do
+    assert capture_io(fn ->
+             Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"])
+           end) =~ "The file generated"
+
+    assert File.read!(@path) =~ "RELEASE_COOKIE="
+  end
+end

From 14054cd004d91e89644c31d61b08d50ff0df09dd Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 16 Jul 2020 08:52:14 +0300
Subject: [PATCH 095/208] update task messages

---
 installation/init.d/pleroma          |  1 +
 lib/mix/tasks/pleroma/release_env.ex | 32 +++++++++++++++++++---------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma
index 384536f7e..e908cda1b 100755
--- a/installation/init.d/pleroma
+++ b/installation/init.d/pleroma
@@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
+export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 
diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
index cbbbdeff6..63030c5cc 100644
--- a/lib/mix/tasks/pleroma/release_env.ex
+++ b/lib/mix/tasks/pleroma/release_env.ex
@@ -23,14 +23,15 @@ def run(["gen" | rest]) do
         ]
       )
 
-    env_path =
+    file_path =
       get_option(
         options,
         :path,
         "Environment file path",
-        "config/pleroma.env"
+        "./config/pleroma.env"
       )
-      |> Path.expand()
+
+    env_path = Path.expand(file_path)
 
     proceed? =
       if File.exists?(env_path) do
@@ -45,13 +46,24 @@ def run(["gen" | rest]) do
       end
 
     if proceed? do
-      do_generate(env_path)
+      case do_generate(env_path) do
+        {:error, reason} ->
+          shell_error(
+            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
+          )
 
-      shell_info(
-        "The file generated: #{env_path}.\nTo use the enviroment file need to add the line ';EnvironmentFile=#{
-          env_path
-        }' in service file (/installation/pleroma.service)."
-      )
+        _ ->
+          shell_info("\nThe file generated: #{env_path}.\n")
+
+          shell_info("""
+          WARNING: before start pleroma app please to made the file read-only and non-modifiable.
+            Example:
+              chmod 0444 #{file_path}
+              chattr +i #{file_path}
+          """)
+      end
+    else
+      shell_info("\nThe file is exist. #{env_path}.\n")
     end
   end
 
@@ -59,6 +71,6 @@ def do_generate(path) do
     content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
 
     File.mkdir_p!(Path.dirname(path))
-    File.write!(path, content)
+    File.write(path, content)
   end
 end

From 499df7b73aa723aab9c623754b531307b98cf584 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Thu, 16 Jul 2020 13:30:17 +0000
Subject: [PATCH 096/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/mix/tasks/pleroma/release_env.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
index 63030c5cc..4d8b6ff27 100644
--- a/lib/mix/tasks/pleroma/release_env.ex
+++ b/lib/mix/tasks/pleroma/release_env.ex
@@ -56,7 +56,7 @@ def run(["gen" | rest]) do
           shell_info("\nThe file generated: #{env_path}.\n")
 
           shell_info("""
-          WARNING: before start pleroma app please to made the file read-only and non-modifiable.
+          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
             Example:
               chmod 0444 #{file_path}
               chattr +i #{file_path}

From e2333f757a763f7ec7105df6b4de4585fc3eee05 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Thu, 16 Jul 2020 13:30:28 +0000
Subject: [PATCH 097/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/mix/tasks/pleroma/release_env.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
index 4d8b6ff27..9da74ffcf 100644
--- a/lib/mix/tasks/pleroma/release_env.ex
+++ b/lib/mix/tasks/pleroma/release_env.ex
@@ -38,7 +38,7 @@ def run(["gen" | rest]) do
         get_option(
           options,
           :force,
-          "Environment file is exist. Do you want overwritten the #{env_path} file? (y/n)",
+          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
           "n"
         ) === "y"
       else

From 2f6bbd53b5b2da2bf1e3e396b8edbe23c5345b81 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 16 Jul 2020 16:35:09 +0300
Subject: [PATCH 098/208] fix docs

---
 docs/installation/otp_en.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 07fabfc0d..676b10699 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -162,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "export $( cat /opt/pleroma/config/pleroma.env | xargs); ./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance

From 595da6080dc12675eb7673a3190b88ad5a3712ed Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Wed, 5 Aug 2020 19:04:12 +0300
Subject: [PATCH 099/208] fixed install docs

---
 docs/installation/debian_based_en.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 6a9026d94..6badfeed7 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -101,6 +101,10 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
+* Generate the environment file: `sudo -Hu pleroma mix pleroma.release_env gen`
+  * Input path to env file or keep default value `./config/pleroma.env`
+
+
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
 ```shell
@@ -181,6 +185,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
+* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell

From cf53e300f8e850aeacb88e6e6b4d465378acc199 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 7 Aug 2020 15:55:08 +0300
Subject: [PATCH 100/208] added generate the release env to `pleroma.instance
 gen`

---
 docs/administration/CLI_tasks/instance.md |  1 +
 docs/installation/debian_based_en.md      |  3 ---
 lib/mix/tasks/pleroma/instance.ex         | 13 ++++++++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md
index d6913280a..d922b44df 100644
--- a/docs/administration/CLI_tasks/instance.md
+++ b/docs/administration/CLI_tasks/instance.md
@@ -40,3 +40,4 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
+- `--skip-release-env` - skip generation the release environment file
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 6badfeed7..b9fc4e112 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -101,9 +101,6 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
-* Generate the environment file: `sudo -Hu pleroma mix pleroma.release_env gen`
-  * Input path to env file or keep default value `./config/pleroma.env`
-
 
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index fc21ae062..caa7dfbd3 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -36,7 +36,8 @@ def run(["gen" | rest]) do
           listen_port: :string,
           strip_uploads: :string,
           anonymize_uploads: :string,
-          dedupe_uploads: :string
+          dedupe_uploads: :string,
+          skip_release_env: :boolean
         ],
         aliases: [
           o: :output,
@@ -241,6 +242,16 @@ def run(["gen" | rest]) do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
+      if Keyword.get(options, :skip_release_env, false) do
+        shell_info("""
+        Release environment file is skip. Please generate the release env file before start.
+        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
+        """)
+      else
+        shell_info("Generation the environment file:")
+        Mix.Tasks.Pleroma.ReleaseEnv.run(["gen"])
+      end
+
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )

From 2030ffd4904b6ab5e99cefa62887154a49aaf4db Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Sat, 8 Aug 2020 06:49:19 +0300
Subject: [PATCH 101/208] fix test

---
 docs/administration/CLI_tasks/instance.md |  1 +
 lib/mix/tasks/pleroma/instance.ex         | 13 +++++++++++--
 test/mix/tasks/pleroma/instance_test.exs  | 11 ++++++++++-
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md
index d922b44df..982b22bf3 100644
--- a/docs/administration/CLI_tasks/instance.md
+++ b/docs/administration/CLI_tasks/instance.md
@@ -41,3 +41,4 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
 - `--skip-release-env` - skip generation the release environment file
+- `--release-env-file` - release environment file path
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index caa7dfbd3..1915aacd9 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -37,7 +37,8 @@ def run(["gen" | rest]) do
           strip_uploads: :string,
           anonymize_uploads: :string,
           dedupe_uploads: :string,
-          skip_release_env: :boolean
+          skip_release_env: :boolean,
+          release_env_file: :string
         ],
         aliases: [
           o: :output,
@@ -249,7 +250,15 @@ def run(["gen" | rest]) do
         """)
       else
         shell_info("Generation the environment file:")
-        Mix.Tasks.Pleroma.ReleaseEnv.run(["gen"])
+
+        release_env_args =
+          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
+            ["gen", "--path", path]
+          else
+            _ -> ["gen"]
+          end
+
+        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
       end
 
       shell_info(
diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs
index 8a02710ee..fe69a2def 100644
--- a/test/mix/tasks/pleroma/instance_test.exs
+++ b/test/mix/tasks/pleroma/instance_test.exs
@@ -5,6 +5,8 @@
 defmodule Mix.Tasks.Pleroma.InstanceTest do
   use ExUnit.Case
 
+  @release_env_file "./test/pleroma.test.env"
+
   setup do
     File.mkdir_p!(tmp_path())
 
@@ -16,6 +18,8 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
         File.rm_rf(Path.join(static_dir, "robots.txt"))
       end
 
+      if File.exists?(@release_env_file), do: File.rm_rf(@release_env_file)
+
       Pleroma.Config.put([:instance, :static_dir], static_dir)
     end)
 
@@ -69,7 +73,9 @@ test "running gen" do
         "--dedupe-uploads",
         "n",
         "--anonymize-uploads",
-        "n"
+        "n",
+        "--release-env-file",
+        @release_env_file
       ])
     end
 
@@ -91,6 +97,9 @@ test "running gen" do
     assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]"
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
+    assert File.exists?(@release_env_file)
+
+    assert File.read!(@release_env_file) =~ ~r/^RELEASE_COOKIE=.*/
   end
 
   defp generated_setup_psql do

From 724e4b7f00041667b9fa61fd1b7dd51c8eeaa102 Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Thu, 15 Oct 2020 21:03:48 +0000
Subject: [PATCH 102/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/admin_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 5f3a78a0f..9c477feab 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :skip_thread_containment,
       :pleroma_settings_store,
       :raw_fields,
-      :is_discoverable,
+      :discoverable,
       :actor_type
     ])
     |> Map.merge(%{

From 3b5a7a6b14f4c09d1d371d6fcb49bece84d6c3e1 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 16 Oct 2020 00:32:20 +0200
Subject: [PATCH 103/208] federation_status: New endpoint showing unreachable
 instances

---
 lib/pleroma/instances.ex                      |  1 +
 lib/pleroma/instances/instance.ex             | 11 +++++
 .../controllers/instances_controller.ex       | 19 +++++++++
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/instances_controller_test.exs | 40 +++++++++++++++++++
 5 files changed, 72 insertions(+)
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
 create mode 100644 test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 557e8decf..7315bd7cb 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Instances do
   defdelegate reachable?(url_or_host), to: @adapter
   defdelegate set_reachable(url_or_host), to: @adapter
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+  defdelegate get_consistently_unreachable(), to: @adapter
 
   def set_consistently_unreachable(url_or_host),
     do: set_unreachable(url_or_host, reachability_datetime_threshold())
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index f0f601469..df471a39d 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -119,6 +119,17 @@ def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host)
 
   def set_unreachable(_, _), do: {:error, nil}
 
+  def get_consistently_unreachable do
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    from(i in Instance,
+      where: ^reachability_datetime_threshold > i.unreachable_since,
+      order_by: i.unreachable_since,
+      select: {i.host, i.unreachable_since}
+    )
+    |> Repo.all()
+  end
+
   defp parse_datetime(datetime) when is_binary(datetime) do
     NaiveDateTime.from_iso8601(datetime)
   end
diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
new file mode 100644
index 000000000..bd95cb523
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.InstancesController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Instances
+
+  # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController
+
+  def show(conn, _params) do
+    unreachable =
+      Instances.get_consistently_unreachable()
+      |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end)
+
+    json(conn, %{"unreachable" => unreachable})
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d2d939989..5f9a749e4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -373,6 +373,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:api)
     get("/accounts/:id/scrobbles", ScrobbleController, :index)
+    get("/federation_status", InstancesController, :show)
   end
 
   scope "/api/v1", Pleroma.Web.MastodonAPI do
diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
new file mode 100644
index 000000000..9ce901ce3
--- /dev/null
+++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaApi.InstancesControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Instances
+
+  setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
+
+  setup do
+    constant = "http://consistently-unreachable.name/"
+    eventual = "http://eventually-unreachable.com/path"
+
+    {:ok, %Pleroma.Instances.Instance{unreachable_since: constant_unreachable}} =
+      Instances.set_consistently_unreachable(constant)
+
+    _eventual_unrechable = Instances.set_unreachable(eventual)
+
+    %{constant_unreachable: constant_unreachable, constant: constant}
+  end
+
+  test "GET /api/v1/pleroma/federation_status", %{
+    conn: conn,
+    constant_unreachable: constant_unreachable,
+    constant: constant
+  } do
+    constant_host = URI.parse(constant).host
+
+    assert conn
+           |> put_req_header("content-type", "application/json")
+           |> get("/api/v1/pleroma/federation_status")
+           |> json_response(200) == %{
+             "unreachable" => %{constant_host => to_string(constant_unreachable)}
+           }
+
+    # |> json_response_and_validate_schema(200)
+  end
+end

From aafdc975bdd38f74cdf5d3f8517d41c5dd76c56b Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 16 Oct 2020 01:13:52 +0200
Subject: [PATCH 104/208] federation_status: Add ApiSpec

---
 .../operations/pleroma_instances_operation.ex | 40 +++++++++++++++++++
 .../controllers/instances_controller.ex       |  4 +-
 .../controllers/instances_controller_test.exs |  4 +-
 3 files changed, 44 insertions(+), 4 deletions(-)
 create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex
new file mode 100644
index 000000000..2c455b0df
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex
@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaInstancesOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["PleromaInstances"],
+      summary: "Instances federation status",
+      description: "Information about instances deemed unreachable by the server",
+      operationId: "PleromaInstances.show",
+      responses: %{
+        200 => Operation.response("PleromaInstances", "application/json", pleroma_instances())
+      }
+    }
+  end
+
+  def pleroma_instances do
+    %Schema{
+      type: :object,
+      properties: %{
+        unreachable: %Schema{
+          type: :object,
+          properties: %{hostname: %Schema{type: :string, format: :"date-time"}}
+        }
+      },
+      example: %{
+        "unreachable" => %{"consistently-unreachable.name" => "2020-10-14 22:07:58.216473"}
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
index bd95cb523..c577f1d1e 100644
--- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -7,7 +7,9 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
 
   alias Pleroma.Instances
 
-  # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesOperation
 
   def show(conn, _params) do
     unreachable =
diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
index 9ce901ce3..13491ed9c 100644
--- a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
@@ -31,10 +31,8 @@ test "GET /api/v1/pleroma/federation_status", %{
     assert conn
            |> put_req_header("content-type", "application/json")
            |> get("/api/v1/pleroma/federation_status")
-           |> json_response(200) == %{
+           |> json_response_and_validate_schema(200) == %{
              "unreachable" => %{constant_host => to_string(constant_unreachable)}
            }
-
-    # |> json_response_and_validate_schema(200)
   end
 end

From 1b8fd7e65af980c42b72f584c2a957b12ca5c78b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 16 Oct 2020 17:32:05 +0000
Subject: [PATCH 105/208] Adds feature to permit e.g., local admins and
 community moderators to automatically follow all newly registered accounts

---
 config/config.exs                |  1 +
 config/description.exs           | 11 +++++++++++
 docs/configuration/cheatsheet.md |  1 +
 lib/pleroma/user.ex              | 11 +++++++++++
 test/pleroma/user_test.exs       | 18 ++++++++++++++++++
 5 files changed, 42 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 2c6142360..0f3785d12 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -235,6 +235,7 @@
     "text/bbcode"
   ],
   autofollowed_nicknames: [],
+  autofollowing_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
   max_report_comment_size: 1000,
diff --git a/config/description.exs b/config/description.exs
index 2a1898922..2bbb12540 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -837,6 +837,17 @@
           "rinpatch"
         ]
       },
+      %{
+        key: :autofollowing_nicknames,
+        type: {:list, :string},
+        description:
+          "Set to nicknames of (local) users that automatically follows every newly registered user",
+        suggestions: [
+          "admin",
+          "info",
+          "moderator",
+        ]
+      },
       %{
         key: :attachment_links,
         type: :boolean,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 0b13d7e88..f4b4b6c3c 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index dc41d0001..2a3495103 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -765,6 +765,16 @@ defp autofollow_users(user) do
     follow_all(user, autofollowed_users)
   end
 
+  defp autofollowing_users(user) do
+    candidates = Config.get([:instance, :autofollowing_nicknames])
+
+    User.Query.build(%{nickname: candidates, local: true, deactivated: false})
+    |> Repo.all()
+    |> Enum.each(&follow(&1, user, :follow_accept))
+
+    {:ok, :success}
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset) do
@@ -774,6 +784,7 @@ def register(%Ecto.Changeset{} = changeset) do
 
   def post_register_action(%User{} = user) do
     with {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 7220ce846..9ae52d594 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -388,6 +388,7 @@ test "fetches correct profile for nickname beginning with number" do
     }
 
     setup do: clear_config([:instance, :autofollowed_nicknames])
+    setup do: clear_config([:instance, :autofollowing_nicknames])
     setup do: clear_config([:welcome])
     setup do: clear_config([:instance, :account_activation_required])
 
@@ -408,6 +409,23 @@ test "it autofollows accounts that are set for it" do
       refute User.following?(registered_user, remote_user)
     end
 
+    test "it adds automatic followers for new registered accounts" do
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      Pleroma.Config.put([:instance, :autofollowing_nicknames], [
+        user1.nickname,
+        user2.nickname
+      ])
+
+      cng = User.register_changeset(%User{}, @full_user_data)
+
+      {:ok, registered_user} = User.register(cng)
+
+      assert User.following?(user1, registered_user)
+      assert User.following?(user2, registered_user)
+    end
+
     test "it sends a welcome message if it is set" do
       welcome_user = insert(:user)
       Pleroma.Config.put([:welcome, :direct_message, :enabled], true)

From 405f27b4f8c430ddf7c47b09c5c693f5a17bcbc2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 16 Oct 2020 17:41:44 +0000
Subject: [PATCH 106/208] The suggestions are problematic as they need to be
 real local account names

---
 config/description.exs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 2a1898922..0da1da57d 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -829,13 +829,7 @@
         key: :autofollowed_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that every new user should automatically follow",
-        suggestions: [
-          "lain",
-          "kaniini",
-          "lanodan",
-          "rinpatch"
-        ]
+          "Set to nicknames of (local) users that every new user should automatically follow"
       },
       %{
         key: :attachment_links,

From efd6572ffbf4cce86e28d351d3cbddd0a5334980 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 16 Oct 2020 17:43:44 +0000
Subject: [PATCH 107/208] Remove suggestions

---
 config/description.exs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 2bbb12540..79e1368d0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -841,12 +841,7 @@
         key: :autofollowing_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that automatically follows every newly registered user",
-        suggestions: [
-          "admin",
-          "info",
-          "moderator",
-        ]
+          "Set to nicknames of (local) users that automatically follows every newly registered user"
       },
       %{
         key: :attachment_links,

From d54233760f4c006d89aa80e0ae78cb6910fc74ab Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 17 Oct 2020 13:33:57 +0300
Subject: [PATCH 108/208] [#3053] Post-merge fix.

---
 lib/pleroma/web/feed/user_controller.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index b66fdf275..a5013d2c0 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Feed.UserController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.ActivityPubController

From cb3ee4d543861e65a1de974c3920fdecdcf6a6a7 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Sat, 17 Oct 2020 11:52:33 -0500
Subject: [PATCH 109/208] Fix duplicate Added sections in the changelog

---
 CHANGELOG.md | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36a84b1a8..05e94581a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
 
 ### Changed
 
@@ -23,11 +26,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
 
-### Added
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
-- Pleroma API: Importing the mutes users from CSV files.
-- Experimental websocket-based federation between Pleroma instances.
-
 <details>
   <summary>API Changes</summary>
 

From d16336e7fbf7851ee5f50a17747067f2be741581 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 17 Oct 2020 16:54:05 +0000
Subject: [PATCH 110/208] Document autofollowing_nicknames

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36a84b1a8..fd5b9034d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Setting `:instance, autofollowing_nicknames` to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 
 ### Changed
 

From 524fb0e4c2561f4a2e4c8e58519df991f034c901 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 18 Oct 2020 21:22:21 +0300
Subject: [PATCH 111/208] [#1668] Restricted access to app metrics endpoint by
 default. Added ability to configure IP whitelist for this endpoint. Added
 tests and documentation.

---
 CHANGELOG.md                                  |  2 +
 config/config.exs                             |  7 +-
 docs/API/prometheus.md                        | 26 ++++++-
 lib/pleroma/helpers/inet_helper.ex            | 19 +++++
 lib/pleroma/web/endpoint.ex                   | 40 +++++++++--
 .../web/endpoint/metrics_exporter_test.exs    | 69 +++++++++++++++++++
 6 files changed, 154 insertions(+), 9 deletions(-)
 create mode 100644 lib/pleroma/helpers/inet_helper.ex
 create mode 100644 test/pleroma/web/endpoint/metrics_exporter_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05e94581a..9f6a31f23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,12 +12,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
 - Pleroma API: Importing the mutes users from CSV files.
 - Experimental websocket-based federation between Pleroma instances.
+- App metrics: ability to restrict access to specified IP whitelist.
 
 ### Changed
 
 - **Breaking** Requires `libmagic` (or `file`) to guess file types.
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
 - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
diff --git a/config/config.exs b/config/config.exs
index 2c6142360..a7aae5802 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -636,7 +636,12 @@
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
 
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: false,
+  auth: false,
+  ip_whitelist: [],
+  path: "/api/pleroma/app_metrics",
+  format: :text
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 25,
diff --git a/docs/API/prometheus.md b/docs/API/prometheus.md
index 19c564e3c..a5158d905 100644
--- a/docs/API/prometheus.md
+++ b/docs/API/prometheus.md
@@ -2,15 +2,37 @@
 
 Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
 
+Config example:
+
+```
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: true,
+  auth: {:basic, "myusername", "mypassword"},
+  ip_whitelist: ["127.0.0.1"],
+  path: "/api/pleroma/app_metrics",
+  format: :text
+```
+
+* `enabled` (Pleroma extension) enables the endpoint
+* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
+* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
+* `format` sets the output format (`:text` or `:protobuf`)
+* `path` sets the path to app metrics page 
+
+
 ## `/api/pleroma/app_metrics`
+
 ### Exports Prometheus application metrics
+
 * Method: `GET`
-* Authentication: not required
+* Authentication: not required by default (see configuration options above)
 * Params: none
-* Response: JSON
+* Response: text
 
 ## Grafana
+
 ### Config example
+
 The following is a config example to use with [Grafana](https://grafana.com)
 
 ```
diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex
new file mode 100644
index 000000000..126f82381
--- /dev/null
+++ b/lib/pleroma/helpers/inet_helper.ex
@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.InetHelper do
+  def parse_address(ip) when is_tuple(ip) do
+    {:ok, ip}
+  end
+
+  def parse_address(ip) when is_binary(ip) do
+    ip
+    |> String.to_charlist()
+    |> parse_address()
+  end
+
+  def parse_address(ip) do
+    :inet.parse_address(ip)
+  end
+end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 56562c12f..1a8fdd8b9 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Web.Endpoint do
 
   require Pleroma.Constants
 
+  alias Pleroma.Config
+
   socket("/socket", Pleroma.Web.UserSocket)
 
   plug(Pleroma.Web.Plugs.SetLocalePlug)
@@ -86,19 +88,19 @@ defmodule Pleroma.Web.Endpoint do
   plug(Plug.Parsers,
     parsers: [
       :urlencoded,
-      {:multipart, length: {Pleroma.Config, :get, [[:instance, :upload_limit]]}},
+      {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}},
       :json
     ],
     pass: ["*/*"],
     json_decoder: Jason,
-    length: Pleroma.Config.get([:instance, :upload_limit]),
+    length: Config.get([:instance, :upload_limit]),
     body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []}
   )
 
   plug(Plug.MethodOverride)
   plug(Plug.Head)
 
-  secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag])
+  secure_cookies = Config.get([__MODULE__, :secure_cookie_flag])
 
   cookie_name =
     if secure_cookies,
@@ -106,7 +108,7 @@ defmodule Pleroma.Web.Endpoint do
       else: "pleroma_key"
 
   extra =
-    Pleroma.Config.get([__MODULE__, :extra_cookie_attrs])
+    Config.get([__MODULE__, :extra_cookie_attrs])
     |> Enum.join(";")
 
   # The session will be stored in the cookie and signed,
@@ -116,7 +118,7 @@ defmodule Pleroma.Web.Endpoint do
     Plug.Session,
     store: :cookie,
     key: cookie_name,
-    signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
+    signing_salt: Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
     http_only: true,
     secure: secure_cookies,
     extra: extra
@@ -136,8 +138,34 @@ defmodule MetricsExporter do
     use Prometheus.PlugExporter
   end
 
+  defmodule MetricsExporterCaller do
+    @behaviour Plug
+
+    def init(opts), do: opts
+
+    def call(conn, opts) do
+      prometheus_config = Application.get_env(:prometheus, MetricsExporter, [])
+      ip_whitelist = List.wrap(prometheus_config[:ip_whitelist])
+
+      cond do
+        !prometheus_config[:enabled] ->
+          conn
+
+        ip_whitelist != [] and
+            !Enum.find(ip_whitelist, fn ip ->
+              Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip}
+            end) ->
+          conn
+
+        true ->
+          MetricsExporter.call(conn, opts)
+      end
+    end
+  end
+
   plug(PipelineInstrumenter)
-  plug(MetricsExporter)
+
+  plug(MetricsExporterCaller)
 
   plug(Pleroma.Web.Router)
 
diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs
new file mode 100644
index 000000000..f954cc1e7
--- /dev/null
+++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs
@@ -0,0 +1,69 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Endpoint.MetricsExporterTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Web.Endpoint.MetricsExporter
+
+  defp config do
+    Application.get_env(:prometheus, MetricsExporter)
+  end
+
+  describe "with default config" do
+    test "does NOT expose app metrics", %{conn: conn} do
+      conn
+      |> get(config()[:path])
+      |> json_response(404)
+    end
+  end
+
+  describe "when enabled" do
+    setup do
+      initial_config = config()
+      on_exit(fn -> Application.put_env(:prometheus, MetricsExporter, initial_config) end)
+
+      Application.put_env(
+        :prometheus,
+        MetricsExporter,
+        Keyword.put(initial_config, :enabled, true)
+      )
+    end
+
+    test "serves app metrics", %{conn: conn} do
+      conn = get(conn, config()[:path])
+      assert response = response(conn, 200)
+
+      for metric <- [
+            "http_requests_total",
+            "http_request_duration_microseconds",
+            "phoenix_controller_render_duration",
+            "phoenix_controller_call_duration",
+            "telemetry_scrape_duration",
+            "erlang_vm_memory_atom_bytes_total"
+          ] do
+        assert response =~ ~r/#{metric}/
+      end
+    end
+
+    test "when IP whitelist configured, " <>
+           "serves app metrics only if client IP is whitelisted",
+         %{conn: conn} do
+      Application.put_env(
+        :prometheus,
+        MetricsExporter,
+        Keyword.put(config(), :ip_whitelist, ["127.127.127.127", {1, 1, 1, 1}, '255.255.255.255'])
+      )
+
+      conn
+      |> get(config()[:path])
+      |> json_response(404)
+
+      conn
+      |> Map.put(:remote_ip, {127, 127, 127, 127})
+      |> get(config()[:path])
+      |> response(200)
+    end
+  end
+end

From 98f32cf8204113c6d019653c22e446e558147248 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 19 Oct 2020 15:30:32 +0400
Subject: [PATCH 112/208] Fix tests

---
 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +-
 test/backup_test.exs                                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index e52c77ff2..8e3d081f3 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 23c08b680..078e03621 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -19,7 +19,7 @@ defmodule Pleroma.BackupTest do
   setup do
     clear_config([Pleroma.Upload, :uploader])
     clear_config([Pleroma.Backup, :limit_days])
-    clear_config([Pleroma.Emails.Mailer, :enabled])
+    clear_config([Pleroma.Emails.Mailer, :enabled], true)
   end
 
   test "it requries enabled email" do

From 39fd4d7639a0ab47bc5ae3839b10701fc301a1be Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 19 Oct 2020 15:40:50 +0200
Subject: [PATCH 113/208] Transmogrifier: Downcase incoming Hashtags

Also, set sensitive to true if we have an nsfw hashtag present.
---
 .../web/activity_pub/transmogrifier.ex        |  9 ++-
 .../fixtures/mastodon-post-activity-nsfw.json | 68 +++++++++++++++++++
 .../web/activity_pub/transmogrifier_test.exs  | 10 +++
 3 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 test/fixtures/mastodon-post-activity-nsfw.json

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index d7dd9fe6b..2f51f9e39 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -40,6 +40,7 @@ def fix_object(object, options \\ []) do
     |> fix_in_reply_to(options)
     |> fix_emoji
     |> fix_tag
+    |> set_sensitive
     |> fix_content_map
     |> fix_addressing
     |> fix_summary
@@ -313,7 +314,11 @@ def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
     tags =
       tag
       |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
-      |> Enum.map(fn data -> String.slice(data["name"], 1..-1) end)
+      |> Enum.map(fn %{"name" => name} ->
+        name
+        |> String.slice(1..-1)
+        |> String.downcase()
+      end)
 
     Map.put(object, "tag", tag ++ tags)
   end
@@ -927,7 +932,7 @@ def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end
 
-  def set_sensitive(%{"sensitive" => true} = object) do
+  def set_sensitive(%{"sensitive" => _} = object) do
     object
   end
 
diff --git a/test/fixtures/mastodon-post-activity-nsfw.json b/test/fixtures/mastodon-post-activity-nsfw.json
new file mode 100644
index 000000000..70729a1bd
--- /dev/null
+++ b/test/fixtures/mastodon-post-activity-nsfw.json
@@ -0,0 +1,68 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "Emoji": "toot:Emoji",
+            "Hashtag": "as:Hashtag",
+            "atomUri": "ostatus:atomUri",
+            "conversation": "ostatus:conversation",
+            "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "movedTo": "as:movedTo",
+            "ostatus": "http://ostatus.org#",
+            "toot": "http://joinmastodon.org/ns#"
+        }
+    ],
+    "actor": "http://mastodon.example.org/users/admin",
+    "cc": [
+        "http://mastodon.example.org/users/admin/followers",
+        "http://localtesting.pleroma.lol/users/lain"
+    ],
+    "id": "http://mastodon.example.org/users/admin/statuses/99512778738411822/activity",
+    "nickname": "lain",
+    "object": {
+        "atomUri": "http://mastodon.example.org/users/admin/statuses/99512778738411822",
+        "attachment": [],
+        "attributedTo": "http://mastodon.example.org/users/admin",
+        "cc": [
+            "http://mastodon.example.org/users/admin/followers",
+            "http://localtesting.pleroma.lol/users/lain"
+        ],
+        "content": "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span> #moo</p>",
+        "conversation": "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation",
+        "id": "http://mastodon.example.org/users/admin/statuses/99512778738411822",
+        "inReplyTo": null,
+        "inReplyToAtomUri": null,
+        "published": "2018-02-12T14:08:20Z",
+        "summary": "cw",
+        "tag": [
+            {
+                "href": "http://localtesting.pleroma.lol/users/lain",
+                "name": "@lain@localtesting.pleroma.lol",
+                "type": "Mention"
+            },
+            {
+                "href": "http://mastodon.example.org/tags/nsfw",
+                "name": "#NSFW",
+                "type": "Hashtag"
+            }
+        ],
+        "to": [
+            "https://www.w3.org/ns/activitystreams#Public"
+        ],
+        "type": "Note",
+        "url": "http://mastodon.example.org/@admin/99512778738411822"
+    },
+    "published": "2018-02-12T14:08:20Z",
+    "signature": {
+        "created": "2018-02-12T14:08:20Z",
+        "creator": "http://mastodon.example.org/users/admin#main-key",
+        "signatureValue": "rnNfcopkc6+Ju73P806popcfwrK9wGYHaJVG1/ZvrlEbWVDzaHjkXqj9Q3/xju5l8CSn9tvSgCCtPFqZsFQwn/pFIFUcw7ZWB2xi4bDm3NZ3S4XQ8JRaaX7og5hFxAhWkGhJhAkfxVnOg2hG+w2d/7d7vRVSC1vo5ip4erUaA/PkWusZvPIpxnRWoXaxJsFmVx0gJgjpJkYDyjaXUlp+jmaoseeZ4EPQUWqHLKJ59PRG0mg8j2xAjYH9nQaN14qMRmTGPxY8gfv/CUFcatA+8VJU9KEsJkDAwLVvglydNTLGrxpAJU78a2eaht0foV43XUIZGe3DKiJPgE+UOKGCJw==",
+        "type": "RsaSignature2017"
+    },
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "type": "Create"
+}
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
index 561674f01..8d8f63cc1 100644
--- a/test/pleroma/web/activity_pub/transmogrifier_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -206,6 +206,16 @@ test "it works for incoming notices" do
       assert user.note_count == 1
     end
 
+    test "it works for incoming notices without the sensitive property but an nsfw hashtag" do
+      data = File.read!("test/fixtures/mastodon-post-activity-nsfw.json") |> Poison.decode!()
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+      object_data = Object.normalize(data["object"], false).data
+
+      assert object_data["sensitive"] == true
+    end
+
     test "it works for incoming notices with hashtags" do
       data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!()
 

From e97b254c6bb798d200f381e5adbde2177bcbc0df Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 19 Oct 2020 15:46:24 +0200
Subject: [PATCH 114/208] Transmogrifier: Refactor and unify incoming tag
 handling

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 2f51f9e39..39c8f7e39 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -323,14 +323,12 @@ def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
     Map.put(object, "tag", tag ++ tags)
   end
 
-  def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object) do
-    combined = [tag, String.slice(hashtag, 1..-1)]
-
-    Map.put(object, "tag", combined)
+  def fix_tag(%{"tag" => %{} = tag} = object) do
+    object
+    |> Map.put("tag", [tag])
+    |> fix_tag
   end
 
-  def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag])
-
   def fix_tag(object), do: object
 
   # content map usually only has one language so this will do for now.

From c1976d5b19fbceaecf1f52711fe35e1c7d5312aa Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 19 Oct 2020 18:14:39 +0400
Subject: [PATCH 115/208] Fix credo warnings

---
 test/{ => pleroma}/backup_test.exs                                | 0
 .../web/pleroma_api/controllers/backup_controller_test.exs        | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename test/{ => pleroma}/backup_test.exs (100%)
 rename test/{ => pleroma}/web/pleroma_api/controllers/backup_controller_test.exs (100%)

diff --git a/test/backup_test.exs b/test/pleroma/backup_test.exs
similarity index 100%
rename from test/backup_test.exs
rename to test/pleroma/backup_test.exs
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
similarity index 100%
rename from test/web/pleroma_api/controllers/backup_controller_test.exs
rename to test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs

From ccd1e75e35e9b6d82f4f7f4768b624a8a07bd686 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 20 Oct 2020 02:36:45 +0200
Subject: [PATCH 116/208] CI: Install file-dev in alpine release targets

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e65cae9d8..06cf614c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -217,7 +217,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make cmake
+  - apk add git gcc g++ musl-dev make cmake file-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force

From 3a28aa8814e186eea67df7da00df1636823291f3 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 20 Oct 2020 15:13:20 +0300
Subject: [PATCH 117/208] [#1668] Added :prometheus group config to
 config/description.exs.

---
 config/description.exs | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/config/description.exs b/config/description.exs
index 2a1898922..0fe86ded7 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3722,5 +3722,41 @@
         suggestions: [2]
       }
     ]
+  },
+  %{
+    group: :prometheus,
+    key: Pleroma.Web.Endpoint.MetricsExporter,
+    type: :group,
+    description: "Prometheus app metrics endpoint configuration",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "[Pleroma extension] Enables app metrics endpoint."
+      },
+      %{
+        key: :ip_whitelist,
+        type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
+        description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+      },
+      %{
+        key: :auth,
+        type: [:boolean, :tuple],
+        description: "Enables HTTP Basic Auth for app metrics endpoint.",
+        suggestion: [false, {:basic, "myusername", "mypassword"}]
+      },
+      %{
+        key: :path,
+        type: :string,
+        description: "App metrics endpoint URI path.",
+        suggestions: ["/api/pleroma/app_metrics"]
+      },
+      %{
+        key: :format,
+        type: :atom,
+        description: "App metrics endpoint output format.",
+        suggestions: [:text, :protobuf]
+      }
+    ]
   }
 ]

From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 20 Oct 2020 17:16:58 +0400
Subject: [PATCH 118/208] Rename `Pleroma.Backup` to `Pleroma.User.Backup`

---
 config/config.exs                                      |  2 +-
 config/description.exs                                 |  2 +-
 docs/configuration/cheatsheet.md                       |  2 +-
 lib/pleroma/{ => user}/backup.ex                       |  4 ++--
 .../web/admin_api/controllers/admin_api_controller.ex  |  2 +-
 .../web/pleroma_api/controllers/backup_controller.ex   |  7 ++++---
 lib/pleroma/web/pleroma_api/views/backup_view.ex       |  2 +-
 lib/pleroma/workers/backup_worker.ex                   |  4 ++--
 test/pleroma/{ => user}/backup_test.exs                | 10 +++++-----
 .../controllers/admin_api_controller_test.exs          |  6 +++---
 .../pleroma_api/controllers/backup_controller_test.exs |  2 +-
 11 files changed, 22 insertions(+), 21 deletions(-)
 rename lib/pleroma/{ => user}/backup.ex (98%)
 rename test/pleroma/{ => user}/backup_test.exs (97%)

diff --git a/config/config.exs b/config/config.exs
index 63e386250..c758c818c 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -831,7 +831,7 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
-config :pleroma, Pleroma.Backup,
+config :pleroma, Pleroma.User.Backup,
   purge_after_days: 30,
   limit_days: 7,
   dir: nil
diff --git a/config/description.exs b/config/description.exs
index 88f2a6133..9f23b6d3d 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3731,7 +3731,7 @@
   },
   %{
     group: :pleroma,
-    key: Pleroma.Backup,
+    key: Pleroma.User.Backup,
     type: :group,
     description: "Account Backup",
     children: [
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index aafc43f3d..b40a2aebf 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1077,7 +1077,7 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
-## Account Backup
+## Pleroma.User.Backup
 
 !!! note
     Requires enabled email
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/user/backup.ex
similarity index 98%
rename from lib/pleroma/backup.ex
rename to lib/pleroma/user/backup.ex
index 629e879a7..a9041fd94 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/user/backup.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Backup do
+defmodule Pleroma.User.Backup do
   use Ecto.Schema
 
   import Ecto.Changeset
@@ -65,7 +65,7 @@ defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
   defp validate_limit(user, nil) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
-        days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
         diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
 
         if diff > days do
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index a4f0d7d34..0a27c5861 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -685,7 +685,7 @@ def stats(conn, params) do
 
   def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_by_nickname(nickname),
-         {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
+         {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
       ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
 
       json(conn, "")
diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index 8e3d081f3..bd7b36880 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.User.Backup
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
@@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
 
   def index(%{assigns: %{user: user}} = conn, _params) do
-    backups = Pleroma.Backup.list(user)
+    backups = Backup.list(user)
     render(conn, "index.json", backups: backups)
   end
 
   def create(%{assigns: %{user: user}} = conn, _params) do
-    with {:ok, _} <- Pleroma.Backup.create(user) do
-      backups = Pleroma.Backup.list(user)
+    with {:ok, _} <- Backup.create(user) do
+      backups = Backup.list(user)
       render(conn, "index.json", backups: backups)
     end
   end
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
index bf40a001e..af75876aa 100644
--- a/lib/pleroma/web/pleroma_api/views/backup_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Web.CommonAPI.Utils
 
   def render("show.json", %{backup: %Backup{} = backup}) do
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 65754b6a2..5b4985983 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do
   use Oban.Worker, queue: :backup, max_attempts: 1
 
   alias Oban.Job
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
 
   def process(backup, admin_user_id \\ nil) do
     %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
@@ -15,7 +15,7 @@ def process(backup, admin_user_id \\ nil) do
   end
 
   def schedule_deletion(backup) do
-    days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
+    days = Pleroma.Config.get([Backup, :purge_after_days])
     time = 60 * 60 * 24 * days
     scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
 
diff --git a/test/pleroma/backup_test.exs b/test/pleroma/user/backup_test.exs
similarity index 97%
rename from test/pleroma/backup_test.exs
rename to test/pleroma/user/backup_test.exs
index 078e03621..5ad587833 100644
--- a/test/pleroma/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.BackupTest do
+defmodule Pleroma.User.BackupTest do
   use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
 
@@ -10,7 +10,7 @@ defmodule Pleroma.BackupTest do
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.CommonAPI
@@ -18,7 +18,7 @@ defmodule Pleroma.BackupTest do
 
   setup do
     clear_config([Pleroma.Upload, :uploader])
-    clear_config([Pleroma.Backup, :limit_days])
+    clear_config([Backup, :limit_days])
     clear_config([Pleroma.Emails.Mailer, :enabled], true)
   end
 
@@ -44,7 +44,7 @@ test "it creates a backup record and an Oban job" do
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
-    limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
+    limit_days = Pleroma.Config.get([Backup, :limit_days])
     assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
     backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
@@ -76,7 +76,7 @@ test "it process a backup record" do
   end
 
   test "it removes outdated backups after creating a fresh one" do
-    Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
+    Pleroma.Config.put([Backup, :limit_days], -1)
     Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     user = insert(:user)
 
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 34d48c2c1..5efe8ef71 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2038,7 +2038,7 @@ test "it creates a backup", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert [backup] = Repo.all(Pleroma.Backup)
+      assert [backup] = Repo.all(Pleroma.User.Backup)
 
       ObanHelpers.perform_all()
 
@@ -2079,7 +2079,7 @@ test "it doesn't limit admins", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert [_backup] = Repo.all(Pleroma.Backup)
+      assert [_backup] = Repo.all(Pleroma.User.Backup)
 
       assert "" ==
                conn
@@ -2088,7 +2088,7 @@ test "it doesn't limit admins", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert Repo.aggregate(Pleroma.Backup, :count) == 2
+      assert Repo.aggregate(Pleroma.User.Backup, :count) == 2
     end
   end
 end
diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
index b2ac74c7d..f1941f6dd 100644
--- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Web.PleromaAPI.BackupView
 
   setup do

From 50d428088017e0383d8b35d4ab1b831f40646ab0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 20 Oct 2020 16:18:24 +0300
Subject: [PATCH 119/208] [#1668] Formatting fix.

---
 config/description.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/description.exs b/config/description.exs
index 0fe86ded7..11755d757 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3737,7 +3737,8 @@
       %{
         key: :ip_whitelist,
         type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
-        description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+        description:
+          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
       },
       %{
         key: :auth,

From 034ac43f3a91178694d3c621c52ce68207ec4f69 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 20 Oct 2020 17:47:04 +0400
Subject: [PATCH 120/208] Fix credo warnings

---
 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +-
 test/pleroma/user/backup_test.exs                            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index bd7b36880..dd0a2e22f 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.User.Backup
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
index 5ad587833..513798911 100644
--- a/test/pleroma/user/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -10,9 +10,9 @@ defmodule Pleroma.User.BackupTest do
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
-  alias Pleroma.User.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User.Backup
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
 

From 54d99cbb729bbce96cd833c820e2eda8bb745652 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 20 Oct 2020 22:10:20 +0200
Subject: [PATCH 121/208] CI: Install libmagic-dev in debian release targets

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 06cf614c2..fd0c5c8d4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -198,7 +198,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
-  - apt-get update && apt-get install -y cmake
+  - apt-get update && apt-get install -y cmake libmagic-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force

From b18b93bbed925e5058d941662e93f0a46a27c325 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 21 Oct 2020 21:42:21 +0400
Subject: [PATCH 122/208] Fix formatting and typos in "Managing frontends"
 guide

---
 docs/administration/CLI_tasks/frontend.md | 93 +++++++++++++++--------
 1 file changed, 60 insertions(+), 33 deletions(-)

diff --git a/docs/administration/CLI_tasks/frontend.md b/docs/administration/CLI_tasks/frontend.md
index 7d1c1e937..d4a48cb56 100644
--- a/docs/administration/CLI_tasks/frontend.md
+++ b/docs/administration/CLI_tasks/frontend.md
@@ -1,12 +1,23 @@
 # Managing frontends
 
-`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
 
 Frontend can be installed either from local zip file, or automatically downloaded from the web.
 
-You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently, known `<frontend>` values are:
 
-Currently known `<frontend>` values are:
 - [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
 - [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
 - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
@@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below.
 
 For a frontend configured under the `available` key, it's enough to install it by name.
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install pleroma
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma
+    ```
 
-This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+=== "From Source"
 
-You can override any of the details. To install a pleroma build from a different url, you could do this:
+    ```sh
+    mix pleroma.frontend install pleroma
+    ```
 
-```sh tab="OPT"
-./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
-```
+This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
-```
+You can override any of the details. To install a pleroma build from a different URL, you could do this:
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+    ```
 
 Similarly, you can also install from a local zip file.
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+    ```
 
-The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+=== "From Source"
 
-Careful: This folder will be completely replaced on installation
+    ```sh
+    mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+    ```
+
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
+
+Careful: This folder will be completely replaced on installation.
 
 ## Example installation for an unknown frontend
 
-The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+The installation process is the same, but you will have to give all the needed options on the command line. For example:
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+    ```
 
-If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+    ```
+
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 

From 2ca98f2d94e2976ae35998aecff27809d4b066cf Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Wed, 21 Oct 2020 19:40:37 +0000
Subject: [PATCH 123/208] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/pleroma_api/controllers/instances_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
index c577f1d1e..9e97480df 100644
--- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
   def show(conn, _params) do
     unreachable =
       Instances.get_consistently_unreachable()
-      |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end)
+      |> Map.new(fn {host, date} -> {host, to_string(date)} end)
 
     json(conn, %{"unreachable" => unreachable})
   end

From 9ef46ce4103f338d5bb75278013a63d3ae418d7e Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 21 Sep 2020 09:33:51 +0300
Subject: [PATCH 124/208] added 'unconfirmed' filter to admin/users

---
 lib/pleroma/user/query.ex                     |   5 +
 .../controllers/admin_api_controller.ex       |  55 +--
 .../controllers/admin_api_controller_test.exs | 399 +++++-------------
 test/pleroma/web/admin_api/search_test.exs    |  11 +
 4 files changed, 131 insertions(+), 339 deletions(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 2440bf890..2933e7fb4 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -43,6 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
+            need_confirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -156,6 +157,10 @@ defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
+  defp compose_query({:need_confirmed, _}, query) do
+    where(query, [u], u.confirmation_pending)
+  end
+
   defp compose_query({:followers, %User{id: id}}, query) do
     query
     |> where([u], u.id != ^id)
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index bdd3e195d..acfbeb0c8 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -5,7 +5,8 @@
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
 
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+  import Pleroma.Web.ControllerHelper,
+    only: [json_response: 3, fetch_integer_param: 3]
 
   alias Pleroma.Config
   alias Pleroma.MFA
@@ -100,12 +101,9 @@ def user_delete(conn, %{"nickname" => nickname}) do
   end
 
   def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users =
-      nicknames
-      |> Enum.map(&User.get_cached_by_nickname/1)
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
 
-    users
-    |> Enum.each(fn user ->
+    Enum.each(users, fn user ->
       {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
       Pipeline.common_pipeline(delete_data, local: true)
     end)
@@ -367,16 +365,18 @@ def list_users(conn, params) do
     {page, page_size} = page_params(params)
     filters = maybe_parse_filters(params["filters"])
 
-    search_params = %{
-      query: params["query"],
-      page: page,
-      page_size: page_size,
-      tags: params["tags"],
-      name: params["name"],
-      email: params["email"]
-    }
+    search_params =
+      %{
+        query: params["query"],
+        page: page,
+        page_size: page_size,
+        tags: params["tags"],
+        name: params["name"],
+        email: params["email"]
+      }
+      |> Map.merge(filters)
 
-    with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do
+    with {:ok, users, count} <- Search.user(search_params) do
       json(
         conn,
         AccountView.render("index.json",
@@ -388,7 +388,7 @@ def list_users(conn, params) do
     end
   end
 
-  @filters ~w(local external active deactivated need_approval is_admin is_moderator)
+  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
 
   @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
   defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
@@ -682,24 +682,9 @@ def stats(conn, params) do
   end
 
   defp page_params(params) do
-    {get_page(params["page"]), get_page_size(params["page_size"])}
-  end
-
-  defp get_page(page_string) when is_nil(page_string), do: 1
-
-  defp get_page(page_string) do
-    case Integer.parse(page_string) do
-      {page, _} -> page
-      :error -> 1
-    end
-  end
-
-  defp get_page_size(page_size_string) when is_nil(page_size_string), do: @users_page_size
-
-  defp get_page_size(page_size_string) do
-    case Integer.parse(page_size_string) do
-      {page_size, _} -> page_size
-      :error -> @users_page_size
-    end
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
   end
 end
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index cba6b43d3..686b53a80 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -369,23 +369,7 @@ test "Show", %{conn: conn} do
 
       conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
 
-      expected = %{
-        "deactivated" => false,
-        "id" => to_string(user.id),
-        "local" => true,
-        "nickname" => user.nickname,
-        "roles" => %{"admin" => false, "moderator" => false},
-        "tags" => [],
-        "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-        "display_name" => HTML.strip_tags(user.name || user.nickname),
-        "confirmation_pending" => false,
-        "approval_pending" => false,
-        "url" => user.ap_id,
-        "registration_reason" => nil,
-        "actor_type" => "Person"
-      }
-
-      assert expected == json_response(conn, 200)
+      assert user_response(user) == json_response(conn, 200)
     end
 
     test "when the user doesn't exist", %{conn: conn} do
@@ -652,51 +636,20 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
 
       users =
         [
-          %{
-            "deactivated" => admin.deactivated,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => false,
-            "tags" => ["foo", "bar"],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => user2.deactivated,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user2.ap_id,
-            "registration_reason" => "I'm a chill dude",
-            "actor_type" => "Person"
-          }
+          user_response(
+            admin,
+            %{"roles" => %{"admin" => true, "moderator" => false}}
+          ),
+          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
+          user_response(
+            user2,
+            %{
+              "local" => true,
+              "approval_pending" => true,
+              "registration_reason" => "I'm a chill dude",
+              "actor_type" => "Person"
+            }
+          )
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -757,23 +710,7 @@ test "regular search", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user, %{"local" => true})]
              }
     end
 
@@ -786,23 +723,7 @@ test "search by domain", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -815,23 +736,7 @@ test "search by full nickname", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -844,23 +749,7 @@ test "search by display name", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -873,23 +762,7 @@ test "search by email", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -902,23 +775,7 @@ test "regular search with page size", %{conn: conn} do
       assert json_response(conn1, 200) == %{
                "count" => 2,
                "page_size" => 1,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
 
       conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
@@ -926,23 +783,7 @@ test "regular search with page size", %{conn: conn} do
       assert json_response(conn2, 200) == %{
                "count" => 2,
                "page_size" => 1,
-               "users" => [
-                 %{
-                   "deactivated" => user2.deactivated,
-                   "id" => user2.id,
-                   "nickname" => user2.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user2.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user2)]
              }
     end
 
@@ -962,23 +803,7 @@ test "only local users" do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -992,51 +817,14 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
 
       users =
         [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => admin.deactivated,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
+          user_response(user),
+          user_response(admin, %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(old_admin, %{
             "deactivated" => false,
-            "id" => old_admin.id,
-            "local" => true,
-            "nickname" => old_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "tags" => [],
-            "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => old_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -1047,6 +835,30 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
              }
     end
 
+    test "only unconfirmed users", %{conn: conn} do
+      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
+      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+      insert(:user, confirmation_pending: false)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> json_response(200)
+
+      users =
+        Enum.map([old_user, sad_user], fn user ->
+          user_response(user, %{
+            "confirmation_pending" => true,
+            "approval_pending" => false
+          })
+        end)
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
+    end
+
     test "only unapproved users", %{conn: conn} do
       user =
         insert(:user,
@@ -1175,21 +987,22 @@ test "load users with tags list", %{conn: conn} do
 
       users =
         [
-          %{
-            "deactivated" => false,
-            "id" => user1.id,
-            "nickname" => user1.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user1.local,
-            "tags" => ["first"],
-            "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user1.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
+          user_response(
+            user1,
+            %{
+              "deactivated" => false,
+              "roles" => %{"admin" => false, "moderator" => false},
+              "local" => user1.local,
+              "tags" => ["first"],
+              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
+              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
+              "confirmation_pending" => false,
+              "approval_pending" => false,
+              "url" => user1.ap_id,
+              "registration_reason" => nil,
+              "actor_type" => "Person"
+            }
+          ),
           %{
             "deactivated" => false,
             "id" => user2.id,
@@ -1253,23 +1066,7 @@ test "it works with multiple filters" do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => user.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -1282,21 +1079,7 @@ test "it omits relay user", %{admin: admin, conn: conn} do
                "count" => 1,
                "page_size" => 50,
                "users" => [
-                 %{
-                   "deactivated" => admin.deactivated,
-                   "id" => admin.id,
-                   "nickname" => admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => admin.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
+                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
                ]
              }
     end
@@ -1368,21 +1151,10 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admi
     conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
 
     assert json_response(conn, 200) ==
-             %{
-               "deactivated" => !user.deactivated,
-               "id" => user.id,
-               "nickname" => user.nickname,
-               "roles" => %{"admin" => false, "moderator" => false},
-               "local" => true,
-               "tags" => [],
-               "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-               "display_name" => HTML.strip_tags(user.name || user.nickname),
-               "confirmation_pending" => false,
-               "approval_pending" => false,
-               "url" => user.ap_id,
-               "registration_reason" => nil,
-               "actor_type" => "Person"
-             }
+             user_response(
+               user,
+               %{"deactivated" => !user.deactivated}
+             )
 
     log_entry = Repo.one(ModerationLog)
 
@@ -2024,6 +1796,25 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
+
+  defp user_response(user, attrs \\ %{}) do
+    %{
+      "deactivated" => user.deactivated,
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "roles" => %{"admin" => false, "moderator" => false},
+      "local" => user.local,
+      "tags" => [],
+      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+      "display_name" => HTML.strip_tags(user.name || user.nickname),
+      "confirmation_pending" => false,
+      "approval_pending" => false,
+      "url" => user.ap_id,
+      "registration_reason" => nil,
+      "actor_type" => "Person"
+    }
+    |> Map.merge(attrs)
+  end
 end
 
 # Needed for testing
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index ceec64f1e..27ca396e6 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -178,6 +178,17 @@ test "it returns unapproved user" do
       assert count == 1
     end
 
+    test "it returns unconfirmed user" do
+      unconfirmed = insert(:user, confirmation_pending: true)
+      insert(:user)
+      insert(:user)
+
+      {:ok, _results, total} = Search.user()
+      {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true})
+      assert total == 3
+      assert count == 1
+    end
+
     test "it returns non-discoverable users" do
       insert(:user)
       insert(:user, is_discoverable: false)

From cf4f3937946eb0232be8bf8ddba08d365b4e17ee Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 21 Sep 2020 15:01:03 +0300
Subject: [PATCH 125/208] added AdminApi.UserController

---
 .../controllers/admin_api_controller.ex       | 242 +----
 .../admin_api/controllers/user_controller.ex  | 280 +++++
 lib/pleroma/web/router.ex                     |  22 +-
 .../controllers/admin_api_controller_test.exs | 837 ---------------
 .../controllers/user_controller_test.exs      | 983 ++++++++++++++++++
 5 files changed, 1275 insertions(+), 1089 deletions(-)
 create mode 100644 lib/pleroma/web/admin_api/controllers/user_controller.ex
 create mode 100644 test/web/admin_api/controllers/user_controller_test.exs

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index acfbeb0c8..df5817cfa 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -14,12 +14,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Builder
-  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.AdminAPI.ModerationLogView
-  alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Router
@@ -29,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
+    when action in [:right_get, :show_user_credentials]
   )
 
   plug(
@@ -38,12 +35,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [
            :get_password_reset,
            :force_password_reset,
-           :user_delete,
-           :users_create,
-           :user_toggle_activation,
-           :user_activate,
-           :user_deactivate,
-           :user_approve,
            :tag_users,
            :untag_users,
            :right_add,
@@ -55,12 +46,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          ]
   )
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:follows"], admin: true}
-    when action in [:user_follow, :user_unfollow]
-  )
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:statuses"], admin: true}
@@ -96,129 +81,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   action_fallback(AdminAPI.FallbackController)
 
-  def user_delete(conn, %{"nickname" => nickname}) do
-    user_delete(conn, %{"nicknames" => [nickname]})
-  end
-
-  def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-
-    Enum.each(users, fn user ->
-      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
-      Pipeline.common_pipeline(delete_data, local: true)
-    end)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "delete"
-    })
-
-    json(conn, nicknames)
-  end
-
-  def user_follow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.follow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "follow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def user_unfollow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.unfollow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "unfollow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
-    changesets =
-      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
-        user_data = %{
-          nickname: nickname,
-          name: nickname,
-          email: email,
-          password: password,
-          password_confirmation: password,
-          bio: "."
-        }
-
-        User.register_changeset(%User{}, user_data, need_confirmation: false)
-      end)
-      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
-        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
-      end)
-
-    case Pleroma.Repo.transaction(changesets) do
-      {:ok, users} ->
-        res =
-          users
-          |> Map.values()
-          |> Enum.map(fn user ->
-            {:ok, user} = User.post_register_action(user)
-
-            user
-          end)
-          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
-
-        ModerationLog.insert_log(%{
-          actor: admin,
-          subjects: Map.values(users),
-          action: "create"
-        })
-
-        json(conn, res)
-
-      {:error, id, changeset, _} ->
-        res =
-          Enum.map(changesets.operations, fn
-            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
-              AccountView.render("create-error.json", %{changeset: changeset})
-
-            {_, {:changeset, current_changeset, _}} ->
-              AccountView.render("create-error.json", %{changeset: current_changeset})
-          end)
-
-        conn
-        |> put_status(:conflict)
-        |> json(res)
-    end
-  end
-
-  def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
-      conn
-      |> put_view(AccountView)
-      |> render("show.json", %{user: user})
-    else
-      _ -> {:error, :not_found}
-    end
-  end
-
   def list_instance_statuses(conn, %{"instance" => instance} = params) do
     with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
@@ -272,69 +134,6 @@ def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}
     end
   end
 
-  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    user = User.get_cached_by_nickname(nickname)
-
-    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
-
-    action = if user.deactivated, do: "activate", else: "deactivate"
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: [user],
-      action: action
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("show.json", %{user: updated_user})
-  end
-
-  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, false)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "activate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, true)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "deactivate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.approve(users)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "approve"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: updated_users})
-  end
-
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
     with {:ok, _} <- User.tag(nicknames, tags) do
       ModerationLog.insert_log(%{
@@ -361,45 +160,6 @@ def untag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "
     end
   end
 
-  def list_users(conn, params) do
-    {page, page_size} = page_params(params)
-    filters = maybe_parse_filters(params["filters"])
-
-    search_params =
-      %{
-        query: params["query"],
-        page: page,
-        page_size: page_size,
-        tags: params["tags"],
-        name: params["name"],
-        email: params["email"]
-      }
-      |> Map.merge(filters)
-
-    with {:ok, users, count} <- Search.user(search_params) do
-      json(
-        conn,
-        AccountView.render("index.json",
-          users: users,
-          count: count,
-          page_size: page_size
-        )
-      )
-    end
-  end
-
-  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
-
-  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
-  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
-
-  defp maybe_parse_filters(filters) do
-    filters
-    |> String.split(",")
-    |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Map.new(&{String.to_existing_atom(&1), true})
-  end
-
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames
diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
new file mode 100644
index 000000000..5e049aa0f
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -0,0 +1,280 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.UserController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper,
+    only: [fetch_integer_param: 3]
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Search
+
+  @users_page_size 50
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:accounts"], admin: true}
+    when action in [:list, :show]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:accounts"], admin: true}
+    when action in [
+           :delete,
+           :create,
+           :toggle_activation,
+           :activate,
+           :deactivate,
+           :approve
+         ]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:follows"], admin: true}
+    when action in [:follow, :unfollow]
+  )
+
+  action_fallback(AdminAPI.FallbackController)
+
+  def delete(conn, %{"nickname" => nickname}) do
+    delete(conn, %{"nicknames" => [nickname]})
+  end
+
+  def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+
+    Enum.each(users, fn user ->
+      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
+      Pipeline.common_pipeline(delete_data, local: true)
+    end)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "delete"
+    })
+
+    json(conn, nicknames)
+  end
+
+  def follow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.follow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "follow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def unfollow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.unfollow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "unfollow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
+    changesets =
+      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
+        user_data = %{
+          nickname: nickname,
+          name: nickname,
+          email: email,
+          password: password,
+          password_confirmation: password,
+          bio: "."
+        }
+
+        User.register_changeset(%User{}, user_data, need_confirmation: false)
+      end)
+      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
+        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
+      end)
+
+    case Pleroma.Repo.transaction(changesets) do
+      {:ok, users} ->
+        res =
+          users
+          |> Map.values()
+          |> Enum.map(fn user ->
+            {:ok, user} = User.post_register_action(user)
+
+            user
+          end)
+          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
+
+        ModerationLog.insert_log(%{
+          actor: admin,
+          subjects: Map.values(users),
+          action: "create"
+        })
+
+        json(conn, res)
+
+      {:error, id, changeset, _} ->
+        res =
+          Enum.map(changesets.operations, fn
+            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
+              AccountView.render("create-error.json", %{changeset: changeset})
+
+            {_, {:changeset, current_changeset, _}} ->
+              AccountView.render("create-error.json", %{changeset: current_changeset})
+          end)
+
+        conn
+        |> put_status(:conflict)
+        |> json(res)
+    end
+  end
+
+  def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      conn
+      |> put_view(AccountView)
+      |> render("show.json", %{user: user})
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
+
+    action = if user.deactivated, do: "activate", else: "deactivate"
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: [user],
+      action: action
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("show.json", %{user: updated_user})
+  end
+
+  def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "deactivate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.approve(users)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "approve"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: updated_users})
+  end
+
+  def list(conn, params) do
+    {page, page_size} = page_params(params)
+    filters = maybe_parse_filters(params["filters"])
+
+    search_params =
+      %{
+        query: params["query"],
+        page: page,
+        page_size: page_size,
+        tags: params["tags"],
+        name: params["name"],
+        email: params["email"]
+      }
+      |> Map.merge(filters)
+
+    with {:ok, users, count} <- Search.user(search_params) do
+      json(
+        conn,
+        AccountView.render("index.json",
+          users: users,
+          count: count,
+          page_size: page_size
+        )
+      )
+    end
+  end
+
+  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
+
+  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
+  defp maybe_parse_filters(filters) do
+    filters
+    |> String.split(",")
+    |> Enum.filter(&Enum.member?(@filters, &1))
+    |> Map.new(&{String.to_existing_atom(&1), true})
+  end
+
+  defp page_params(params) do
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d2d939989..3a9605778 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -129,16 +129,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
 
-    post("/users/follow", AdminAPIController, :user_follow)
-    post("/users/unfollow", AdminAPIController, :user_unfollow)
-
     put("/users/disable_mfa", AdminAPIController, :disable_mfa)
-    delete("/users", AdminAPIController, :user_delete)
-    post("/users", AdminAPIController, :users_create)
-    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
-    patch("/users/activate", AdminAPIController, :user_activate)
-    patch("/users/deactivate", AdminAPIController, :user_deactivate)
-    patch("/users/approve", AdminAPIController, :user_approve)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
 
@@ -161,6 +152,15 @@ defmodule Pleroma.Web.Router do
       :right_delete_multiple
     )
 
+    post("/users/follow", UserController, :follow)
+    post("/users/unfollow", UserController, :unfollow)
+    delete("/users", UserController, :delete)
+    post("/users", UserController, :create)
+    patch("/users/:nickname/toggle_activation", UserController, :toggle_activation)
+    patch("/users/activate", UserController, :activate)
+    patch("/users/deactivate", UserController, :deactivate)
+    patch("/users/approve", UserController, :approve)
+
     get("/relay", RelayController, :index)
     post("/relay", RelayController, :follow)
     delete("/relay", RelayController, :unfollow)
@@ -175,8 +175,8 @@ defmodule Pleroma.Web.Router do
     get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials)
     patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials)
 
-    get("/users", AdminAPIController, :list_users)
-    get("/users/:nickname", AdminAPIController, :user_show)
+    get("/users", UserController, :list)
+    get("/users/:nickname", UserController, :show)
     get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses)
     get("/users/:nickname/chats", AdminAPIController, :list_user_chats)
 
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 686b53a80..34b26dddf 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -7,22 +7,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   use Oban.Testing, repo: Pleroma.Repo
 
   import ExUnit.CaptureLog
-  import Mock
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
   alias Pleroma.Activity
   alias Pleroma.Config
-  alias Pleroma.HTML
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
-  alias Pleroma.Web
-  alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.MediaProxy
 
   setup_all do
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -153,284 +148,6 @@ test "GET /api/pleroma/admin/users/:nickname requires " <>
     end
   end
 
-  describe "DELETE /api/pleroma/admin/users" do
-    test "single user", %{admin: admin, conn: conn} do
-      clear_config([:instance, :federating], true)
-
-      user =
-        insert(:user,
-          avatar: %{"url" => [%{"href" => "https://someurl"}]},
-          banner: %{"url" => [%{"href" => "https://somebanner"}]},
-          bio: "Hello world!",
-          name: "A guy"
-        )
-
-      # Create some activities to check they got deleted later
-      follower = insert(:user)
-      {:ok, _} = CommonAPI.post(user, %{status: "test"})
-      {:ok, _, _, _} = CommonAPI.follow(user, follower)
-      {:ok, _, _, _} = CommonAPI.follow(follower, user)
-      user = Repo.get(User, user.id)
-      assert user.note_count == 1
-      assert user.follower_count == 1
-      assert user.following_count == 1
-      refute user.deactivated
-
-      with_mock Pleroma.Web.Federator,
-        publish: fn _ -> nil end,
-        perform: fn _, _ -> nil end do
-        conn =
-          conn
-          |> put_req_header("accept", "application/json")
-          |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}")
-
-        ObanHelpers.perform_all()
-
-        assert User.get_by_nickname(user.nickname).deactivated
-
-        log_entry = Repo.one(ModerationLog)
-
-        assert ModerationLog.get_log_entry_message(log_entry) ==
-                 "@#{admin.nickname} deleted users: @#{user.nickname}"
-
-        assert json_response(conn, 200) == [user.nickname]
-
-        user = Repo.get(User, user.id)
-        assert user.deactivated
-
-        assert user.avatar == %{}
-        assert user.banner == %{}
-        assert user.note_count == 0
-        assert user.follower_count == 0
-        assert user.following_count == 0
-        assert user.bio == ""
-        assert user.name == nil
-
-        assert called(Pleroma.Web.Federator.publish(:_))
-      end
-    end
-
-    test "multiple users", %{admin: admin, conn: conn} do
-      user_one = insert(:user)
-      user_two = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> delete("/api/pleroma/admin/users", %{
-          nicknames: [user_one.nickname, user_two.nickname]
-        })
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}"
-
-      response = json_response(conn, 200)
-      assert response -- [user_one.nickname, user_two.nickname] == []
-    end
-  end
-
-  describe "/api/pleroma/admin/users" do
-    test "Create", %{conn: conn} do
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "lain",
-              "email" => "lain@example.org",
-              "password" => "test"
-            },
-            %{
-              "nickname" => "lain2",
-              "email" => "lain2@example.org",
-              "password" => "test"
-            }
-          ]
-        })
-
-      response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type"))
-      assert response == ["success", "success"]
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == []
-    end
-
-    test "Cannot create user with existing email", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "lain",
-              "email" => user.email,
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => user.email,
-                   "nickname" => "lain"
-                 },
-                 "error" => "email has already been taken",
-                 "type" => "error"
-               }
-             ]
-    end
-
-    test "Cannot create user with existing nickname", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => user.nickname,
-              "email" => "someuser@plerama.social",
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => "someuser@plerama.social",
-                   "nickname" => user.nickname
-                 },
-                 "error" => "nickname has already been taken",
-                 "type" => "error"
-               }
-             ]
-    end
-
-    test "Multiple user creation works in transaction", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "newuser",
-              "email" => "newuser@pleroma.social",
-              "password" => "test"
-            },
-            %{
-              "nickname" => "lain",
-              "email" => user.email,
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => user.email,
-                   "nickname" => "lain"
-                 },
-                 "error" => "email has already been taken",
-                 "type" => "error"
-               },
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => "newuser@pleroma.social",
-                   "nickname" => "newuser"
-                 },
-                 "error" => "",
-                 "type" => "error"
-               }
-             ]
-
-      assert User.get_by_nickname("newuser") === nil
-    end
-  end
-
-  describe "/api/pleroma/admin/users/:nickname" do
-    test "Show", %{conn: conn} do
-      user = insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
-
-      assert user_response(user) == json_response(conn, 200)
-    end
-
-    test "when the user doesn't exist", %{conn: conn} do
-      user = build(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
-
-      assert %{"error" => "Not found"} == json_response(conn, 404)
-    end
-  end
-
-  describe "/api/pleroma/admin/users/follow" do
-    test "allows to force-follow another user", %{admin: admin, conn: conn} do
-      user = insert(:user)
-      follower = insert(:user)
-
-      conn
-      |> put_req_header("accept", "application/json")
-      |> post("/api/pleroma/admin/users/follow", %{
-        "follower" => follower.nickname,
-        "followed" => user.nickname
-      })
-
-      user = User.get_cached_by_id(user.id)
-      follower = User.get_cached_by_id(follower.id)
-
-      assert User.following?(follower, user)
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}"
-    end
-  end
-
-  describe "/api/pleroma/admin/users/unfollow" do
-    test "allows to force-unfollow another user", %{admin: admin, conn: conn} do
-      user = insert(:user)
-      follower = insert(:user)
-
-      User.follow(follower, user)
-
-      conn
-      |> put_req_header("accept", "application/json")
-      |> post("/api/pleroma/admin/users/unfollow", %{
-        "follower" => follower.nickname,
-        "followed" => user.nickname
-      })
-
-      user = User.get_cached_by_id(user.id)
-      follower = User.get_cached_by_id(follower.id)
-
-      refute User.following?(follower, user)
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}"
-    end
-  end
-
   describe "PUT /api/pleroma/admin/users/tag" do
     setup %{conn: conn} do
       user1 = insert(:user, %{tags: ["x"]})
@@ -627,541 +344,6 @@ test "/api/pleroma/admin/users/:nickname/password_reset", %{conn: conn} do
     assert Regex.match?(~r/(http:\/\/|https:\/\/)/, resp["link"])
   end
 
-  describe "GET /api/pleroma/admin/users" do
-    test "renders users array for the first page", %{conn: conn, admin: admin} do
-      user = insert(:user, local: false, tags: ["foo", "bar"])
-      user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude")
-
-      conn = get(conn, "/api/pleroma/admin/users?page=1")
-
-      users =
-        [
-          user_response(
-            admin,
-            %{"roles" => %{"admin" => true, "moderator" => false}}
-          ),
-          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
-          user_response(
-            user2,
-            %{
-              "local" => true,
-              "approval_pending" => true,
-              "registration_reason" => "I'm a chill dude",
-              "actor_type" => "Person"
-            }
-          )
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 3,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "pagination works correctly with service users", %{conn: conn} do
-      service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido")
-
-      insert_list(25, :user)
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users1} =
-               conn
-               |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users1) == 10
-      assert service1 not in users1
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users2} =
-               conn
-               |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users2) == 10
-      assert service1 not in users2
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users3} =
-               conn
-               |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users3) == 6
-      assert service1 not in users3
-    end
-
-    test "renders empty array for the second page", %{conn: conn} do
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?page=2")
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => []
-             }
-    end
-
-    test "regular search", %{conn: conn} do
-      user = insert(:user, nickname: "bob")
-
-      conn = get(conn, "/api/pleroma/admin/users?query=bo")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user, %{"local" => true})]
-             }
-    end
-
-    test "search by domain", %{conn: conn} do
-      user = insert(:user, nickname: "nickname@domain.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?query=domain.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by full nickname", %{conn: conn} do
-      user = insert(:user, nickname: "nickname@domain.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by display name", %{conn: conn} do
-      user = insert(:user, name: "Display name")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?name=display")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by email", %{conn: conn} do
-      user = insert(:user, email: "email@example.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?email=email@example.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "regular search with page size", %{conn: conn} do
-      user = insert(:user, nickname: "aalice")
-      user2 = insert(:user, nickname: "alice")
-
-      conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1")
-
-      assert json_response(conn1, 200) == %{
-               "count" => 2,
-               "page_size" => 1,
-               "users" => [user_response(user)]
-             }
-
-      conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
-
-      assert json_response(conn2, 200) == %{
-               "count" => 2,
-               "page_size" => 1,
-               "users" => [user_response(user2)]
-             }
-    end
-
-    test "only local users" do
-      admin = insert(:user, is_admin: true, nickname: "john")
-      token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user, nickname: "bob")
-
-      insert(:user, nickname: "bobb", local: false)
-
-      conn =
-        build_conn()
-        |> assign(:user, admin)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?query=bo&filters=local")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "only local users with no query", %{conn: conn, admin: old_admin} do
-      admin = insert(:user, is_admin: true, nickname: "john")
-      user = insert(:user, nickname: "bob")
-
-      insert(:user, nickname: "bobb", local: false)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=local")
-
-      users =
-        [
-          user_response(user),
-          user_response(admin, %{
-            "roles" => %{"admin" => true, "moderator" => false}
-          }),
-          user_response(old_admin, %{
-            "deactivated" => false,
-            "roles" => %{"admin" => true, "moderator" => false}
-          })
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 3,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "only unconfirmed users", %{conn: conn} do
-      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
-      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
-
-      insert(:user, nickname: "happyboy", approval_pending: false)
-      insert(:user, confirmation_pending: false)
-
-      result =
-        conn
-        |> get("/api/pleroma/admin/users?filters=need_confirmed")
-        |> json_response(200)
-
-      users =
-        Enum.map([old_user, sad_user], fn user ->
-          user_response(user, %{
-            "confirmation_pending" => true,
-            "approval_pending" => false
-          })
-        end)
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
-    end
-
-    test "only unapproved users", %{conn: conn} do
-      user =
-        insert(:user,
-          nickname: "sadboy",
-          approval_pending: true,
-          registration_reason: "Plz let me in!"
-        )
-
-      insert(:user, nickname: "happyboy", approval_pending: false)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
-
-      users =
-        [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user.ap_id,
-            "registration_reason" => "Plz let me in!",
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "load only admins", %{conn: conn, admin: admin} do
-      second_admin = insert(:user, is_admin: true)
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=is_admin")
-
-      users =
-        [
-          %{
-            "deactivated" => false,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => false,
-            "id" => second_admin.id,
-            "nickname" => second_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => second_admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => second_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "load only moderators", %{conn: conn} do
-      moderator = insert(:user, is_moderator: true)
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => moderator.id,
-                   "nickname" => moderator.nickname,
-                   "roles" => %{"admin" => false, "moderator" => true},
-                   "local" => moderator.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => moderator.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
-             }
-    end
-
-    test "load users with tags list", %{conn: conn} do
-      user1 = insert(:user, tags: ["first"])
-      user2 = insert(:user, tags: ["second"])
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second")
-
-      users =
-        [
-          user_response(
-            user1,
-            %{
-              "deactivated" => false,
-              "roles" => %{"admin" => false, "moderator" => false},
-              "local" => user1.local,
-              "tags" => ["first"],
-              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-              "confirmation_pending" => false,
-              "approval_pending" => false,
-              "url" => user1.ap_id,
-              "registration_reason" => nil,
-              "actor_type" => "Person"
-            }
-          ),
-          %{
-            "deactivated" => false,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user2.local,
-            "tags" => ["second"],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user2.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "`active` filters out users pending approval", %{token: token} do
-      insert(:user, approval_pending: true)
-      %{id: user_id} = insert(:user, approval_pending: false)
-      %{id: admin_id} = token.user
-
-      conn =
-        build_conn()
-        |> assign(:user, token.user)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?filters=active")
-
-      assert %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => [
-                 %{"id" => ^admin_id},
-                 %{"id" => ^user_id}
-               ]
-             } = json_response(conn, 200)
-    end
-
-    test "it works with multiple filters" do
-      admin = insert(:user, nickname: "john", is_admin: true)
-      token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user, nickname: "bob", local: false, deactivated: true)
-
-      insert(:user, nickname: "ken", local: true, deactivated: true)
-      insert(:user, nickname: "bobb", local: false, deactivated: false)
-
-      conn =
-        build_conn()
-        |> assign(:user, admin)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?filters=deactivated,external")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "it omits relay user", %{admin: admin, conn: conn} do
-      assert %User{} = Relay.get_actor()
-
-      conn = get(conn, "/api/pleroma/admin/users")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [
-                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
-               ]
-             }
-    end
-  end
-
-  test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do
-    user_one = insert(:user, deactivated: true)
-    user_two = insert(:user, deactivated: true)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/activate",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["deactivated"]) == [false, false]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do
-    user_one = insert(:user, deactivated: false)
-    user_two = insert(:user, deactivated: false)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/deactivate",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["deactivated"]) == [true, true]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do
-    user_one = insert(:user, approval_pending: true)
-    user_two = insert(:user, approval_pending: true)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/approve",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do
-    user = insert(:user)
-
-    conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
-
-    assert json_response(conn, 200) ==
-             user_response(
-               user,
-               %{"deactivated" => !user.deactivated}
-             )
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} deactivated users: @#{user.nickname}"
-  end
-
   describe "PUT disable_mfa" do
     test "returns 200 and disable 2fa", %{conn: conn} do
       user =
@@ -1796,25 +978,6 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
-
-  defp user_response(user, attrs \\ %{}) do
-    %{
-      "deactivated" => user.deactivated,
-      "id" => user.id,
-      "nickname" => user.nickname,
-      "roles" => %{"admin" => false, "moderator" => false},
-      "local" => user.local,
-      "tags" => [],
-      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-      "display_name" => HTML.strip_tags(user.name || user.nickname),
-      "confirmation_pending" => false,
-      "approval_pending" => false,
-      "url" => user.ap_id,
-      "registration_reason" => nil,
-      "actor_type" => "Person"
-    }
-    |> Map.merge(attrs)
-  end
 end
 
 # Needed for testing
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
new file mode 100644
index 000000000..347258058
--- /dev/null
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -0,0 +1,983 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.UserControllerTest do
+  use Pleroma.Web.ConnCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mock
+  import Pleroma.Factory
+
+  alias Pleroma.Config
+  alias Pleroma.HTML
+  alias Pleroma.ModerationLog
+  alias Pleroma.Repo
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MediaProxy
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    :ok
+  end
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  test "with valid `admin_token` query parameter, skips OAuth scopes check" do
+    clear_config([:admin_token], "password123")
+
+    user = insert(:user)
+
+    conn = get(build_conn(), "/api/pleroma/admin/users/#{user.nickname}?admin_token=password123")
+
+    assert json_response(conn, 200)
+  end
+
+  describe "with [:auth, :enforce_oauth_admin_scope_usage]," do
+    setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], true)
+
+    test "GET /api/pleroma/admin/users/:nickname requires admin:read:accounts or broader scope",
+         %{admin: admin} do
+      user = insert(:user)
+      url = "/api/pleroma/admin/users/#{user.nickname}"
+
+      good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"])
+      good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"])
+      good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"])
+
+      bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts"])
+      bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"])
+      bad_token3 = nil
+
+      for good_token <- [good_token1, good_token2, good_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, 200)
+      end
+
+      for good_token <- [good_token1, good_token2, good_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, nil)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+
+      for bad_token <- [bad_token1, bad_token2, bad_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, bad_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+    end
+  end
+
+  describe "unless [:auth, :enforce_oauth_admin_scope_usage]," do
+    setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], false)
+
+    test "GET /api/pleroma/admin/users/:nickname requires " <>
+           "read:accounts or admin:read:accounts or broader scope",
+         %{admin: admin} do
+      user = insert(:user)
+      url = "/api/pleroma/admin/users/#{user.nickname}"
+
+      good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"])
+      good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"])
+      good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"])
+      good_token4 = insert(:oauth_token, user: admin, scopes: ["read:accounts"])
+      good_token5 = insert(:oauth_token, user: admin, scopes: ["read"])
+
+      good_tokens = [good_token1, good_token2, good_token3, good_token4, good_token5]
+
+      bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts:partial"])
+      bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"])
+      bad_token3 = nil
+
+      for good_token <- good_tokens do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, 200)
+      end
+
+      for good_token <- good_tokens do
+        conn =
+          build_conn()
+          |> assign(:user, nil)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+
+      for bad_token <- [bad_token1, bad_token2, bad_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, bad_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/users" do
+    test "single user", %{admin: admin, conn: conn} do
+      clear_config([:instance, :federating], true)
+
+      user =
+        insert(:user,
+          avatar: %{"url" => [%{"href" => "https://someurl"}]},
+          banner: %{"url" => [%{"href" => "https://somebanner"}]},
+          bio: "Hello world!",
+          name: "A guy"
+        )
+
+      # Create some activities to check they got deleted later
+      follower = insert(:user)
+      {:ok, _} = CommonAPI.post(user, %{status: "test"})
+      {:ok, _, _, _} = CommonAPI.follow(user, follower)
+      {:ok, _, _, _} = CommonAPI.follow(follower, user)
+      user = Repo.get(User, user.id)
+      assert user.note_count == 1
+      assert user.follower_count == 1
+      assert user.following_count == 1
+      refute user.deactivated
+
+      with_mock Pleroma.Web.Federator,
+        publish: fn _ -> nil end,
+        perform: fn _, _ -> nil end do
+        conn =
+          conn
+          |> put_req_header("accept", "application/json")
+          |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}")
+
+        ObanHelpers.perform_all()
+
+        assert User.get_by_nickname(user.nickname).deactivated
+
+        log_entry = Repo.one(ModerationLog)
+
+        assert ModerationLog.get_log_entry_message(log_entry) ==
+                 "@#{admin.nickname} deleted users: @#{user.nickname}"
+
+        assert json_response(conn, 200) == [user.nickname]
+
+        user = Repo.get(User, user.id)
+        assert user.deactivated
+
+        assert user.avatar == %{}
+        assert user.banner == %{}
+        assert user.note_count == 0
+        assert user.follower_count == 0
+        assert user.following_count == 0
+        assert user.bio == ""
+        assert user.name == nil
+
+        assert called(Pleroma.Web.Federator.publish(:_))
+      end
+    end
+
+    test "multiple users", %{admin: admin, conn: conn} do
+      user_one = insert(:user)
+      user_two = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> delete("/api/pleroma/admin/users", %{
+          nicknames: [user_one.nickname, user_two.nickname]
+        })
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}"
+
+      response = json_response(conn, 200)
+      assert response -- [user_one.nickname, user_two.nickname] == []
+    end
+  end
+
+  describe "/api/pleroma/admin/users" do
+    test "Create", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "lain",
+              "email" => "lain@example.org",
+              "password" => "test"
+            },
+            %{
+              "nickname" => "lain2",
+              "email" => "lain2@example.org",
+              "password" => "test"
+            }
+          ]
+        })
+
+      response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type"))
+      assert response == ["success", "success"]
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == []
+    end
+
+    test "Cannot create user with existing email", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "lain",
+              "email" => user.email,
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => user.email,
+                   "nickname" => "lain"
+                 },
+                 "error" => "email has already been taken",
+                 "type" => "error"
+               }
+             ]
+    end
+
+    test "Cannot create user with existing nickname", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => user.nickname,
+              "email" => "someuser@plerama.social",
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => "someuser@plerama.social",
+                   "nickname" => user.nickname
+                 },
+                 "error" => "nickname has already been taken",
+                 "type" => "error"
+               }
+             ]
+    end
+
+    test "Multiple user creation works in transaction", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "newuser",
+              "email" => "newuser@pleroma.social",
+              "password" => "test"
+            },
+            %{
+              "nickname" => "lain",
+              "email" => user.email,
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => user.email,
+                   "nickname" => "lain"
+                 },
+                 "error" => "email has already been taken",
+                 "type" => "error"
+               },
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => "newuser@pleroma.social",
+                   "nickname" => "newuser"
+                 },
+                 "error" => "",
+                 "type" => "error"
+               }
+             ]
+
+      assert User.get_by_nickname("newuser") === nil
+    end
+  end
+
+  describe "/api/pleroma/admin/users/:nickname" do
+    test "Show", %{conn: conn} do
+      user = insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
+
+      assert user_response(user) == json_response(conn, 200)
+    end
+
+    test "when the user doesn't exist", %{conn: conn} do
+      user = build(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
+
+      assert %{"error" => "Not found"} == json_response(conn, 404)
+    end
+  end
+
+  describe "/api/pleroma/admin/users/follow" do
+    test "allows to force-follow another user", %{admin: admin, conn: conn} do
+      user = insert(:user)
+      follower = insert(:user)
+
+      conn
+      |> put_req_header("accept", "application/json")
+      |> post("/api/pleroma/admin/users/follow", %{
+        "follower" => follower.nickname,
+        "followed" => user.nickname
+      })
+
+      user = User.get_cached_by_id(user.id)
+      follower = User.get_cached_by_id(follower.id)
+
+      assert User.following?(follower, user)
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}"
+    end
+  end
+
+  describe "/api/pleroma/admin/users/unfollow" do
+    test "allows to force-unfollow another user", %{admin: admin, conn: conn} do
+      user = insert(:user)
+      follower = insert(:user)
+
+      User.follow(follower, user)
+
+      conn
+      |> put_req_header("accept", "application/json")
+      |> post("/api/pleroma/admin/users/unfollow", %{
+        "follower" => follower.nickname,
+        "followed" => user.nickname
+      })
+
+      user = User.get_cached_by_id(user.id)
+      follower = User.get_cached_by_id(follower.id)
+
+      refute User.following?(follower, user)
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}"
+    end
+  end
+
+  describe "GET /api/pleroma/admin/users" do
+    test "renders users array for the first page", %{conn: conn, admin: admin} do
+      user = insert(:user, local: false, tags: ["foo", "bar"])
+      user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude")
+
+      conn = get(conn, "/api/pleroma/admin/users?page=1")
+
+      users =
+        [
+          user_response(
+            admin,
+            %{"roles" => %{"admin" => true, "moderator" => false}}
+          ),
+          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
+          user_response(
+            user2,
+            %{
+              "local" => true,
+              "approval_pending" => true,
+              "registration_reason" => "I'm a chill dude",
+              "actor_type" => "Person"
+            }
+          )
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 3,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "pagination works correctly with service users", %{conn: conn} do
+      service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido")
+
+      insert_list(25, :user)
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users1} =
+               conn
+               |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users1) == 10
+      assert service1 not in users1
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users2} =
+               conn
+               |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users2) == 10
+      assert service1 not in users2
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users3} =
+               conn
+               |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users3) == 6
+      assert service1 not in users3
+    end
+
+    test "renders empty array for the second page", %{conn: conn} do
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?page=2")
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => []
+             }
+    end
+
+    test "regular search", %{conn: conn} do
+      user = insert(:user, nickname: "bob")
+
+      conn = get(conn, "/api/pleroma/admin/users?query=bo")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user, %{"local" => true})]
+             }
+    end
+
+    test "search by domain", %{conn: conn} do
+      user = insert(:user, nickname: "nickname@domain.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?query=domain.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by full nickname", %{conn: conn} do
+      user = insert(:user, nickname: "nickname@domain.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by display name", %{conn: conn} do
+      user = insert(:user, name: "Display name")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?name=display")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by email", %{conn: conn} do
+      user = insert(:user, email: "email@example.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?email=email@example.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "regular search with page size", %{conn: conn} do
+      user = insert(:user, nickname: "aalice")
+      user2 = insert(:user, nickname: "alice")
+
+      conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1")
+
+      assert json_response(conn1, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [user_response(user)]
+             }
+
+      conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
+
+      assert json_response(conn2, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [user_response(user2)]
+             }
+    end
+
+    test "only local users" do
+      admin = insert(:user, is_admin: true, nickname: "john")
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user, nickname: "bob")
+
+      insert(:user, nickname: "bobb", local: false)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?query=bo&filters=local")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "only local users with no query", %{conn: conn, admin: old_admin} do
+      admin = insert(:user, is_admin: true, nickname: "john")
+      user = insert(:user, nickname: "bob")
+
+      insert(:user, nickname: "bobb", local: false)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=local")
+
+      users =
+        [
+          user_response(user),
+          user_response(admin, %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(old_admin, %{
+            "deactivated" => false,
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 3,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "only unconfirmed users", %{conn: conn} do
+      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
+      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+      insert(:user, confirmation_pending: false)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> json_response(200)
+
+      users =
+        Enum.map([old_user, sad_user], fn user ->
+          user_response(user, %{
+            "confirmation_pending" => true,
+            "approval_pending" => false
+          })
+        end)
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
+    end
+
+    test "only unapproved users", %{conn: conn} do
+      user =
+        insert(:user,
+          nickname: "sadboy",
+          approval_pending: true,
+          registration_reason: "Plz let me in!"
+        )
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
+
+      users =
+        [
+          %{
+            "deactivated" => user.deactivated,
+            "id" => user.id,
+            "nickname" => user.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => true,
+            "tags" => [],
+            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(user.name || user.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => true,
+            "url" => user.ap_id,
+            "registration_reason" => "Plz let me in!",
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "load only admins", %{conn: conn, admin: admin} do
+      second_admin = insert(:user, is_admin: true)
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=is_admin")
+
+      users =
+        [
+          %{
+            "deactivated" => false,
+            "id" => admin.id,
+            "nickname" => admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => admin.local,
+            "tags" => [],
+            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => admin.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          },
+          %{
+            "deactivated" => false,
+            "id" => second_admin.id,
+            "nickname" => second_admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => second_admin.local,
+            "tags" => [],
+            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => second_admin.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "load only moderators", %{conn: conn} do
+      moderator = insert(:user, is_moderator: true)
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 %{
+                   "deactivated" => false,
+                   "id" => moderator.id,
+                   "nickname" => moderator.nickname,
+                   "roles" => %{"admin" => false, "moderator" => true},
+                   "local" => moderator.local,
+                   "tags" => [],
+                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
+                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
+                   "confirmation_pending" => false,
+                   "approval_pending" => false,
+                   "url" => moderator.ap_id,
+                   "registration_reason" => nil,
+                   "actor_type" => "Person"
+                 }
+               ]
+             }
+    end
+
+    test "load users with tags list", %{conn: conn} do
+      user1 = insert(:user, tags: ["first"])
+      user2 = insert(:user, tags: ["second"])
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second")
+
+      users =
+        [
+          user_response(
+            user1,
+            %{
+              "deactivated" => false,
+              "roles" => %{"admin" => false, "moderator" => false},
+              "local" => user1.local,
+              "tags" => ["first"],
+              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
+              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
+              "confirmation_pending" => false,
+              "approval_pending" => false,
+              "url" => user1.ap_id,
+              "registration_reason" => nil,
+              "actor_type" => "Person"
+            }
+          ),
+          %{
+            "deactivated" => false,
+            "id" => user2.id,
+            "nickname" => user2.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => user2.local,
+            "tags" => ["second"],
+            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => user2.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "`active` filters out users pending approval", %{token: token} do
+      insert(:user, approval_pending: true)
+      %{id: user_id} = insert(:user, approval_pending: false)
+      %{id: admin_id} = token.user
+
+      conn =
+        build_conn()
+        |> assign(:user, token.user)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?filters=active")
+
+      assert %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => [
+                 %{"id" => ^admin_id},
+                 %{"id" => ^user_id}
+               ]
+             } = json_response(conn, 200)
+    end
+
+    test "it works with multiple filters" do
+      admin = insert(:user, nickname: "john", is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user, nickname: "bob", local: false, deactivated: true)
+
+      insert(:user, nickname: "ken", local: true, deactivated: true)
+      insert(:user, nickname: "bobb", local: false, deactivated: false)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?filters=deactivated,external")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "it omits relay user", %{admin: admin, conn: conn} do
+      assert %User{} = Relay.get_actor()
+
+      conn = get(conn, "/api/pleroma/admin/users")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
+               ]
+             }
+    end
+  end
+
+  test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do
+    user_one = insert(:user, deactivated: true)
+    user_two = insert(:user, deactivated: true)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/activate",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["deactivated"]) == [false, false]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do
+    user_one = insert(:user, deactivated: false)
+    user_two = insert(:user, deactivated: false)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/deactivate",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["deactivated"]) == [true, true]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do
+    user_one = insert(:user, approval_pending: true)
+    user_two = insert(:user, approval_pending: true)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/approve",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do
+    user = insert(:user)
+
+    conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
+
+    assert json_response(conn, 200) ==
+             user_response(
+               user,
+               %{"deactivated" => !user.deactivated}
+             )
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} deactivated users: @#{user.nickname}"
+  end
+
+  defp user_response(user, attrs \\ %{}) do
+    %{
+      "deactivated" => user.deactivated,
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "roles" => %{"admin" => false, "moderator" => false},
+      "local" => user.local,
+      "tags" => [],
+      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+      "display_name" => HTML.strip_tags(user.name || user.nickname),
+      "confirmation_pending" => false,
+      "approval_pending" => false,
+      "url" => user.ap_id,
+      "registration_reason" => nil,
+      "actor_type" => "Person"
+    }
+    |> Map.merge(attrs)
+  end
+end

From 46b420aa602050d7b3bff33a6b51d54852b2adb3 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 09:39:49 +0300
Subject: [PATCH 126/208] need_confirmed -> unconfirmed

---
 lib/pleroma/user/query.ex                                | 4 ++--
 lib/pleroma/web/admin_api/controllers/user_controller.ex | 2 +-
 test/pleroma/web/admin_api/search_test.exs               | 2 +-
 test/web/admin_api/controllers/user_controller_test.exs  | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 2933e7fb4..711439eef 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -43,7 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
-            need_confirmed: boolean(),
+            unconfirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -157,7 +157,7 @@ defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
-  defp compose_query({:need_confirmed, _}, query) do
+  defp compose_query({:unconfirmed, _}, query) do
     where(query, [u], u.confirmation_pending)
   end
 
diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index 5e049aa0f..db6ef3220 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -259,7 +259,7 @@ def list(conn, params) do
     end
   end
 
-  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
+  @filters ~w(local external active deactivated need_approval unconfirmed is_admin is_moderator)
 
   @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
   defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index 27ca396e6..82da86f7f 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -184,7 +184,7 @@ test "it returns unconfirmed user" do
       insert(:user)
 
       {:ok, _results, total} = Search.user()
-      {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true})
+      {:ok, [^unconfirmed], count} = Search.user(%{unconfirmed: true})
       assert total == 3
       assert count == 1
     end
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index 347258058..6384f8ddc 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -644,7 +644,7 @@ test "only unconfirmed users", %{conn: conn} do
 
       result =
         conn
-        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> get("/api/pleroma/admin/users?filters=unconfirmed")
         |> json_response(200)
 
       users =

From 60663150b5b936c831bdc0cfeade30867e536317 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:04:01 +0300
Subject: [PATCH 127/208] admin user search: added filter by `actor_type`

---
 lib/pleroma/user/query.ex                  |  7 ++++++-
 test/pleroma/web/admin_api/search_test.exs | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 711439eef..7ef2a1455 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -56,7 +56,8 @@ defmodule Pleroma.User.Query do
             ap_id: [String.t()],
             order_by: term(),
             select: term(),
-            limit: pos_integer()
+            limit: pos_integer(),
+            actor_types: [String.t()]
           }
           | map()
 
@@ -115,6 +116,10 @@ defp compose_query({:is_admin, bool}, query) do
     where(query, [u], u.is_admin == ^bool)
   end
 
+  defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do
+    where(query, [u], u.actor_type in ^actor_types)
+  end
+
   defp compose_query({:is_moderator, bool}, query) do
     where(query, [u], u.is_moderator == ^bool)
   end
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index 82da86f7f..92a116c65 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -143,6 +143,20 @@ test "it returns users with tags" do
       assert user2 in users
     end
 
+    test "it returns users by actor_types" do
+      user_service = insert(:user, actor_type: "Service")
+      user_application = insert(:user, actor_type: "Application")
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      {:ok, [^user_service], 1} = Search.user(%{actor_types: ["Service"]})
+      {:ok, [^user_application], 1} = Search.user(%{actor_types: ["Application"]})
+      {:ok, [^user1, ^user2], 2} = Search.user(%{actor_types: ["Person"]})
+
+      {:ok, [^user_service, ^user1, ^user2], 3} =
+        Search.user(%{actor_types: ["Person", "Service"]})
+    end
+
     test "it returns user by display name" do
       user = insert(:user, name: "Display name")
       insert(:user)

From 44e5a57d1a1b36ad2971763f3a8cab82bdb94b08 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:37:55 +0300
Subject: [PATCH 128/208] admin api: added user filters by `actor_types`

---
 .../admin_api/controllers/user_controller.ex  |   3 +-
 .../controllers/user_controller_test.exs      | 164 ++++++++----------
 2 files changed, 76 insertions(+), 91 deletions(-)

diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index db6ef3220..15cec9b09 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -243,7 +243,8 @@ def list(conn, params) do
         page_size: page_size,
         tags: params["tags"],
         name: params["name"],
-        email: params["email"]
+        email: params["email"],
+        actor_types: params["actor_types"]
       }
       |> Map.merge(filters)
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index 6384f8ddc..b8a927e56 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -673,23 +673,10 @@ test "only unapproved users", %{conn: conn} do
 
       users =
         [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user.ap_id,
-            "registration_reason" => "Plz let me in!",
-            "actor_type" => "Person"
-          }
+          user_response(user,
+            %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
+          )
         ]
-        |> Enum.sort_by(& &1["nickname"])
 
       assert json_response(conn, 200) == %{
                "count" => 1,
@@ -707,36 +694,14 @@ test "load only admins", %{conn: conn, admin: admin} do
 
       users =
         [
-          %{
+          user_response(admin, %{
             "deactivated" => false,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(second_admin, %{
             "deactivated" => false,
-            "id" => second_admin.id,
-            "nickname" => second_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => second_admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => second_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -758,25 +723,73 @@ test "load only moderators", %{conn: conn} do
                "count" => 1,
                "page_size" => 50,
                "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => moderator.id,
-                   "nickname" => moderator.nickname,
-                   "roles" => %{"admin" => false, "moderator" => true},
-                   "local" => moderator.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => moderator.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
+                 user_response(moderator, %{
+                       "deactivated" => false,
+                       "roles" => %{"admin" => false, "moderator" => true}
+                               })
                ]
              }
     end
 
+    test "load users with actor_type is Person", %{admin: admin, conn: conn} do
+      insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      response = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Person"]})
+      |> json_response(200)
+
+      users =
+        [
+          user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}),
+          user_response(user1),
+          user_response(user2)
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert response == %{"count" => 3, "page_size" => 50, "users" => users}
+    end
+
+    test "load users with actor_type is Person and Service", %{admin: admin, conn: conn} do
+      user_service = insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      response  = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
+      |> json_response(200)
+
+      users =
+        [
+          user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}),
+          user_response(user1),
+          user_response(user2),
+          user_response(user_service, %{"actor_type" => "Service"})
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert response == %{"count" => 4, "page_size" => 50, "users" => users}
+    end
+
+    test "load users with actor_type is Service", %{conn: conn} do
+      user_service = insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+      insert(:user)
+      insert(:user)
+
+      response = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Service"]})
+      |> json_response(200)
+      users = [user_response(user_service, %{"actor_type" => "Service"})]
+
+      assert response == %{"count" => 1, "page_size" => 50, "users" => users }
+    end
+
     test "load users with tags list", %{conn: conn} do
       user1 = insert(:user, tags: ["first"])
       user2 = insert(:user, tags: ["second"])
@@ -787,37 +800,8 @@ test "load users with tags list", %{conn: conn} do
 
       users =
         [
-          user_response(
-            user1,
-            %{
-              "deactivated" => false,
-              "roles" => %{"admin" => false, "moderator" => false},
-              "local" => user1.local,
-              "tags" => ["first"],
-              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-              "confirmation_pending" => false,
-              "approval_pending" => false,
-              "url" => user1.ap_id,
-              "registration_reason" => nil,
-              "actor_type" => "Person"
-            }
-          ),
-          %{
-            "deactivated" => false,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user2.local,
-            "tags" => ["second"],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user2.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+          user_response(user1, %{"tags" => ["first"]}),
+          user_response(user2, %{"tags" => ["second"]})
         ]
         |> Enum.sort_by(& &1["nickname"])
 

From 51189ad365a462163a4c98a7b5cb7ae74f2d634a Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:40:50 +0300
Subject: [PATCH 129/208] update docs

---
 docs/API/admin_api.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 7bf13daef..f7b5bcae7 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -20,12 +20,14 @@ Configuration options:
     - `external`: only external users
     - `active`: only active users
     - `need_approval`: only unapproved users
+    - `unconfirmed`: only unconfirmed users
     - `deactivated`: only deactivated users
     - `is_admin`: users with admin role
     - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
   - *optional* `tags`: **[string]** tags list
+  - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
 - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`

From add26817e3a8756d2f6aea256c6d4b2070a8070b Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:50:47 +0300
Subject: [PATCH 130/208] update changelog

---
 CHANGELOG.md                                  |  2 +
 .../controllers/user_controller_test.exs      | 42 ++++++++++---------
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05e94581a..82c5eaf8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Importing the mutes users from CSV files.
 - Admin API: Importing emoji from a zip file
 - Pleroma API: Pagination for remote/local packs and emoji.
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
 
 </details>
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index b8a927e56..da26caf25 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -671,12 +671,12 @@ test "only unapproved users", %{conn: conn} do
 
       conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
 
-      users =
-        [
-          user_response(user,
-            %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
-          )
-        ]
+      users = [
+        user_response(
+          user,
+          %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
+        )
+      ]
 
       assert json_response(conn, 200) == %{
                "count" => 1,
@@ -724,9 +724,9 @@ test "load only moderators", %{conn: conn} do
                "page_size" => 50,
                "users" => [
                  user_response(moderator, %{
-                       "deactivated" => false,
-                       "roles" => %{"admin" => false, "moderator" => true}
-                               })
+                   "deactivated" => false,
+                   "roles" => %{"admin" => false, "moderator" => true}
+                 })
                ]
              }
     end
@@ -738,9 +738,10 @@ test "load users with actor_type is Person", %{admin: admin, conn: conn} do
       user1 = insert(:user)
       user2 = insert(:user)
 
-      response = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Person"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Person"]})
+        |> json_response(200)
 
       users =
         [
@@ -760,9 +761,10 @@ test "load users with actor_type is Person and Service", %{admin: admin, conn: c
       user1 = insert(:user)
       user2 = insert(:user)
 
-      response  = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
+        |> json_response(200)
 
       users =
         [
@@ -782,12 +784,14 @@ test "load users with actor_type is Service", %{conn: conn} do
       insert(:user)
       insert(:user)
 
-      response = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Service"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Service"]})
+        |> json_response(200)
+
       users = [user_response(user_service, %{"actor_type" => "Service"})]
 
-      assert response == %{"count" => 1, "page_size" => 50, "users" => users }
+      assert response == %{"count" => 1, "page_size" => 50, "users" => users}
     end
 
     test "load users with tags list", %{conn: conn} do

From ef627b9391e0dde8adf03c0132fbc2eeac6bdede Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 22 Oct 2020 12:04:23 +0300
Subject: [PATCH 131/208] fix module name

---
 lib/pleroma/web/admin_api/controllers/user_controller.ex        | 2 +-
 .../web/admin_api/controllers/user_controller_test.exs          | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test/{ => pleroma}/web/admin_api/controllers/user_controller_test.exs (100%)

diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index 15cec9b09..a2a1c875d 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -9,13 +9,13 @@ defmodule Pleroma.Web.AdminAPI.UserController do
     only: [fetch_integer_param: 3]
 
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.AdminAPI.Search
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   @users_page_size 50
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
similarity index 100%
rename from test/web/admin_api/controllers/user_controller_test.exs
rename to test/pleroma/web/admin_api/controllers/user_controller_test.exs

From 8d251096fe71d289cf6a50f3d5638fb43b07c24a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 22 Oct 2020 12:22:08 +0200
Subject: [PATCH 132/208] SideEffects: Correctly handle chat messages sent to
 yourself

---
 lib/pleroma/web/activity_pub/side_effects.ex |  1 +
 test/pleroma/web/common_api_test.exs         | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index d421ca7af..0fff5faf2 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -306,6 +306,7 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
 
       streamables =
         [[actor, recipient], [recipient, actor]]
+        |> Enum.uniq()
         |> Enum.map(fn [user, other_user] ->
           if user.local do
             {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index f5d09f396..d8dc90173 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -95,6 +95,20 @@ test "it blocks and does not federate if outgoing blocks are disabled", %{
   describe "posting chat messages" do
     setup do: clear_config([:instance, :chat_limit])
 
+    test "it posts a self-chat" do
+      author = insert(:user)
+      recipient = author
+
+      {:ok, activity} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          "remember to buy milk when milk truk arive"
+        )
+
+      assert activity.data["type"] == "Create"
+    end
+
     test "it posts a chat message without content but with an attachment" do
       author = insert(:user)
       recipient = insert(:user)

From 3fdc2a0d1e96b45bf71a822afeda72ea667e2b37 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 22 Oct 2020 12:23:07 +0200
Subject: [PATCH 133/208] Changelog: Add information about self-chats

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05e94581a..afeaa930b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- Allow sending chat messages to yourself
 
 ## Unreleased (Patch)
 

From 8a55de1d785178786b9ae7c185ea1d1f187f7470 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 22 Oct 2020 13:54:15 +0300
Subject: [PATCH 134/208] [#3059] Fixed Phoenix 1.5 telemetry processing.

---
 lib/pleroma/application.ex  | 5 ++++-
 lib/pleroma/web/endpoint.ex | 2 ++
 mix.exs                     | 2 ++
 mix.lock                    | 1 +
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index d7e3a649f..51e9dda3b 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -151,7 +151,10 @@ defp setup_instrumenters do
 
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
-    Pleroma.Web.Endpoint.Instrumenter.setup()
+
+    # Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
+    # Pleroma.Web.Endpoint.Instrumenter.setup()
+    PrometheusPhx.setup()
   end
 
   defp cachex_children do
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 56562c12f..f15aab8cb 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -9,6 +9,8 @@ defmodule Pleroma.Web.Endpoint do
 
   socket("/socket", Pleroma.Web.UserSocket)
 
+  plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]
+
   plug(Pleroma.Web.Plugs.SetLocalePlug)
   plug(CORSPlug)
   plug(Pleroma.Web.Plugs.HTTPSecurityPlug)
diff --git a/mix.exs b/mix.exs
index c943d3e17..c9fca1b6e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -171,6 +171,8 @@ defp deps do
        override: true},
       {:prometheus_plugs, "~> 1.1"},
       {:prometheus_phoenix, "~> 1.3"},
+      # Note: once `prometheus_phx` is integrated into `prometheus_phoenix`, remove the former:
+      {:prometheus_phx, github: "theblitzapp/prometheus-phx"},
       {:prometheus_ecto, "~> 1.4"},
       {:recon, "~> 2.5"},
       {:quack, "~> 0.1.1"},
diff --git a/mix.lock b/mix.lock
index 8972959da..159804e04 100644
--- a/mix.lock
+++ b/mix.lock
@@ -103,6 +103,7 @@
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"},
   "prometheus_ex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/prometheus.ex.git", "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5", [ref: "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5"]},
   "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"},
+  "prometheus_phx": {:git, "https://github.com/theblitzapp/prometheus-phx.git", "0c950ac2d145b1ee3fc8ee5c3290ccb9ef2331e9", []},
   "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"},
   "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},

From 8a68673eeda5bcf314051ef83d5a402cf280a52b Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 22 Oct 2020 14:07:33 +0300
Subject: [PATCH 135/208] [#3059] Formatting fix.

---
 lib/pleroma/web/endpoint.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index f15aab8cb..d0e01f3d9 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.Endpoint do
 
   socket("/socket", Pleroma.Web.UserSocket)
 
-  plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]
+  plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
 
   plug(Pleroma.Web.Plugs.SetLocalePlug)
   plug(CORSPlug)

From 14900164b8e56d8dc5506016e5a0b55632306e34 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 22 Oct 2020 17:58:25 +0300
Subject: [PATCH 136/208] [#3059] Used forked prometheus-phx to remove log
 spam.

---
 mix.exs  | 4 +++-
 mix.lock | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index c9fca1b6e..e0da696ce 100644
--- a/mix.exs
+++ b/mix.exs
@@ -172,7 +172,9 @@ defp deps do
       {:prometheus_plugs, "~> 1.1"},
       {:prometheus_phoenix, "~> 1.3"},
       # Note: once `prometheus_phx` is integrated into `prometheus_phoenix`, remove the former:
-      {:prometheus_phx, github: "theblitzapp/prometheus-phx"},
+      {:prometheus_phx,
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/prometheus-phx.git",
+       branch: "no-logging"},
       {:prometheus_ecto, "~> 1.4"},
       {:recon, "~> 2.5"},
       {:quack, "~> 0.1.1"},
diff --git a/mix.lock b/mix.lock
index 159804e04..07238f550 100644
--- a/mix.lock
+++ b/mix.lock
@@ -103,7 +103,7 @@
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"},
   "prometheus_ex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/prometheus.ex.git", "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5", [ref: "a4e9beb3c1c479d14b352fd9d6dd7b1f6d7deee5"]},
   "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"},
-  "prometheus_phx": {:git, "https://github.com/theblitzapp/prometheus-phx.git", "0c950ac2d145b1ee3fc8ee5c3290ccb9ef2331e9", []},
+  "prometheus_phx": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/prometheus-phx.git", "9cd8f248c9381ffedc799905050abce194a97514", [branch: "no-logging"]},
   "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"},
   "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"},
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},

From bd033aba482edab9f0f76fcfb4fa90ad6afc5126 Mon Sep 17 00:00:00 2001
From: shironeko <shironeko@tesaguri.club>
Date: Thu, 22 Oct 2020 08:59:44 +0000
Subject: [PATCH 137/208] Translated using Weblate (Chinese (Simplified))

Currently translated at 45.2% (48 of 106 strings)

Translation: Pleroma/Pleroma backend
Translate-URL: https://translate.pleroma.social/projects/pleroma/pleroma/zh_Hans/
---
 priv/gettext/zh_Hans/LC_MESSAGES/errors.po | 48 +++++++++++-----------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/priv/gettext/zh_Hans/LC_MESSAGES/errors.po b/priv/gettext/zh_Hans/LC_MESSAGES/errors.po
index 4f029d558..8b24d4a86 100644
--- a/priv/gettext/zh_Hans/LC_MESSAGES/errors.po
+++ b/priv/gettext/zh_Hans/LC_MESSAGES/errors.po
@@ -3,8 +3,8 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2020-09-20 13:18+0000\n"
-"PO-Revision-Date: 2020-09-20 14:48+0000\n"
-"Last-Translator: Kana <gudzpoz@live.com>\n"
+"PO-Revision-Date: 2020-10-22 18:25+0000\n"
+"Last-Translator: shironeko <shironeko@tesaguri.club>\n"
 "Language-Team: Chinese (Simplified) <https://translate.pleroma.social/"
 "projects/pleroma/pleroma/zh_Hans/>\n"
 "Language: zh_Hans\n"
@@ -49,7 +49,7 @@ msgstr "是被保留的"
 
 ## From Ecto.Changeset.validate_confirmation/3
 msgid "does not match confirmation"
-msgstr ""
+msgstr "与验证不符"
 
 ## From Ecto.Changeset.no_assoc_constraint/3
 msgid "is still associated with this entry"
@@ -138,12 +138,12 @@ msgstr "不能获取收藏"
 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:438
 #, elixir-format
 msgid "Can't like object"
-msgstr ""
+msgstr "不能喜欢对象"
 
 #: lib/pleroma/web/common_api/utils.ex:563
 #, elixir-format
 msgid "Cannot post an empty status without attachments"
-msgstr ""
+msgstr "无法发送空白且不包含附件的状态"
 
 #: lib/pleroma/web/common_api/utils.ex:511
 #, elixir-format
@@ -153,100 +153,100 @@ msgstr ""
 #: lib/pleroma/config/config_db.ex:191
 #, elixir-format
 msgid "Config with params %{params} not found"
-msgstr ""
+msgstr "无法找到包含参数 %{params} 的配置"
 
 #: lib/pleroma/web/common_api/common_api.ex:181
 #: lib/pleroma/web/common_api/common_api.ex:185
 #, elixir-format
 msgid "Could not delete"
-msgstr ""
+msgstr "无法删除"
 
 #: lib/pleroma/web/common_api/common_api.ex:231
 #, elixir-format
 msgid "Could not favorite"
-msgstr ""
+msgstr "无法收藏"
 
 #: lib/pleroma/web/common_api/common_api.ex:453
 #, elixir-format
 msgid "Could not pin"
-msgstr ""
+msgstr "无法置顶"
 
 #: lib/pleroma/web/common_api/common_api.ex:278
 #, elixir-format
 msgid "Could not unfavorite"
-msgstr ""
+msgstr "无法取消收藏"
 
 #: lib/pleroma/web/common_api/common_api.ex:463
 #, elixir-format
 msgid "Could not unpin"
-msgstr ""
+msgstr "无法取消置顶"
 
 #: lib/pleroma/web/common_api/common_api.ex:216
 #, elixir-format
 msgid "Could not unrepeat"
-msgstr ""
+msgstr "无法取消转发"
 
 #: lib/pleroma/web/common_api/common_api.ex:512
 #: lib/pleroma/web/common_api/common_api.ex:521
 #, elixir-format
 msgid "Could not update state"
-msgstr ""
+msgstr "无法更新状态"
 
 #: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:207
 #, elixir-format
 msgid "Error."
-msgstr ""
+msgstr "错误。"
 
 #: lib/pleroma/web/twitter_api/twitter_api.ex:106
 #, elixir-format
 msgid "Invalid CAPTCHA"
-msgstr ""
+msgstr "无效的验证码"
 
 #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:116
 #: lib/pleroma/web/oauth/oauth_controller.ex:568
 #, elixir-format
 msgid "Invalid credentials"
-msgstr ""
+msgstr "无效的凭据"
 
 #: lib/pleroma/plugs/ensure_authenticated_plug.ex:38
 #, elixir-format
 msgid "Invalid credentials."
-msgstr ""
+msgstr "无效的凭据。"
 
 #: lib/pleroma/web/common_api/common_api.ex:355
 #, elixir-format
 msgid "Invalid indices"
-msgstr ""
+msgstr "无效的索引"
 
 #: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:29
 #, elixir-format
 msgid "Invalid parameters"
-msgstr ""
+msgstr "无效的参数"
 
 #: lib/pleroma/web/common_api/utils.ex:414
 #, elixir-format
 msgid "Invalid password."
-msgstr ""
+msgstr "无效的密码。"
 
 #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:220
 #, elixir-format
 msgid "Invalid request"
-msgstr ""
+msgstr "无效的请求"
 
 #: lib/pleroma/web/twitter_api/twitter_api.ex:109
 #, elixir-format
 msgid "Kocaptcha service unavailable"
-msgstr ""
+msgstr "Kocaptcha 服务不可用"
 
 #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:112
 #, elixir-format
 msgid "Missing parameters"
-msgstr ""
+msgstr "缺少参数"
 
 #: lib/pleroma/web/common_api/utils.ex:547
 #, elixir-format
 msgid "No such conversation"
-msgstr ""
+msgstr "没有该对话"
 
 #: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:388
 #: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:414 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:456

From 60e379ce0b74bbe1b0f40a954aec040beab20e4e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 23 Oct 2020 13:53:01 +0200
Subject: [PATCH 138/208] User: Correctly handle whitespace names.

---
 lib/pleroma/user.ex                           |  5 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  |  4 --
 test/fixtures/mewmew_no_name.json             | 46 +++++++++++++++++++
 .../web/activity_pub/activity_pub_test.exs    | 11 +++++
 4 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 test/fixtures/mewmew_no_name.json

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index dc41d0001..72f507f1e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -426,7 +426,6 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
       params,
       [
         :bio,
-        :name,
         :emoji,
         :ap_id,
         :inbox,
@@ -455,7 +454,9 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :accepts_chat_messages
       ]
     )
-    |> validate_required([:name, :ap_id])
+    |> cast(params, [:name], empty_values: [])
+    |> validate_required([:ap_id])
+    |> validate_required([:name], trim: false)
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, @email_regex)
     |> validate_length(:bio, max: bio_limit)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d17c892a7..df18db603 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1378,10 +1378,6 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
-      {:error, "Object has been deleted" = e} ->
-        Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
-        {:error, e}
-
       {:error, {:reject, reason} = e} ->
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
diff --git a/test/fixtures/mewmew_no_name.json b/test/fixtures/mewmew_no_name.json
new file mode 100644
index 000000000..532d4cf70
--- /dev/null
+++ b/test/fixtures/mewmew_no_name.json
@@ -0,0 +1,46 @@
+{
+   "@context" : [
+      "https://www.w3.org/ns/activitystreams",
+      "https://princess.cat/schemas/litepub-0.1.jsonld",
+      {
+         "@language" : "und"
+      }
+   ],
+   "attachment" : [],
+   "capabilities" : {
+      "acceptsChatMessages" : true
+   },
+   "discoverable" : false,
+   "endpoints" : {
+      "oauthAuthorizationEndpoint" : "https://princess.cat/oauth/authorize",
+      "oauthRegistrationEndpoint" : "https://princess.cat/api/v1/apps",
+      "oauthTokenEndpoint" : "https://princess.cat/oauth/token",
+      "sharedInbox" : "https://princess.cat/inbox",
+      "uploadMedia" : "https://princess.cat/api/ap/upload_media"
+   },
+   "followers" : "https://princess.cat/users/mewmew/followers",
+   "following" : "https://princess.cat/users/mewmew/following",
+   "icon" : {
+      "type" : "Image",
+      "url" : "https://princess.cat/media/12794fb50e86911e65be97f69196814049dcb398a2f8b58b99bb6591576e648c.png?name=blobcatpresentpink.png"
+   },
+   "id" : "https://princess.cat/users/mewmew",
+   "image" : {
+      "type" : "Image",
+      "url" : "https://princess.cat/media/05d8bf3953ab6028fc920494ffc643fbee9dcef40d7bdd06f107e19acbfbd7f9.png"
+   },
+   "inbox" : "https://princess.cat/users/mewmew/inbox",
+   "manuallyApprovesFollowers" : true,
+   "name" : " ",
+   "outbox" : "https://princess.cat/users/mewmew/outbox",
+   "preferredUsername" : "mewmew",
+   "publicKey" : {
+      "id" : "https://princess.cat/users/mewmew#main-key",
+      "owner" : "https://princess.cat/users/mewmew",
+      "publicKeyPem" : "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAru7VpygVef4zrFwnj0Mh\nrbO/2z2EdKN3rERtNrT8zWsLXNLQ50lfpRPnGDrd+xq7Rva4EIu0d5KJJ9n4vtY0\nuxK3On9vA2oyjLlR9O0lI3XTrHJborG3P7IPXrmNUMFpHiFHNqHp5tugUrs1gUFq\n7tmOmM92IP4Wjk8qNHFcsfnUbaPTX7sNIhteQKdi5HrTb/6lrEIe4G/FlMKRqxo3\nRNHuv6SNFQuiUKvFzjzazvjkjvBSm+aFROgdHa2tKl88StpLr7xmuY8qNFCRT6W0\nLacRp6c8ah5f03Kd+xCBVhCKvKaF1K0ERnQTBiitUh85md+Mtx/CoDoLnmpnngR3\nvQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+   },
+   "summary" : "please reply to my posts as direct messages if you have many followers",
+   "tag" : [],
+   "type" : "Person",
+   "url" : "https://princess.cat/users/mewmew"
+}
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 6ac883b23..43bd14ee6 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -2273,4 +2273,15 @@ test "`following` still contains self-replies by friends" do
       assert length(activities) == 2
     end
   end
+
+  test "allow fetching of accounts with an empty string name field" do
+    Tesla.Mock.mock(fn
+      %{method: :get, url: "https://princess.cat/users/mewmew"} ->
+        file = File.read!("test/fixtures/mewmew_no_name.json")
+        %Tesla.Env{status: 200, body: file}
+    end)
+
+    {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew")
+    assert user.name == " "
+  end
 end

From a999e8b0b73372ed129c5a26acfe73e5f7478c5b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 23 Oct 2020 13:55:08 +0200
Subject: [PATCH 139/208] Changelog: Add info about whitespace name remote
 users.

---
 CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index afeaa930b..1f15a8b95 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,7 +47,8 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
-- Allow sending chat messages to yourself
+- Allow sending chat messages to yourself.
+- Fix remote users with a whitespace name.
 
 ## Unreleased (Patch)
 

From e7b0840b88838f9e14bd2b09060d89c4a656966c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 23 Oct 2020 15:32:32 -0500
Subject: [PATCH 140/208] NoNewPrivileges breaks ability to send email via
 sendmail because it restricts ability to run setuid/setgid binaries

---
 installation/pleroma.service | 2 --
 1 file changed, 2 deletions(-)

diff --git a/installation/pleroma.service b/installation/pleroma.service
index ee00a3b7a..63e83ed6e 100644
--- a/installation/pleroma.service
+++ b/installation/pleroma.service
@@ -31,8 +31,6 @@ ProtectHome=true
 ProtectSystem=full
 ; Sets up a new /dev mount for the process and only adds API pseudo devices like /dev/null, /dev/zero or /dev/random but not physical devices. Disabled by default because it may not work on devices like the Raspberry Pi.
 PrivateDevices=false
-; Ensures that the service process and all its children can never gain new privileges through execve().
-NoNewPrivileges=true
 ; Drops the sysadmin capability from the daemon.
 CapabilityBoundingSet=~CAP_SYS_ADMIN
 

From ac692ff3e945a67a64d62f942f2fe5a4ed5e98ea Mon Sep 17 00:00:00 2001
From: Kana <gudzpoz@live.com>
Date: Sun, 25 Oct 2020 08:28:17 +0000
Subject: [PATCH 141/208] Fix link reference

---
 docs/installation/otp_en.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 676b10699..98360bcf7 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -43,7 +43,7 @@ Other than things bundled in the OTP release Pleroma depends on:
 
 ### Installing optional packages
 
-Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
   * exiftool

From 1de4ff8b4d7c0e1042a0a8463c29c7914255c301 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 26 Oct 2020 03:23:47 +0100
Subject: [PATCH 142/208] clients.md: Add AndStatus, note supported protocols

---
 docs/clients.md | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/docs/clients.md b/docs/clients.md
index 1e2c14f1b..3d81763e1 100644
--- a/docs/clients.md
+++ b/docs/clients.md
@@ -7,97 +7,105 @@ Feel free to contact us to be added to this list!
 - Homepage: <https://www.pleroma.com/#desktopApp>
 - Source Code: <https://github.com/roma-apps/roma-desktop>
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Social
 - Source Code: <https://gitlab.gnome.org/World/Social>
 - Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
 - Platforms: Linux (GNOME)
 - Note(2019-01-28): Not at a pre-alpha stage yet
+- Features: MastoAPI
 
 ### Whalebird
 - Homepage: <https://whalebird.org/>
 - Source Code: <https://github.com/h3poteto/whalebird-desktop>
 - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ## Handheld
+### AndStatus
+- Homepage: <http://andstatus.org/>
+- Source Code: <https://github.com/andstatus/andstatus/>
+- Platforms: Android
+- Features: MastoAPI, ActivityPub (Client-to-Server)
+
 ### Amaroq
 - Homepage: <https://itunes.apple.com/us/app/amaroq-for-mastodon/id1214116200>
 - Source Code: <https://github.com/ReticentJohn/Amaroq>
 - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
 - Platforms: iOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Fedilab
 - Homepage: <https://fedilab.app/>
 - Source Code: <https://framagit.org/tom79/fedilab/>
 - Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready, Moderation, Text Formatting
+- Features: MastoAPI, Streaming Ready, Moderation, Text Formatting
 
 ### Kyclos
 - Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
 - Platforms: SailfishOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Husky
 - Source code: <https://git.mentality.rip/FWGS/Husky>
 - Contact: [@Husky@enigmatic.observer](https://enigmatic.observer/users/Husky)
 - Platforms: Android
-- Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
+- Features: MastoAPI, No Streaming, Emoji Reactions, Text Formatting, FE Stickers
 
 ### Fedi
 - Homepage: <https://www.fediapp.com/>
 - Source Code: Proprietary, but gratis
 - Platforms: iOS, Android
-- Features: Pleroma-specific features like Reactions
+- Features: MastoAPI, Pleroma-specific features like Reactions
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>
 - Source Code: <https://github.com/tuskyapp/Tusky>
 - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
 - Source Code: <https://github.com/TwidereProject/Twidere-Android/>
 - Contact: <me@mariotaku.org>
 - Platform: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Indigenous
 - Homepage: <https://indigenous.realize.be/>
 - Source Code: <https://github.com/swentel/indigenous-android/>
 - Contact: [@swentel@realize.be](https://realize.be)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
 - Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Pinafore
 - Homepage: <https://pinafore.social/>
 - Source Code: <https://github.com/nolanlawson/pinafore>
 - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
 - Note: Pleroma support is a secondary goal
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Sengi
 - Homepage: <https://nicolasconstant.github.io/sengi/>
 - Source Code: <https://github.com/NicolasConstant/sengi>
 - Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
+- Features: MastoAPI
 
 ### DashFE
 - Source Code: <https://notabug.org/daisuke/DashboardFE>
@@ -107,3 +115,4 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://git.freesoftwareextremist.com/bloat/>
 - Contact: [@r@freesoftwareextremist.com](https://freesoftwareextremist.com/users/r)
 - Features: Does not requires JavaScript
+- Features: MastoAPI

From de6d49c8cec84a530f2835313c95064ae8df3604 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 26 Oct 2020 16:33:26 +0100
Subject: [PATCH 143/208] ActivityPub: Add back debug call + explanation.

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index df18db603..13869f897 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1378,6 +1378,11 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
+      # If this has been deleted, only log a debug and not an error
+      {:error, "Object has been deleted" = e} ->
+        Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+        {:error, e}
+
       {:error, {:reject, reason} = e} ->
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}

From cbe41408e4f77da55d59ee3d4d26d002a1f20f02 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 27 Oct 2020 14:37:48 -0500
Subject: [PATCH 144/208] phoenix_controller_render_duration is no longer
 available in telemetry of Phoenix 1.5+

---
 test/pleroma/web/endpoint/metrics_exporter_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs
index f954cc1e7..875addc96 100644
--- a/test/pleroma/web/endpoint/metrics_exporter_test.exs
+++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs
@@ -38,7 +38,6 @@ test "serves app metrics", %{conn: conn} do
       for metric <- [
             "http_requests_total",
             "http_request_duration_microseconds",
-            "phoenix_controller_render_duration",
             "phoenix_controller_call_duration",
             "telemetry_scrape_duration",
             "erlang_vm_memory_atom_bytes_total"

From d28f72a55af9442719ff01fe7052802c285f6ea8 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 27 Oct 2020 22:58:55 +0300
Subject: [PATCH 145/208] FrontStatic plug: excluded invalid url

---
 lib/pleroma/web/plugs/frontend_static.ex      | 26 +++++++++++--------
 .../web/plugs/frontend_static_plug_test.exs   | 21 +++++++++++++++
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/web/plugs/frontend_static.ex b/lib/pleroma/web/plugs/frontend_static.ex
index ceb10dcf8..1b0b36813 100644
--- a/lib/pleroma/web/plugs/frontend_static.ex
+++ b/lib/pleroma/web/plugs/frontend_static.ex
@@ -34,22 +34,26 @@ def init(opts) do
   end
 
   def call(conn, opts) do
-    frontend_type = Map.get(opts, :frontend_type, :primary)
-    path = file_path("", frontend_type)
-
-    if path do
-      conn
-      |> call_static(opts, path)
+    with false <- invalid_path?(conn.path_info),
+         frontend_type <- Map.get(opts, :frontend_type, :primary),
+         path when not is_nil(path) <- file_path("", frontend_type) do
+      call_static(conn, opts, path)
     else
-      conn
+      _ ->
+        conn
     end
   end
 
-  defp call_static(conn, opts, from) do
-    opts =
-      opts
-      |> Map.put(:from, from)
+  defp invalid_path?(list) do
+    invalid_path?(list, :binary.compile_pattern(["/", "\\", ":", "\0"]))
+  end
 
+  defp invalid_path?([h | _], _match) when h in [".", "..", ""], do: true
+  defp invalid_path?([h | t], match), do: String.contains?(h, match) or invalid_path?(t)
+  defp invalid_path?([], _match), do: false
+
+  defp call_static(conn, opts, from) do
+    opts = Map.put(opts, :from, from)
     Plug.Static.call(conn, opts)
   end
 end
diff --git a/test/pleroma/web/plugs/frontend_static_plug_test.exs b/test/pleroma/web/plugs/frontend_static_plug_test.exs
index f6f7d7bdb..8b7b022fc 100644
--- a/test/pleroma/web/plugs/frontend_static_plug_test.exs
+++ b/test/pleroma/web/plugs/frontend_static_plug_test.exs
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do
   use Pleroma.Web.ConnCase
+  import Mock
 
   @dir "test/tmp/instance_static"
 
@@ -53,4 +54,24 @@ test "overrides existing static files for the `pleroma/admin` path", %{conn: con
     index = get(conn, "/pleroma/admin/")
     assert html_response(index, 200) == "from frontend plug"
   end
+
+  test "exclude invalid path", %{conn: conn} do
+    name = "pleroma-fe"
+    ref = "dist"
+    clear_config([:media_proxy, :enabled], true)
+    clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+    clear_config([:frontends, :primary], %{"name" => name, "ref" => ref})
+    path = "#{@dir}/frontends/#{name}/#{ref}"
+
+    File.mkdir_p!("#{path}/proxy/rr/ss")
+    File.write!("#{path}/proxy/rr/ss/Ek7w8WPVcAApOvN.jpg:large", "FB image")
+
+    url =
+      Pleroma.Web.MediaProxy.encode_url("https://pbs.twimg.com/media/Ek7w8WPVcAApOvN.jpg:large")
+
+    with_mock Pleroma.ReverseProxy,
+      call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+      assert %Plug.Conn{status: :success} = get(conn, url)
+    end
+  end
 end

From 4f90077767b416f3469fe7c8acfaa6932c579ec2 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 28 Oct 2020 15:32:44 +0400
Subject: [PATCH 146/208] Fix warning

---
 test/pleroma/user/backup_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
index 513798911..f68e4a029 100644
--- a/test/pleroma/user/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -82,7 +82,7 @@ test "it removes outdated backups after creating a fresh one" do
 
     assert {:ok, job1} = Backup.create(user)
 
-    assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
+    assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
     assert {:ok, job2} = Backup.create(user)
     assert Pleroma.Repo.aggregate(Backup, :count) == 2
     assert {:ok, backup2} = ObanHelpers.perform(job2)

From da4a1e57b11d5600788b90f21d18bbcd97f6849f Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 28 Oct 2020 19:09:38 +0300
Subject: [PATCH 147/208] @doc fix.

---
 lib/pleroma/web/static_fe/static_fe_controller.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index 76b82589f..bdec0897a 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -19,7 +19,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
 
   @page_keys ["max_id", "min_id", "limit", "since_id", "order"]
 
-  @doc "Renders requested local public activity"
+  @doc "Renders requested local public activity or public activities of requested user"
   def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     with %Activity{local: true} = activity <-
            Activity.get_by_id_with_object(notice_id),
@@ -46,7 +46,6 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     end
   end
 
-  @doc "Renders public activities of requested user"
   def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
     with {_, %User{local: true} = user} <-
            {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)},

From 9f5f7dc9f956359204fc44a0627e20fd9765d8bd Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 28 Oct 2020 22:29:52 +0300
Subject: [PATCH 148/208] Fixed User.is_discoverable attribute rendering in
 Admin API User view.

---
 lib/pleroma/web/admin_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index bda7ea19c..8bac24d3e 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :skip_thread_containment,
       :pleroma_settings_store,
       :raw_fields,
-      :discoverable,
+      :is_discoverable,
       :actor_type
     ])
     |> Map.merge(%{

From 89c356d19fde3349a741522e7940a281487d9b08 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 29 Oct 2020 14:22:07 -0500
Subject: [PATCH 149/208] Improve Keyword descriptions for AdminFE

---
 config/description.exs | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 0bfa9979f..798cbe2ad 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1757,28 +1757,37 @@
     related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
     label: "MRF Keyword",
     type: :group,
-    description: "Reject or Word-Replace messages with a keyword or regex",
+    description:
+      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
     children: [
       %{
         key: :reject,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being rejected.
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :federated_timeline_removal,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :replace,
         type: {:list, :tuple},
-        description:
-          "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
-        suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
+        description: """
+          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+          **Replacement**: a string. Leaving the field empty is permitted.
+        """
       }
     ]
   },

From 241bd061fc60a5c90c172f46f3b4e576ba660aaf Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 16 Oct 2020 18:28:27 +0000
Subject: [PATCH 150/208] ConversationView: add current user to conversations,
 according to Mastodon behaviour

---
 lib/pleroma/web/mastodon_api/views/conversation_view.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index a91994915..cf34933ab 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -33,12 +33,10 @@ def render("participation.json", %{participation: participation, for: user}) do
       end
 
     activity = Activity.get_by_id_with_object(last_activity_id)
-    # Conversations return all users except the current user.
-    users = Enum.reject(participation.recipients, &(&1.id == user.id))
 
     %{
       id: participation.id |> to_string(),
-      accounts: render(AccountView, "index.json", users: users, for: user),
+      accounts: render(AccountView, "index.json", users: participation.recipients, for: user),
       unread: !participation.read,
       last_status:
         render(StatusView, "show.json",

From 390a12d4c892e58e12546a78bc02dcc0e3a3484b Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 15:58:06 +0000
Subject: [PATCH 151/208] ConversationControllerTest: fix test

---
 .../mastodon_api/controllers/conversation_controller_test.exs  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index b23b22752..afc24027b 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -54,7 +54,8 @@ test "returns correct conversations", %{
              ] = response
 
       account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 2
+      assert length(res_accounts) == 3
+      assert user_one.id in account_ids
       assert user_two.id in account_ids
       assert user_three.id in account_ids
       assert is_binary(res_id)

From 149589c842e677a082436db927834dd6f1b10cb5 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 16:01:17 +0000
Subject: [PATCH 152/208] ConversationViewTest: fix test

---
 .../web/mastodon_api/views/conversation_view_test.exs       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index 2e8203c9b..bd58fb254 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -37,8 +37,10 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
 
-    assert [account] = conversation.accounts
-    assert account.id == other_user.id
+    account_ids = Enum.map(conversation.accounts, & &1["id"])
+    assert length(conversation.accounts) == 2
+    assert user.id in account_ids
+    assert other_user.id in account_ids
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end
 end

From 630eb0f939013db721c78e9b33e4e8bdc8232834 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 19:12:42 +0000
Subject: [PATCH 153/208] ConversationViewTest: fix test #2

---
 test/pleroma/web/mastodon_api/views/conversation_view_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index bd58fb254..81a471cb5 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -37,7 +37,7 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
 
-    account_ids = Enum.map(conversation.accounts, & &1["id"])
+    account_ids = Enum.map(conversation.accounts, & &1.id)
     assert length(conversation.accounts) == 2
     assert user.id in account_ids
     assert other_user.id in account_ids

From 9b93eef71550eabf55b9728b6c8925a4dede222d Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:01:58 +0100
Subject: [PATCH 154/208] ConversationView: fix last_status.account being
 empty, fix current user being included in group conversations

---
 .../mastodon_api/views/conversation_view.ex   | 12 +++++++--
 .../conversation_controller_test.exs          | 25 +++++++++++++++++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index cf34933ab..4636c00e3 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -34,14 +34,22 @@ def render("participation.json", %{participation: participation, for: user}) do
 
     activity = Activity.get_by_id_with_object(last_activity_id)
 
+    # Conversations return all users except current user when current user is not only participant
+    users = if length(participation.recipients) > 1 do
+      Enum.reject(participation.recipients, &(&1.id == user.id))
+    else
+      participation.recipients
+    end
+
     %{
       id: participation.id |> to_string(),
-      accounts: render(AccountView, "index.json", users: participation.recipients, for: user),
+      accounts: render(AccountView, "index.json", users: users, for: user),
       unread: !participation.read,
       last_status:
         render(StatusView, "show.json",
           activity: activity,
-          direct_conversation_id: participation.id
+          direct_conversation_id: participation.id,
+          for: user
         )
     }
   end
diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index afc24027b..8d07cff3f 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -54,16 +54,37 @@ test "returns correct conversations", %{
              ] = response
 
       account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 3
-      assert user_one.id in account_ids
+      assert length(res_accounts) == 2
+      assert user_one.id not in account_ids
       assert user_two.id in account_ids
       assert user_three.id in account_ids
       assert is_binary(res_id)
       assert unread == false
       assert res_last_status["id"] == direct.id
+      assert res_last_status["account"]["id"] == user_one.id
       assert Participation.unread_count(user_one) == 0
     end
 
+    test "special behaviour when conversation have only one user", %{
+      user: user_one,
+      user_two: user_two,
+      conn: conn
+    } do
+      {:ok, direct} = create_direct_message(user_one, [])
+
+      res_conn = get(conn, "/api/v1/conversations")
+
+      assert response = json_response_and_validate_schema(res_conn, 200)
+      assert [
+               %{
+                 "accounts" => res_accounts,
+                 "last_status" => res_last_status
+               }
+             ] = response
+      assert length(res_accounts) == 1
+      assert res_accounts[0]["id"] == user_one.id
+    end
+
     test "observes limit params", %{
       user: user_one,
       user_two: user_two,

From 5591dc02486c30e4b80061706f7368d4b788b431 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:07:01 +0100
Subject: [PATCH 155/208] Add entry in changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11820d313..c62d20868 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,8 @@ switched to a new configuration mechanism, however it was not officially removed
 - Allow sending chat messages to yourself.
 - Fix remote users with a whitespace name.
 - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
+- Mastodon API: Current user is now included in conversation if it's the only participant
+- Mastodon API: Fixed last_status.account being not filled with account data
 
 ## Unreleased (Patch)
 

From 0552a08dfd4daeca69abca0274bbd6db018e5edb Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:10:19 +0100
Subject: [PATCH 156/208] ConversationControllerTest: fix test, fix formatting

---
 .../controllers/conversation_controller_test.exs             | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index 8d07cff3f..291b6b295 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -75,14 +75,17 @@ test "special behaviour when conversation have only one user", %{
       res_conn = get(conn, "/api/v1/conversations")
 
       assert response = json_response_and_validate_schema(res_conn, 200)
+
       assert [
                %{
                  "accounts" => res_accounts,
                  "last_status" => res_last_status
                }
              ] = response
+
+      account_ids = Enum.map(res_accounts, & &1["id"])
       assert length(res_accounts) == 1
-      assert res_accounts[0]["id"] == user_one.id
+      assert user_one.id in account_ids
     end
 
     test "observes limit params", %{

From d63ec02f31e5ee7bb278c4247a83900aceb9193a Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:25:13 +0100
Subject: [PATCH 157/208] ConversationView: fix formatting

---
 .../web/mastodon_api/views/conversation_view.ex       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index 4636c00e3..545778165 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -35,11 +35,12 @@ def render("participation.json", %{participation: participation, for: user}) do
     activity = Activity.get_by_id_with_object(last_activity_id)
 
     # Conversations return all users except current user when current user is not only participant
-    users = if length(participation.recipients) > 1 do
-      Enum.reject(participation.recipients, &(&1.id == user.id))
-    else
-      participation.recipients
-    end
+    users =
+      if length(participation.recipients) > 1 do
+        Enum.reject(participation.recipients, &(&1.id == user.id))
+      else
+        participation.recipients
+      end
 
     %{
       id: participation.id |> to_string(),

From 1042c30fa53e838f3acae2c176f47997fa425755 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:37:15 +0100
Subject: [PATCH 158/208] ConversationViewTest: fix test

---
 .../pleroma/web/mastodon_api/views/conversation_view_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index 81a471cb5..cd02158f9 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -36,10 +36,10 @@ test "represents a Mastodon Conversation entity" do
 
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
+    assert conversation.last_status.account.id == user.id
 
     account_ids = Enum.map(conversation.accounts, & &1.id)
-    assert length(conversation.accounts) == 2
-    assert user.id in account_ids
+    assert length(conversation.accounts) == 1
     assert other_user.id in account_ids
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end

From 1a98476f48db380b5d26b882dad7fba745b3f160 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 30 Oct 2020 16:39:14 +0400
Subject: [PATCH 159/208] Remove unused aliases

---
 .../web/admin_api/controllers/admin_api_controller_test.exs      | 1 -
 test/pleroma/web/admin_api/controllers/chat_controller_test.exs  | 1 -
 .../admin_api/controllers/instance_document_controller_test.exs  | 1 -
 .../web/admin_api/controllers/o_auth_app_controller_test.exs     | 1 -
 test/pleroma/web/admin_api/controllers/relay_controller_test.exs | 1 -
 .../pleroma/web/admin_api/controllers/report_controller_test.exs | 1 -
 .../pleroma/web/admin_api/controllers/status_controller_test.exs | 1 -
 test/pleroma/web/admin_api/controllers/user_controller_test.exs  | 1 -
 .../web/mastodon_api/controllers/timeline_controller_test.exs    | 1 -
 .../web/pleroma_api/controllers/user_import_controller_test.exs  | 1 -
 test/pleroma/web/plugs/http_security_plug_test.exs               | 1 -
 test/pleroma/web/twitter_api/remote_follow_controller_test.exs   | 1 -
 12 files changed, 12 deletions(-)

diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 34b26dddf..e2d6d5f6e 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   import Swoosh.TestAssertions
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
diff --git a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
index bd4c9c9d1..5aefa1e60 100644
--- a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do
 
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Object
   alias Pleroma.Repo
diff --git a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
index 5f7b042f6..ce867dd0e 100644
--- a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
   use Pleroma.Web.ConnCase, async: true
   import Pleroma.Factory
-  alias Pleroma.Config
 
   @dir "test/tmp/instance_static"
   @default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>)
diff --git a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
index ed7c4172c..f388375d1 100644
--- a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.Web
 
   setup do
diff --git a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
index adadf2b5c..b4c5e7567 100644
--- a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.RelayControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.User
diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
index 57946e6bb..fa746d6ea 100644
--- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
   import Pleroma.Factory
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.ReportNote
diff --git a/test/pleroma/web/admin_api/controllers/status_controller_test.exs b/test/pleroma/web/admin_api/controllers/status_controller_test.exs
index eff78fb0a..a18ef9e4b 100644
--- a/test/pleroma/web/admin_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/status_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.StatusControllerTest do
   import Pleroma.Factory
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.User
diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
index da26caf25..5705306c7 100644
--- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do
   import Mock
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.HTML
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
index c6e0268fd..9f1ee0424 100644
--- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
   import Pleroma.Factory
   import Tesla.Mock
 
-  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
diff --git a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
index 433c97e81..68723de71 100644
--- a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do
   use Pleroma.Web.ConnCase
   use Oban.Testing, repo: Pleroma.Repo
 
-  alias Pleroma.Config
   alias Pleroma.Tests.ObanHelpers
 
   import Pleroma.Factory
diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs
index 2297e3dac..df2b5ebb3 100644
--- a/test/pleroma/web/plugs/http_security_plug_test.exs
+++ b/test/pleroma/web/plugs/http_security_plug_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Config
   alias Plug.Conn
 
   describe "http security enabled" do
diff --git a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
index 3852c7ce9..a3e784d13 100644
--- a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
+++ b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Config
   alias Pleroma.MFA
   alias Pleroma.MFA.TOTP
   alias Pleroma.User

From d1698267a27bd5084916f5f6f36d66b1ff2ffc5f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 31 Oct 2020 00:26:11 +0400
Subject: [PATCH 160/208] Fix credo warning

---
 lib/pleroma/web/router.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 9592d0f38..efe67ad7a 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -148,7 +148,7 @@ defmodule Pleroma.Web.Router do
 
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
-    
+
     put("/users/disable_mfa", AdminAPIController, :disable_mfa)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)

From 8e41baff40555ef7c74c8842d6fbfebc2368631a Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sat, 31 Oct 2020 05:50:48 +0300
Subject: [PATCH 161/208] Add idempotency_key to the chat_message entity.

---
 CHANGELOG.md                                          |  1 +
 docs/API/chats.md                                     |  5 ++++-
 lib/pleroma/application.ex                            |  9 ++++++++-
 lib/pleroma/web/activity_pub/side_effects.ex          |  6 ++++++
 lib/pleroma/web/common_api.ex                         |  3 ++-
 .../web/pleroma_api/controllers/chat_controller.ex    | 10 +++++++++-
 .../pleroma_api/views/chat/message_reference_view.ex  | 11 +++++++++++
 .../pleroma_api/controllers/chat_controller_test.exs  |  2 ++
 .../views/chat_message_reference_view_test.exs        |  5 ++++-
 test/pleroma/web/streamer_test.exs                    |  4 +++-
 10 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11820d313..bb02d7b32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Pagination for remote/local packs and emoji.
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
 
 </details>
 
diff --git a/docs/API/chats.md b/docs/API/chats.md
index aa6119670..9857aac67 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -173,11 +173,14 @@ Returned data:
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
     "id": "12",
-    "unread": false
+    "unread": false,
+    "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
   }
 ]
 ```
 
+- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
+
 ### Posting a chat message
 
 Posting a chat message for given Chat id works like this:
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 51e9dda3b..7c4cd9626 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -168,7 +168,11 @@ defp cachex_children do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("chat_message_id_idempotency_key",
+        expiration: chat_message_id_idempotency_key_expiration(),
+        limit: 500_000
+      )
     ]
   end
 
@@ -178,6 +182,9 @@ defp emoji_packs_expiration,
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
+  defp chat_message_id_idempotency_key_expiration,
+    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0fff5faf2..d552e91fc 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -312,6 +312,12 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
             {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
             {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
+            Cachex.put(
+              :chat_message_id_idempotency_key_cache,
+              cm_ref.id,
+              meta[:idempotency_key]
+            )
+
             {
               ["user", "user:pleroma_chat"],
               {user, %{cm_ref | chat: chat, object: object}}
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 60a50b027..318ffc5d0 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -45,7 +45,8 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
          {_, {:ok, %Activity{} = activity, _meta}} <-
            {:common_pipeline,
             Pipeline.common_pipeline(create_activity_data,
-              local: true
+              local: true,
+              idempotency_key: opts[:idempotency_key]
             )} do
       {:ok, activity}
     else
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 6357148d0..2c4d3f135 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -80,7 +80,8 @@ def post_chat_message(
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
          {:ok, activity} <-
            CommonAPI.post_chat_message(user, recipient, params[:content],
-             media_id: params[:media_id]
+             media_id: params[:media_id],
+             idempotency_key: idempotency_key(conn)
            ),
          message <- Object.normalize(activity, false),
          cm_ref <- MessageReference.for_chat_and_object(chat, message) do
@@ -169,4 +170,11 @@ def show(%{assigns: %{user: user}} = conn, %{id: id}) do
       |> render("show.json", chat: chat)
     end
   end
+
+  defp idempotency_key(conn) do
+    case get_req_header(conn, "idempotency-key") do
+      [key] -> key
+      _ -> nil
+    end
+  end
 end
diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
index d4e08b50d..c058fb340 100644
--- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Maps
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
@@ -37,6 +38,7 @@ def render(
           Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object)
         )
     }
+    |> put_idempotency_key()
   end
 
   def render("index.json", opts) do
@@ -47,4 +49,13 @@ def render("index.json", opts) do
       Map.put(opts, :as, :chat_message_reference)
     )
   end
+
+  defp put_idempotency_key(data) do
+    with {:ok, idempotency_key} <- Cachex.get(:chat_message_id_idempotency_key_cache, data.id) do
+      data
+      |> Maps.put_if_present(:idempotency_key, idempotency_key)
+    else
+      _ -> data
+    end
+  end
 end
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index 6381f9757..fa6b9db65 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -82,11 +82,13 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
       result =
         conn
         |> put_req_header("content-type", "application/json")
+        |> put_req_header("idempotency-key", "123")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
         |> json_response_and_validate_schema(200)
 
       assert result["content"] == "Hallo!!"
       assert result["chat_id"] == chat.id |> to_string()
+      assert result["idempotency_key"] == "123"
     end
 
     test "it fails if there is no content", %{conn: conn, user: user} do
diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
index f171a1e55..ae8257870 100644
--- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -25,7 +25,9 @@ test "it displays a chat message" do
     }
 
     {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
-    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:")
+
+    {:ok, activity} =
+      CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123")
 
     chat = Chat.get(user.id, recipient.ap_id)
 
@@ -42,6 +44,7 @@ test "it displays a chat message" do
     assert chat_message[:created_at]
     assert chat_message[:unread] == false
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
+    assert chat_message[:idempotency_key] == "123"
 
     clear_config([:rich_media, :enabled], true)
 
diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs
index 185724a9f..395016da2 100644
--- a/test/pleroma/web/streamer_test.exs
+++ b/test/pleroma/web/streamer_test.exs
@@ -255,7 +255,9 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{
     } do
       other_user = insert(:user)
 
-      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
+      {:ok, create_activity} =
+        CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123")
+
       object = Object.normalize(create_activity, false)
       chat = Chat.get(user.id, other_user.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)

From 8f00d90f9199e384fb1befb677c1c0595a0c854c Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis <vaartis@cock.li>
Date: Sun, 1 Nov 2020 12:05:39 +0300
Subject: [PATCH 162/208] Use Pleroma.HTTP instead of Tesla

Closes #2275

As discovered in the issue, captcha used Tesla.get instead of
Pleroma.HTTP. I've also grep'ed the repo and changed the other place
where this was used.
---
 lib/pleroma/captcha/kocaptcha.ex | 2 +-
 lib/pleroma/emoji/pack.ex        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex
index 337506647..201b55ab4 100644
--- a/lib/pleroma/captcha/kocaptcha.ex
+++ b/lib/pleroma/captcha/kocaptcha.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
   def new do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
 
-    case Tesla.get(endpoint <> "/new") do
+    case Pleroma.HTTP.get(endpoint <> "/new") do
       {:error, _} ->
         %{error: :kocaptcha_service_unavailable}
 
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index 0670f29f1..ca58e5432 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -594,7 +594,7 @@ defp fetch_pack_info(remote_pack, uri, name) do
   end
 
   defp download_archive(url, sha) do
-    with {:ok, %{body: archive}} <- Tesla.get(url) do
+    with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
       if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
         {:ok, archive}
       else
@@ -617,7 +617,7 @@ defp fallback_sha_changed?(pack, data) do
   end
 
   defp update_sha_and_save_metadata(pack, data) do
-    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+    with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
          :ok <- validate_has_all_files(pack, zip) do
       fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
 

From 4caad4e9101c34debfa90d2e89850d4125a471b3 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 2 Nov 2020 05:43:06 +0100
Subject: [PATCH 163/208] =?UTF-8?q?side=5Feffects:=20Don=E2=80=99t=20incre?=
 =?UTF-8?q?ase=5Freplies=5Fcount=20when=20it=E2=80=99s=20an=20Answer?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/pleroma/web/activity_pub/side_effects.ex                   | 2 +-
 .../web/activity_pub/transmogrifier/answer_handling_test.exs   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0fff5faf2..9b1171d07 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -187,7 +187,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
       {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
 
-      if in_reply_to = object.data["inReplyTo"] do
+      if in_reply_to = object.data["inReplyTo"] && object.data["type"] != "Answer" do
         Object.increase_replies_count(in_reply_to)
       end
 
diff --git a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
index 0f6605c3f..e7d85a2c5 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
@@ -27,6 +27,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
       })
 
     object = Object.normalize(activity)
+    assert object.data["repliesCount"] == nil
 
     data =
       File.read!("test/fixtures/mastodon-vote.json")
@@ -41,7 +42,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
     assert answer_object.data["inReplyTo"] == object.data["id"]
 
     new_object = Object.get_by_ap_id(object.data["id"])
-    assert new_object.data["replies_count"] == object.data["replies_count"]
+    assert new_object.data["repliesCount"] == nil
 
     assert Enum.any?(
              new_object.data["oneOf"],

From be52819a112abb66032a56d613eed0233995eef4 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 2 Nov 2020 17:51:54 +0400
Subject: [PATCH 164/208] Hide chats from muted users

---
 .../controllers/chat_controller.ex            | 27 ++++++++-----------
 .../controllers/chat_controller_test.exs      | 22 +++++++++++++++
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 2c4d3f135..8fc70c15a 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -15,7 +15,6 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   import Ecto.Query
@@ -121,9 +120,7 @@ def mark_as_read(
       ) do
     with {:ok, chat} <- Chat.get_by_user_and_id(user, id),
          {_n, _} <- MessageReference.set_all_seen_for_chat(chat, last_read_id) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
@@ -142,32 +139,30 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do
   end
 
   def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
-    blocked_ap_ids = User.blocked_users_ap_ids(user)
+    exclude_users =
+      user
+      |> User.blocked_users_ap_ids()
+      |> Enum.concat(User.muted_users_ap_ids(user))
 
     chats =
-      Chat.for_user_query(user_id)
-      |> where([c], c.recipient not in ^blocked_ap_ids)
+      user_id
+      |> Chat.for_user_query()
+      |> where([c], c.recipient not in ^exclude_users)
       |> Repo.all()
 
-    conn
-    |> put_view(ChatView)
-    |> render("index.json", chats: chats)
+    render(conn, "index.json", chats: chats)
   end
 
   def create(%{assigns: %{user: user}} = conn, %{id: id}) do
     with %User{ap_id: recipient} <- User.get_cached_by_id(id),
          {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
   def show(%{assigns: %{user: user}} = conn, %{id: id}) do
     with {:ok, chat} <- Chat.get_by_user_and_id(user, id) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index fa6b9db65..b0498df2b 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -343,6 +343,28 @@ test "it does not return chats with users you blocked", %{conn: conn, user: user
       assert length(result) == 0
     end
 
+    test "it does not return chats with users you muted", %{conn: conn, user: user} do
+      recipient = insert(:user)
+
+      {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 1
+
+      User.mute(user, recipient)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 0
+    end
+
     test "it returns all chats", %{conn: conn, user: user} do
       Enum.each(1..30, fn _ ->
         recipient = insert(:user)

From 7efc074eadae9b3d6d351e769ead0661f1f4c89c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 12:19:44 -0600
Subject: [PATCH 165/208] Permit fetching individual reports with notes
 preloaded

---
 lib/pleroma/activity.ex                             | 13 +++++++++++++
 .../web/admin_api/controllers/report_controller.ex  |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 17af04257..553834da0 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.ReportNote
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Ecto.Changeset
   import Ecto.Query
@@ -153,6 +154,18 @@ def get_bookmark(%Activity{} = activity, %User{} = user) do
 
   def get_bookmark(_, _), do: nil
 
+  def get_report(activity_id) do
+    opts = %{
+      type: "Flag",
+      skip_preload: true,
+      preload_report_notes: true
+    }
+
+    ActivityPub.fetch_activities_query([], opts)
+    |> where(id: ^activity_id)
+    |> Repo.one()
+  end
+
   def change(struct, params \\ %{}) do
     struct
     |> cast(params, [:data, :recipients])
diff --git a/lib/pleroma/web/admin_api/controllers/report_controller.ex b/lib/pleroma/web/admin_api/controllers/report_controller.ex
index 86da93893..6a0e56f5f 100644
--- a/lib/pleroma/web/admin_api/controllers/report_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/report_controller.ex
@@ -38,7 +38,7 @@ def index(conn, params) do
   end
 
   def show(conn, %{id: id}) do
-    with %Activity{} = report <- Activity.get_by_id(id) do
+    with %Activity{} = report <- Activity.get_report(id) do
       render(conn, "show.json", Report.extract_report_info(report))
     else
       _ -> {:error, :not_found}

From 53dd048590b93da67f9d4abac8cc111424c4d5c0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 15:49:07 -0600
Subject: [PATCH 166/208] Test the note is returned when fetching a single
 report

---
 .../web/admin_api/controllers/report_controller_test.exs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
index fa746d6ea..958e1d3ab 100644
--- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
@@ -37,12 +37,21 @@ test "returns report by its id", %{conn: conn} do
           status_ids: [activity.id]
         })
 
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/pleroma/admin/reports/#{report_id}/notes", %{
+        content: "this is an admin note"
+      })
+
       response =
         conn
         |> get("/api/pleroma/admin/reports/#{report_id}")
         |> json_response_and_validate_schema(:ok)
 
       assert response["id"] == report_id
+
+      [notes] = response["notes"]
+      assert notes["content"] == "this is an admin note"
     end
 
     test "returns 404 when report id is invalid", %{conn: conn} do

From 2f2281fdf1bd3fbd5d82bb437ce3d43ff9043862 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 17:09:56 -0600
Subject: [PATCH 167/208] Ensure URLs for git repos end in .git for older git
 clients like on CentOS 7

---
 mix.exs  | 4 ++--
 mix.lock | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/mix.exs b/mix.exs
index e0da696ce..0691902a6 100644
--- a/mix.exs
+++ b/mix.exs
@@ -134,7 +134,7 @@ defp deps do
       {:cachex, "~> 3.2"},
       {:poison, "~> 3.0", override: true},
       {:tesla,
-       git: "https://github.com/teamon/tesla/",
+       git: "https://github.com/teamon/tesla.git",
        ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30",
        override: true},
       {:castore, "~> 0.1"},
@@ -196,7 +196,7 @@ defp deps do
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
       {:restarter, path: "./restarter"},
       {:majic,
-       git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"},
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", branch: "develop"},
       {:open_api_spex,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
        ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"},
diff --git a/mix.lock b/mix.lock
index 07238f550..e5d9bc693 100644
--- a/mix.lock
+++ b/mix.lock
@@ -66,7 +66,7 @@
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
   "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"},
-  "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
+  "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
   "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},
@@ -115,7 +115,7 @@
   "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"},
   "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"},
   "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
-  "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
+  "tesla": {:git, "https://github.com/teamon/tesla.git", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
   "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
   "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"},

From 179936609f4fbf51575fabd7af11cf14ba570c0c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 2 Nov 2020 06:11:14 +0100
Subject: [PATCH 168/208] favicon: Update to pleroma logo, provided by @shpuld

Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/2270
---
 priv/static/favicon.png | Bin 1603 -> 1583 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/priv/static/favicon.png b/priv/static/favicon.png
index a96d5d25225abd92a95b35c58b301463752cf335..098040a00d025387fc376f534fb6cb5aef842761 100644
GIT binary patch
literal 1583
zcmds2`#TeQ9R6+=vrN&1qgYOg43(tl%qEv6*=#G4TS}C)b6ldcu}eZO;gEK6O_v8n
ziaD&Ba>>eNbsmbbxnFXZT*5f(=l+B9d7k(4KF|App67kupFTH9E{;gVF9-ktl5ot9
zEN9jaz!l}Y4tusqPOwm00u?SF2K;<H02_u0c6e%}_hb$+{$x0+ahgscPKx3WSvkd+
z9!U1!qG1MXV|>g(lWdQw4<~~(U8_<H{Lr<$D-`Z&J5tp#E`D2Z``cKELLF!8Kt-{`
z^xP|cKp*=0nqX1TGo4GH>RahGc^eu%ojoUup2~}eoBjMQ%%>|MWF>KNyn#-f%{7#Y
z#wvSD#nfgyQ9Z`A`jkp^d1PKer-25=`LPbmGU0TqVQU>(L!)pbI6O8KZ{zt)?~$=F
zNeN^nXk$&J*0%%BR;!^#Z<Xz;u)D>$t49{PV=O?3cqEP1JV>%Of~4wTLqD%xsgKzg
zbhIGOPi#SvC<u#czDd?z&}#UTbpvtUhsz=w2Zu)vUV$sC>x^&zE#hLENK3UGQxmdK
zUpP*AlK7>@Sw;oIMaGi2dcHxr>b5&&BDeebhF#`(d$!!**(_OAMKo@-Y|X<;zl3qF
zsf_K}{KpN?Agb=PqTa|Y+E}s)BWhWz<+H^<&O8(lmdkp_m%mdS`R8&_JWjaQlTsXl
zGN@@yjLNw-T_RcW&Q0}Mp7AHKQD8#3Wxkc|m45Hgxt2YtP*(3pnYeWFO=S5%`JV{D
zv1FN@NJ^nxc!78TUQdNhsO-Ukd^^DwIE%*Y3xfftL2EWxTF`WH{C~)8p0-lGBe3=r
zv}&EaU7KfZzhsD>Z<TO;=(K~pVv9j@vn)i3x~@@wmJ4t<?-G^*czHyT=Q@&Fv!Q>l
z^N*XIjG3Vc#?AM~4MI3ZFtj1j{7s6P_p4c1OV{YKsU@rF+4e_bbHmC_rDmulx`l}7
z!n9tO)T;0I`NmJRJpfI$ZHO0Dt>^dnEDt=r{de`52q(`uy{BR31ruU26i^v!9RYXn
zPDL}ZQujqXR%=1&`nr<|eD?{>zR>_c=75+Rt?JsD^KMECq~gm<9%)OVo#!(evv3bm
zOP2ENtqG;Fl(+hzQbjfV*I_34WqS6qo<_d`dS`VGH+x}7NW3ISas(6H)e4uvShG$G
zW|SJV%v(*zrYX(T@g<}EcyyZ$Rv!{5{e7Ri?yAu2>R_kfh%`C6*6k_=n(9ySZ}`e>
zq(OAOO7?Dn6rwX5snFh!?Fe%<7Q*kEvjO<Ym_3ZkjH2Mp5{q0#`yv&~0*m~JFr4e<
z#D{Q}y&H<I4-)*&VC(^gU(m4yLPB-<89{xUZ%<h{ep=7wdtsG8!mwLwE->Q7(j7%!
z#pOT*!i&2&qFumbZ&B-3WgJeze_F%-H|QRpjs_6T*g71^h#Ij8hI@DHu|J%3G9Y7F
zA2`{bF*ypJSlw^agpkz5m$ae2?b;6oaMoWRPBm=?k6R7r?LkINrAI7q4|gasgXZt7
zXx|TeU2N_Yr3MV{m1OP!r*d4!5`m#|Fi!<iHkL&8f#C>&KSxl$HAE>Ov+jQs_e<!3
z<65Y=;IAsc;F2l6G7ZvRO+#-k<n=M9LwjC-x_hT!wn!8(;>1v0L}GT3$H`d!%s}lX
z8;Lh0IkX3dCXLsIh?(n&o!`~79Nu3hEU5P9#aSj1a&5L@)&$SH+XU<!*j&~cb73&W
z*ip8^=kk=pkWqHegU@#}k=xoCt~M8fg|CEKi^wEYSxZwj%{NPuIFYlW=z!=L9_qF%
zYnifN$eoCtCPd8y^`=uVuJu&WeCKov!#|7)*y{?y^p5PX3_L{s0RX|?#je!Wm-8RQ
CN89cI

literal 1603
zcmV-J2E6%+P)<h;3K|Lk000e1NJLTq001Tc001%w1ONa4nsZ^O000IGNkl<Zcmb`!
z3(ydP0SEB^*Z1*Nx}>M3rg3eNl1HVk(c>Z|vaX`mV@f0+I+dn}9<xLbX_D$A^>wE;
ztwfg|tVJYaxe1%cj7~P@+Og5aZ|CblDYV<~mm?u&nW0_&H=?q_loJD-?8mk^AxBCS
zX>OZSl0-CD<Sxq$vRE|MDc_B_$U`1?g;FDuM4aO(g<4zXr=p?-=H%OmLK~c-h9VE^
zZ@3moBHCMGrI{%whH05^A}V=Eg+$CW+i^x3m~x_qhN88_i5ROxzKIx_h$=3(T3^w_
zyYA3gVakcR$~4u-#(X=`UU%yhn`(~d+^@Md)+tCi(aBQl6pLchot`g=y4g>Z8sTH>
z6EQ{;T}=0?X)5FZqS(c1S?HMTA&RY3IpxHadU)5F3KeN=oQXDiUE3T)lsU#@&a*_*
z>>*0LW~%<K)?F1HJ*HwJ>buP;##*bo%{hp;LrW)FYqGlelIUoSxsG+I*pMWmjY*ER
zz-EKImV=0)R%qi9r7GmBL~S>UX-@K#sHBcghB?=r{%M)Y`5MvDSeN8bL|x~bVtXRC
znXRo}%KQo~73gf7!RDK+UdoB%Ra4us>f}q}Y*F0+)eN)JG|lXAy<#!UUo7#F5!M*w
zT94~xRFa69Hk;~M?_>`#%3Pxp(Zx3XEHcDRs;Xm>yF4$dxlyYm5e070OV=b31;*K;
zb`mkbJ*GQ3<-}nBu}&|;Q%=kiRc!D?_7Y`wIyw=9ZE<40;6z&u%~yz}#+vF@Rg*-#
zE4q5!2P!0@$T&OA(jpO^lzP$}%_3@Bs(ZddbhFjninVd7C3g9dKYPI@)l^nXXYIYA
zt==goPP0;3z92T4=jiMu8lg-yRBD>FF4MxN`Z(RC)~Kb#W{q5%B;qs^P0BuEvMSk2
z^zgO|^-RPAK5?Ia`I9r8sfSCvAZqF0vP2B`l1e!o(c3b$y)IfAZGx+m_*_(UzpVy|
z-&>+pA_{XL(I)$eZpJtcGu)=C4&KuZTXa`5NklDFO~m!OC^1F%B%->BZpwb5w#)VK
zoKcP!*IH;i8i-m>P)T)f>z;@jN)>ufQ*)zGYm2PT0YnQkZL(Ec7pmt0zxJ^Ox;x82
zy=~VkNkn5a)b>}gJhphzts3P3qOJbcSmiv;6Y-W+c*RB=T#~&+ffXLM!zkCqP@_Dk
zK@K2#d(mnG%+c2zW8LBd^vprT(eAZTmn7m`TiuZZh;vQ$sEZw?pWj%m#GAG$%Qq7Z
zy{uHs(KY2nGru(7EOp%O6<Y+Kc{T?SHB3yzWX-dO_`XJ#8)l#qQx%C)eXSCAdeSnn
z8=vO@;%svhG2Ha*BSu<jwG-60$didU)^ah&)8>jV>{2NaH4RG=(cBc(6u8pF>>(Ps
zOcT_zPFr*k_01Ngwwf*$+ijQcC8DxVd}zM&#d^PRvuCo8*l4R>{w<oh*mlE2Q}a|%
z=2PqS^R&-!d?FU9pn|wr1yQIl`-rLvL=$6OsGp&BnjmW1VYM}0_LisJj!zRY+sWd0
zHffi`5j7oWuEn++pNN-qH3Dl5aRnA8VyS8VX1*bM<?uv-1(tZ<@I=fM({Pb{mHOO>
zL@d_RZDOYW`8rW#d?Jo<ohN+cnncV~!$$9E;t$U8j=qVQXqKx)pB#!9p=TnhTHqcp
zxhfGe#ZFXmyQV(YGZDAA!NWN;QS5;v5f!bl!VaSo@kcGOOH5YRZdWJbZqvP*LlKqz
z%uU8yrk$8(p(%EY{yLk631Xm{{_aiD*frReLlH$<J4+{<R1w7%c+dpBP0`Ip)Dp!W
z@k2vgZJ=)6wm63;N^}<O#SJbs+M_yKjGrWlIL|%0i2MA*PCv-ui2l}?ZoOMI61SLd
zIo{VPdx;XP^SlrB%i)RfCO9Yii3a%^QO{91GNO9^UqnX_Iw=to5>cc;E6vq6Us;ZX
zXk&3ADx2p5QDlvdMrvb_pXW%3J|-q&j0ILEV!A?OG}Y3ee0SnZj~Of)dd5(9DG@83
zp`8mXbb5}2I8}Xh{nm?Oi5c2C%6w0G!=xO>e*siY3nVrpr!xQm002ovPDHLkV1irG
B2owMS


From c37118e6f26f0305d540047e4ccb8d594d2c0e6b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 3 Nov 2020 13:56:12 +0100
Subject: [PATCH 169/208] Conversations: A few refactors

---
 .../web/mastodon_api/views/conversation_view.ex      |  3 ++-
 .../controllers/conversation_controller_test.exs     | 12 ++++--------
 .../mastodon_api/views/conversation_view_test.exs    |  6 +++---
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index 545778165..82fcff062 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -34,7 +34,8 @@ def render("participation.json", %{participation: participation, for: user}) do
 
     activity = Activity.get_by_id_with_object(last_activity_id)
 
-    # Conversations return all users except current user when current user is not only participant
+    # Conversations return all users except the current user,
+    # except when the current user is the only participant
     users =
       if length(participation.recipients) > 1 do
         Enum.reject(participation.recipients, &(&1.id == user.id))
diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index 291b6b295..c67e584dd 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -65,12 +65,11 @@ test "returns correct conversations", %{
       assert Participation.unread_count(user_one) == 0
     end
 
-    test "special behaviour when conversation have only one user", %{
+    test "includes the user if the user is the only participant", %{
       user: user_one,
-      user_two: user_two,
       conn: conn
     } do
-      {:ok, direct} = create_direct_message(user_one, [])
+      {:ok, _direct} = create_direct_message(user_one, [])
 
       res_conn = get(conn, "/api/v1/conversations")
 
@@ -78,14 +77,11 @@ test "special behaviour when conversation have only one user", %{
 
       assert [
                %{
-                 "accounts" => res_accounts,
-                 "last_status" => res_last_status
+                 "accounts" => [account]
                }
              ] = response
 
-      account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 1
-      assert user_one.id in account_ids
+      assert user_one.id == account["id"]
     end
 
     test "observes limit params", %{
diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index cd02158f9..20c10ba3d 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -38,9 +38,9 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.last_status.id == activity.id
     assert conversation.last_status.account.id == user.id
 
-    account_ids = Enum.map(conversation.accounts, & &1.id)
-    assert length(conversation.accounts) == 1
-    assert other_user.id in account_ids
+    assert [account] = conversation.accounts
+    assert account.id == other_user.id
+
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end
 end

From 1cfc3278c086c9eaa7b2d1bd170e82c8b2aebd78 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:14:00 +0100
Subject: [PATCH 170/208] Poll View: Always return `voters_count`.

---
 lib/pleroma/web/mastodon_api/views/poll_view.ex        | 2 +-
 test/pleroma/web/mastodon_api/views/poll_view_test.exs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 1208dc9a0..4101f21d0 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -19,7 +19,7 @@ def render("show.json", %{object: object, multiple: multiple, options: options}
       expired: expired,
       multiple: multiple,
       votes_count: votes_count,
-      voters_count: (multiple || nil) && voters_count(object),
+      voters_count: voters_count(object),
       options: options,
       voted: voted?(params),
       emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"])
diff --git a/test/pleroma/web/mastodon_api/views/poll_view_test.exs b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
index b7e2f17ef..c655ca438 100644
--- a/test/pleroma/web/mastodon_api/views/poll_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
@@ -44,7 +44,7 @@ test "renders a poll" do
       ],
       voted: false,
       votes_count: 0,
-      voters_count: nil
+      voters_count: 0
     }
 
     result = PollView.render("show.json", %{object: object})

From f09bb814a96c71f24fcf6e403a25e90be9cc684e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:14:48 +0100
Subject: [PATCH 171/208] Changelog: Add info about poll view changes

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ee17d239..8c5a9f9dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
+- Polls now always return a `voters_count`, even if they are single-choice
 
 <details>
   <summary>API Changes</summary>

From 92d252f364ed421f2afcdd135507ced3554eb3f0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:20:09 +0100
Subject: [PATCH 172/208] Poll Schema: Update and fix.

---
 lib/pleroma/web/api_spec/schemas/poll.ex | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex
index c62096db0..0dfa60b97 100644
--- a/lib/pleroma/web/api_spec/schemas/poll.ex
+++ b/lib/pleroma/web/api_spec/schemas/poll.ex
@@ -28,8 +28,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
       },
       votes_count: %Schema{
         type: :integer,
-        nullable: true,
-        description: "How many votes have been received. Number, or null if `multiple` is false."
+        description: "How many votes have been received. Number."
+      },
+      voters_count: %Schema{
+        type: :integer,
+        description: "How many unique accounts have voted. Number."
       },
       voted: %Schema{
         type: :boolean,
@@ -61,7 +64,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
       expired: true,
       multiple: false,
       votes_count: 10,
-      voters_count: nil,
+      voters_count: 10,
       voted: true,
       own_votes: [
         1

From ca95cbe0b48b6c64e6e33addf79e4d212d5f9872 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 4 Nov 2020 16:40:12 +0400
Subject: [PATCH 173/208] Add `with_muted` param to ChatController.index/2

---
 docs/API/chats.md                                          | 4 ++++
 lib/pleroma/web/api_spec/operations/chat_operation.ex      | 6 +++++-
 lib/pleroma/web/api_spec/operations/timeline_operation.ex  | 2 +-
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 7 +++----
 .../web/pleroma_api/controllers/chat_controller_test.exs   | 7 +++++++
 5 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 9857aac67..f50144c86 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -116,6 +116,10 @@ The modified chat message
 This will return a list of chats that you have been involved in, sorted by their
 last update (so new chats will be at the top).
 
+Parameters:
+
+- with_muted: Include chats from muted users (boolean).
+
 Returned data:
 
 ```json
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 0dcfdb354..560b81f17 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.Chat
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
 
@@ -132,7 +133,10 @@ def index_operation do
       tags: ["chat"],
       summary: "Get a list of chats that you participated in",
       operationId: "ChatController.index",
-      parameters: pagination_params(),
+      parameters: [
+        Operation.parameter(:with_muted, :query, BooleanLike, "Include chats from muted users")
+        | pagination_params()
+      ],
       responses: %{
         200 => Operation.response("The chats of the user", "application/json", chats_response())
       },
diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 8e19bace7..1b5ad796f 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -159,7 +159,7 @@ defp local_param do
   end
 
   defp with_muted_param do
-    Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
+    Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end
 
   defp exclude_visibilities_param do
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 8fc70c15a..77564b342 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -138,11 +138,10 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do
     end
   end
 
-  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
+  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
     exclude_users =
-      user
-      |> User.blocked_users_ap_ids()
-      |> Enum.concat(User.muted_users_ap_ids(user))
+      User.blocked_users_ap_ids(user) ++
+        if params[:with_muted], do: [], else: User.muted_users_ap_ids(user)
 
     chats =
       user_id
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index b0498df2b..c1e6a8cc5 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -363,6 +363,13 @@ test "it does not return chats with users you muted", %{conn: conn, user: user}
         |> json_response_and_validate_schema(200)
 
       assert length(result) == 0
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats?with_muted=true")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 1
     end
 
     test "it returns all chats", %{conn: conn, user: user} do

From cb3cd3a761d96a08b1b55f5b277795822aa7e1d7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 15:24:10 +0100
Subject: [PATCH 174/208] TopicsTest: Small addition.

---
 test/pleroma/activity/ir/topics_test.exs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs
index 2e5655334..5e5c2f8da 100644
--- a/test/pleroma/activity/ir/topics_test.exs
+++ b/test/pleroma/activity/ir/topics_test.exs
@@ -104,6 +104,13 @@ test "non-local action produces public:remote topic", %{activity: activity} do
 
       assert Enum.member?(topics, "public:remote:lain.com")
     end
+
+    test "local action doesn't produce public:remote topic", %{activity: activity} do
+      activity = %{activity | local: true, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      refute Enum.member?(topics, "public:remote:lain.com")
+    end
   end
 
   describe "public visibility create events with attachments" do

From eb1e1e74945a55c37e4b0dc7cea1a7c926ff981a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 15:39:32 +0100
Subject: [PATCH 175/208] Changelog: Add info about federation status endpoint

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ee17d239..8951b4523 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
 - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
 
 </details>
 

From 6d850c46dc439e018323fb3580e708131247457b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 17:12:47 +0100
Subject: [PATCH 176/208] AdminEmail: Use AP id as user url.

---
 lib/pleroma/emails/admin_email.ex        | 10 +++-------
 test/pleroma/emails/admin_email_test.exs |  6 +++---
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex
index 8979db2f8..02274554f 100644
--- a/lib/pleroma/emails/admin_email.ex
+++ b/lib/pleroma/emails/admin_email.ex
@@ -18,10 +18,6 @@ defp instance_notify_email do
     Keyword.get(instance_config(), :notify_email, instance_config()[:email])
   end
 
-  defp user_url(user) do
-    Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, user.id)
-  end
-
   def test_email(mail_to \\ nil) do
     html_body = """
     <h3>Instance Test Email</h3>
@@ -69,8 +65,8 @@ def report(to, reporter, account, statuses, comment) do
       end
 
     html_body = """
-    <p>Reported by: <a href="#{user_url(reporter)}">#{reporter.nickname}</a></p>
-    <p>Reported Account: <a href="#{user_url(account)}">#{account.nickname}</a></p>
+    <p>Reported by: <a href="#{reporter.ap_id}">#{reporter.nickname}</a></p>
+    <p>Reported Account: <a href="#{account.ap_id}">#{account.nickname}</a></p>
     #{comment_html}
     #{statuses_html}
     <p>
@@ -86,7 +82,7 @@ def report(to, reporter, account, statuses, comment) do
 
   def new_unapproved_registration(to, account) do
     html_body = """
-    <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
+    <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
     <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """
diff --git a/test/pleroma/emails/admin_email_test.exs b/test/pleroma/emails/admin_email_test.exs
index 155057f3e..0da0699cc 100644
--- a/test/pleroma/emails/admin_email_test.exs
+++ b/test/pleroma/emails/admin_email_test.exs
@@ -19,8 +19,8 @@ test "build report email" do
       AdminEmail.report(to_user, reporter, account, [%{name: "Test", id: "12"}], "Test comment")
 
     status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, "12")
-    reporter_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, reporter.id)
-    account_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, account.id)
+    reporter_url = reporter.ap_id
+    account_url = account.ap_id
 
     assert res.to == [{to_user.name, to_user.email}]
     assert res.from == {config[:name], config[:notify_email]}
@@ -54,7 +54,7 @@ test "new unapproved registration email" do
 
     res = AdminEmail.new_unapproved_registration(to_user, account)
 
-    account_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, account.id)
+    account_url = account.ap_id
 
     assert res.to == [{to_user.name, to_user.email}]
     assert res.from == {config[:name], config[:notify_email]}

From 5ddf0be208b1a2703f24c907ac5a07cb8ea12f8c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 17:13:34 +0100
Subject: [PATCH 177/208] Changelog: Add info about admin email user link
 changes.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a82f7d6c..756939dde 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
 - Polls now always return a `voters_count`, even if they are single-choice
+- Admin Emails: The ap id is used as the user link in emails now.
 
 <details>
   <summary>API Changes</summary>

From 9b2ed1427725cd8c5e8d81e39cc5bbc73a2140bc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 5 Nov 2020 13:23:58 +0100
Subject: [PATCH 178/208] Docs: Add info about expiring mutes.

---
 docs/API/differences_in_mastoapi_responses.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md
index bb1000b0b..3075b6b86 100644
--- a/docs/API/differences_in_mastoapi_responses.md
+++ b/docs/API/differences_in_mastoapi_responses.md
@@ -255,6 +255,10 @@ There is an additional `user:pleroma_chat` stream. Incoming chat messages will m
 
 For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
 
+## User muting and thread muting
+
+Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.

From 60fe4a8393fe874284fbf2f46d598119a5207296 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:00:31 -0600
Subject: [PATCH 179/208] First draft of tips for optimizing BEAM

---
 docs/configuration/optimizing_beam.md | 60 +++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 docs/configuration/optimizing_beam.md

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
new file mode 100644
index 000000000..7a3cfda4b
--- /dev/null
+++ b/docs/configuration/optimizing_beam.md
@@ -0,0 +1,60 @@
+# Optimizing the BEAM
+
+Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by busy waiting or spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance.
+
+This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
+
+More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process and latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
+
+## VPS Provider Recommendations
+
+### Good
+
+* ????
+
+### Bad
+
+* AWS (known to use burst scheduling)
+
+
+## Example configurations
+
+Tuning the BEAM requires you provide a config file normally called [vm.args](http://erlang.org/doc/man/erl.html#emulator-flags). If you are using systemd to manage the service you can modify the unit file as such:
+
+`ExecStart=/usr/bin/elixir --erl '-args_file /opt/pleroma/config/vm.args' -S /usr/bin/mix phx.server`
+
+Check your OS documentation to adopt a similar strategy on other platforms.
+
+### Virtual Machine and/or few CPU cores
+
+Disable the busy-waiting
+
+```
++sbwt none
++sbwtdcpu none
++sbwtdio none
+```
+
+### Dedicated Hardware
+
+Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports
+
+```
++P 16777216
++Q 16777216
++K true
++A 128
++sbt db
++sbwt very_long
++swt very_low
++sub true
++Mulmbcs 32767
++Mumbcgs 1
++Musmbcs 2047
+```
+
+## Additional Reading
+
+* [WhatsApp: Scaling to Millions of Simultaneous Connections](https://www.erlang-factory.com/upload/presentations/558/efsf2012-whatsapp-scaling.pdf)
+* [Preemptive Scheduling and Spinlocks](https://www.uio.no/studier/emner/matnat/ifi/nedlagte-emner/INF3150/h03/annet/slides/preemptive.pdf)
+* [The Curious Case of BEAM CPU Usage](https://stressgrid.com/blog/beam_cpu_usage/)

From 9e90e49ad2d01744bd5385473f2ebd4b2a442e8a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:02:07 -0600
Subject: [PATCH 180/208] Grammar

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 7a3cfda4b..40fa7bfbf 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -4,7 +4,7 @@ Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly opt
 
 This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
 
-More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process and latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
+More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
 
 ## VPS Provider Recommendations
 

From da1862e1d3fea5488b80bfc46ea878468940238c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:04:13 -0600
Subject: [PATCH 181/208] Less confusing I hope

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 40fa7bfbf..00620b35d 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -1,6 +1,6 @@
 # Optimizing the BEAM
 
-Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by busy waiting or spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance.
+Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance.
 
 This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
 

From 620f1d723781ef39079add6326aec24112275253 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:12:13 -0600
Subject: [PATCH 182/208] More grammar fixes

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 00620b35d..fd89c4e99 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -1,6 +1,6 @@
 # Optimizing the BEAM
 
-Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance.
+Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application so it is immediately ready to execute another task. This can be observed with utilities like **top(1)** which will show consistently high CPU usage for the process. Switching between procesess is a rather expensive operation and also clears CPU caches further affecting latency and performance. The goal of busy waiting is to avoid this penalty.
 
 This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
 

From 4999549191b1ac7c50bb3a6398b0bad0f0957b73 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:15:21 -0600
Subject: [PATCH 183/208] Make it clearer the settings go into the vm.args file

---
 docs/configuration/optimizing_beam.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index fd89c4e99..64d35ad2c 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -29,6 +29,7 @@ Check your OS documentation to adopt a similar strategy on other platforms.
 
 Disable the busy-waiting
 
+**vm.args:**
 ```
 +sbwt none
 +sbwtdcpu none
@@ -39,6 +40,7 @@ Disable the busy-waiting
 
 Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports
 
+**vm.args:**
 ```
 +P 16777216
 +Q 16777216

From a9c1f83fd8b2793e6474b14d246e1ef362892467 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 6 Nov 2020 13:16:22 -0600
Subject: [PATCH 184/208] Markdown, you're drunk

---
 docs/configuration/optimizing_beam.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 64d35ad2c..de76086f7 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -30,6 +30,7 @@ Check your OS documentation to adopt a similar strategy on other platforms.
 Disable the busy-waiting
 
 **vm.args:**
+
 ```
 +sbwt none
 +sbwtdcpu none
@@ -41,6 +42,7 @@ Disable the busy-waiting
 Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports
 
 **vm.args:**
+
 ```
 +P 16777216
 +Q 16777216

From cc45c69fff515cb82c4902b67c8edce6b109c819 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 7 Nov 2020 22:09:28 +0300
Subject: [PATCH 185/208] Remove release_env

While taking a final look at instance.gen before releasing I noticed
that the release_env task outputs messages in broken english. Upon
further inspection it seems to have even more severe issues which, in
my opinion, warrant it's at least temporary removal:
- We do not explain what it actually does, anywhere. Neither the task
 docs nor instance.gen, nor installation instructions.
- It does not respect FHS on OTP releases (uses /opt/pleroma/config even
 though we store the config in /etc/pleroma/config.exs).
- It doesn't work on OTP releases, which is the main reason it exists.
Neither systemd nor openrc service files for OTP include it.
- It is not mentioned in install guides other than the ones for Debian
and OTP releases.
---
 .../CLI_tasks/release_environments.md         |  9 ---
 docs/installation/debian_based_en.md          |  1 -
 docs/installation/otp_en.md                   |  3 -
 installation/init.d/pleroma                   |  1 -
 installation/pleroma.service                  |  2 -
 lib/mix/tasks/pleroma/instance.ex             | 22 +-----
 lib/mix/tasks/pleroma/release_env.ex          | 76 -------------------
 test/mix/tasks/pleroma/instance_test.exs      | 11 +--
 test/mix/tasks/pleroma/release_env_test.exs   | 30 --------
 9 files changed, 2 insertions(+), 153 deletions(-)
 delete mode 100644 docs/administration/CLI_tasks/release_environments.md
 delete mode 100644 lib/mix/tasks/pleroma/release_env.ex
 delete mode 100644 test/mix/tasks/pleroma/release_env_test.exs

diff --git a/docs/administration/CLI_tasks/release_environments.md b/docs/administration/CLI_tasks/release_environments.md
deleted file mode 100644
index 36ab43864..000000000
--- a/docs/administration/CLI_tasks/release_environments.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# Generate release environment file
-
-```sh tab="OTP"
- ./bin/pleroma_ctl release_env gen
-```
-
-```sh tab="From Source"
-mix pleroma.release_env gen
-```
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index b9fc4e112..75ceb6595 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -182,7 +182,6 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
-* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 98360bcf7..63eda63ca 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -149,9 +149,6 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
-# Run the environment file generator.
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
-
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma
index e908cda1b..384536f7e 100755
--- a/installation/init.d/pleroma
+++ b/installation/init.d/pleroma
@@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
-export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 
diff --git a/installation/pleroma.service b/installation/pleroma.service
index 63e83ed6e..8338228d8 100644
--- a/installation/pleroma.service
+++ b/installation/pleroma.service
@@ -17,8 +17,6 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
-; Path to the environment file. the file contains RELEASE_COOKIE and etc 
-EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 1915aacd9..fc21ae062 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -36,9 +36,7 @@ def run(["gen" | rest]) do
           listen_port: :string,
           strip_uploads: :string,
           anonymize_uploads: :string,
-          dedupe_uploads: :string,
-          skip_release_env: :boolean,
-          release_env_file: :string
+          dedupe_uploads: :string
         ],
         aliases: [
           o: :output,
@@ -243,24 +241,6 @@ def run(["gen" | rest]) do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
-      if Keyword.get(options, :skip_release_env, false) do
-        shell_info("""
-        Release environment file is skip. Please generate the release env file before start.
-        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
-        """)
-      else
-        shell_info("Generation the environment file:")
-
-        release_env_args =
-          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
-            ["gen", "--path", path]
-          else
-            _ -> ["gen"]
-          end
-
-        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
-      end
-
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )
diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex
deleted file mode 100644
index 9da74ffcf..000000000
--- a/lib/mix/tasks/pleroma/release_env.ex
+++ /dev/null
@@ -1,76 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnv do
-  use Mix.Task
-  import Mix.Pleroma
-
-  @shortdoc "Generate Pleroma environment file."
-  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
-
-  def run(["gen" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [
-          force: :boolean,
-          path: :string
-        ],
-        aliases: [
-          p: :path,
-          f: :force
-        ]
-      )
-
-    file_path =
-      get_option(
-        options,
-        :path,
-        "Environment file path",
-        "./config/pleroma.env"
-      )
-
-    env_path = Path.expand(file_path)
-
-    proceed? =
-      if File.exists?(env_path) do
-        get_option(
-          options,
-          :force,
-          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
-          "n"
-        ) === "y"
-      else
-        true
-      end
-
-    if proceed? do
-      case do_generate(env_path) do
-        {:error, reason} ->
-          shell_error(
-            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
-          )
-
-        _ ->
-          shell_info("\nThe file generated: #{env_path}.\n")
-
-          shell_info("""
-          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
-            Example:
-              chmod 0444 #{file_path}
-              chattr +i #{file_path}
-          """)
-      end
-    else
-      shell_info("\nThe file is exist. #{env_path}.\n")
-    end
-  end
-
-  def do_generate(path) do
-    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
-
-    File.mkdir_p!(Path.dirname(path))
-    File.write(path, content)
-  end
-end
diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs
index fe69a2def..8a02710ee 100644
--- a/test/mix/tasks/pleroma/instance_test.exs
+++ b/test/mix/tasks/pleroma/instance_test.exs
@@ -5,8 +5,6 @@
 defmodule Mix.Tasks.Pleroma.InstanceTest do
   use ExUnit.Case
 
-  @release_env_file "./test/pleroma.test.env"
-
   setup do
     File.mkdir_p!(tmp_path())
 
@@ -18,8 +16,6 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do
         File.rm_rf(Path.join(static_dir, "robots.txt"))
       end
 
-      if File.exists?(@release_env_file), do: File.rm_rf(@release_env_file)
-
       Pleroma.Config.put([:instance, :static_dir], static_dir)
     end)
 
@@ -73,9 +69,7 @@ test "running gen" do
         "--dedupe-uploads",
         "n",
         "--anonymize-uploads",
-        "n",
-        "--release-env-file",
-        @release_env_file
+        "n"
       ])
     end
 
@@ -97,9 +91,6 @@ test "running gen" do
     assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]"
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
-    assert File.exists?(@release_env_file)
-
-    assert File.read!(@release_env_file) =~ ~r/^RELEASE_COOKIE=.*/
   end
 
   defp generated_setup_psql do
diff --git a/test/mix/tasks/pleroma/release_env_test.exs b/test/mix/tasks/pleroma/release_env_test.exs
deleted file mode 100644
index 519f1eba9..000000000
--- a/test/mix/tasks/pleroma/release_env_test.exs
+++ /dev/null
@@ -1,30 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do
-  use ExUnit.Case
-  import ExUnit.CaptureIO, only: [capture_io: 1]
-
-  @path "config/pleroma.test.env"
-
-  def do_clean do
-    if File.exists?(@path) do
-      File.rm_rf(@path)
-    end
-  end
-
-  setup do
-    do_clean()
-    on_exit(fn -> do_clean() end)
-    :ok
-  end
-
-  test "generate pleroma.env" do
-    assert capture_io(fn ->
-             Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"])
-           end) =~ "The file generated"
-
-    assert File.read!(@path) =~ "RELEASE_COOKIE="
-  end
-end

From abf2ec2bbe5226d6558ca1918779ee68df1cab84 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 8 Nov 2020 09:45:35 +0000
Subject: [PATCH 186/208] Update optimizing_beam.md

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index de76086f7..545392f8f 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -4,7 +4,7 @@ Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly opt
 
 This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
 
-More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
+More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
 
 ## VPS Provider Recommendations
 

From e4a21084f0017c873a307ec85d0ef5ea341ce026 Mon Sep 17 00:00:00 2001
From: Sean King <seanking2919@protonmail.com>
Date: Sun, 8 Nov 2020 16:16:20 -0700
Subject: [PATCH 187/208] Fix title on load of Pleroma HTML

---
 lib/pleroma/web/fallback/redirect_controller.ex | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex
index 6f759d559..712991c18 100644
--- a/lib/pleroma/web/fallback/redirect_controller.ex
+++ b/lib/pleroma/web/fallback/redirect_controller.ex
@@ -37,9 +37,11 @@ def redirector_with_meta(conn, params) do
 
     tags = build_tags(conn, params)
     preloads = preload_data(conn, params)
+    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
+      |> String.replace(~r/<title>.+?<\/title>/, title)
       |> String.replace("<!--server-generated-meta-->", tags <> preloads)
 
     conn
@@ -54,9 +56,11 @@ def redirector_with_preload(conn, %{"path" => ["pleroma", "admin"]}) do
   def redirector_with_preload(conn, params) do
     {:ok, index_content} = File.read(index_file_path())
     preloads = preload_data(conn, params)
+    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
+      |> String.replace(~r/<title>.+?<\/title>/, title)
       |> String.replace("<!--server-generated-meta-->", preloads)
 
     conn

From 0c68b9ac137e98867cf8aacfef1f264412cc7b3e Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 10 Nov 2020 10:44:22 +0300
Subject: [PATCH 188/208] escaping summary and other fields in xml templates

---
 lib/pleroma/web/feed/feed_view.ex             |  2 +-
 .../templates/feed/feed/_activity.atom.eex    |  2 +-
 .../web/templates/feed/feed/_activity.rss.eex |  2 +-
 .../pleroma/web/feed/user_controller_test.exs | 77 +++++++------------
 4 files changed, 29 insertions(+), 54 deletions(-)

diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
index 1ae03e7e2..56c024617 100644
--- a/lib/pleroma/web/feed/feed_view.ex
+++ b/lib/pleroma/web/feed/feed_view.ex
@@ -83,7 +83,7 @@ def activity_content(%{"content" => content}) do
 
   def activity_content(_), do: ""
 
-  def activity_context(activity), do: activity.data["context"]
+  def activity_context(activity), do: escape(activity.data["context"])
 
   def attachment_href(attachment) do
     attachment["url"]
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
index 78350f2aa..3fd150c4e 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
@@ -12,7 +12,7 @@
   <link href="<%= activity_context(@activity) %>" rel="ostatus:conversation"/>
 
   <%= if @data["summary"] do %>
-    <summary><%= @data["summary"] %></summary>
+    <summary><%= escape(@data["summary"]) %></summary>
   <% end %>
 
   <%= if @activity.local do %>
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
index a304a16af..42960de7d 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
@@ -12,7 +12,7 @@
   <link rel="ostatus:conversation"><%= activity_context(@activity) %></link>
 
   <%= if @data["summary"] do %>
-    <description><%= @data["summary"] %></description>
+    <description><%= escape(@data["summary"]) %></description>
   <% end %>
 
   <%= if @activity.local do %>
diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs
index eabfe3a63..16f002717 100644
--- a/test/pleroma/web/feed/user_controller_test.exs
+++ b/test/pleroma/web/feed/user_controller_test.exs
@@ -12,16 +12,17 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.Feed.FeedView
 
   setup do: clear_config([:static_fe, :enabled], false)
 
   describe "feed" do
     setup do: clear_config([:feed])
 
-    test "gets an atom feed", %{conn: conn} do
+    setup do
       Config.put(
         [:feed, :post_title],
-        %{max_length: 10, omission: "..."}
+        %{max_length: 15, omission: "..."}
       )
 
       activity = insert(:note_activity)
@@ -29,7 +30,8 @@ test "gets an atom feed", %{conn: conn} do
       note =
         insert(:note,
           data: %{
-            "content" => "This is :moominmamma: note ",
+            "content" => "This & this is :moominmamma: note ",
+            "source" => "This & this is :moominmamma: note ",
             "attachment" => [
               %{
                 "url" => [
@@ -37,7 +39,9 @@ test "gets an atom feed", %{conn: conn} do
                 ]
               }
             ],
-            "inReplyTo" => activity.data["id"]
+            "inReplyTo" => activity.data["id"],
+            "context" => "2hu & as",
+            "summary" => "2hu & as"
           }
         )
 
@@ -48,7 +52,7 @@ test "gets an atom feed", %{conn: conn} do
         insert(:note,
           user: user,
           data: %{
-            "content" => "42 This is :moominmamma: note ",
+            "content" => "42 & This is :moominmamma: note ",
             "inReplyTo" => activity.data["id"]
           }
         )
@@ -56,6 +60,10 @@ test "gets an atom feed", %{conn: conn} do
       note_activity2 = insert(:note_activity, note: note2)
       object = Object.normalize(note_activity)
 
+      [user: user, object: object, max_id: note_activity2.id]
+    end
+
+    test "gets an atom feed", %{conn: conn, user: user, object: object, max_id: max_id} do
       resp =
         conn
         |> put_req_header("accept", "application/atom+xml")
@@ -67,13 +75,15 @@ test "gets an atom feed", %{conn: conn} do
         |> SweetXml.parse()
         |> SweetXml.xpath(~x"//entry/title/text()"l)
 
-      assert activity_titles == ['42 This...', 'This is...']
-      assert resp =~ object.data["content"]
+      assert activity_titles == ['42 &amp; Thi...', 'This &amp; t...']
+      assert resp =~ FeedView.escape(object.data["content"])
+      assert resp =~ FeedView.escape(object.data["summary"])
+      assert resp =~ FeedView.escape(object.data["context"])
 
       resp =
         conn
         |> put_req_header("accept", "application/atom+xml")
-        |> get("/users/#{user.nickname}/feed", %{"max_id" => note_activity2.id})
+        |> get("/users/#{user.nickname}/feed", %{"max_id" => max_id})
         |> response(200)
 
       activity_titles =
@@ -81,47 +91,10 @@ test "gets an atom feed", %{conn: conn} do
         |> SweetXml.parse()
         |> SweetXml.xpath(~x"//entry/title/text()"l)
 
-      assert activity_titles == ['This is...']
+      assert activity_titles == ['This &amp; t...']
     end
 
-    test "gets a rss feed", %{conn: conn} do
-      Pleroma.Config.put(
-        [:feed, :post_title],
-        %{max_length: 10, omission: "..."}
-      )
-
-      activity = insert(:note_activity)
-
-      note =
-        insert(:note,
-          data: %{
-            "content" => "This is :moominmamma: note ",
-            "attachment" => [
-              %{
-                "url" => [
-                  %{"mediaType" => "image/png", "href" => "https://pleroma.gov/image.png"}
-                ]
-              }
-            ],
-            "inReplyTo" => activity.data["id"]
-          }
-        )
-
-      note_activity = insert(:note_activity, note: note)
-      user = User.get_cached_by_ap_id(note_activity.data["actor"])
-
-      note2 =
-        insert(:note,
-          user: user,
-          data: %{
-            "content" => "42 This is :moominmamma: note ",
-            "inReplyTo" => activity.data["id"]
-          }
-        )
-
-      note_activity2 = insert(:note_activity, note: note2)
-      object = Object.normalize(note_activity)
-
+    test "gets a rss feed", %{conn: conn, user: user, object: object, max_id: max_id} do
       resp =
         conn
         |> put_req_header("accept", "application/rss+xml")
@@ -133,13 +106,15 @@ test "gets a rss feed", %{conn: conn} do
         |> SweetXml.parse()
         |> SweetXml.xpath(~x"//item/title/text()"l)
 
-      assert activity_titles == ['42 This...', 'This is...']
-      assert resp =~ object.data["content"]
+      assert activity_titles == ['42 &amp; Thi...', 'This &amp; t...']
+      assert resp =~ FeedView.escape(object.data["content"])
+      assert resp =~ FeedView.escape(object.data["summary"])
+      assert resp =~ FeedView.escape(object.data["context"])
 
       resp =
         conn
         |> put_req_header("accept", "application/rss+xml")
-        |> get("/users/#{user.nickname}/feed.rss", %{"max_id" => note_activity2.id})
+        |> get("/users/#{user.nickname}/feed.rss", %{"max_id" => max_id})
         |> response(200)
 
       activity_titles =
@@ -147,7 +122,7 @@ test "gets a rss feed", %{conn: conn} do
         |> SweetXml.parse()
         |> SweetXml.xpath(~x"//item/title/text()"l)
 
-      assert activity_titles == ['This is...']
+      assert activity_titles == ['This &amp; t...']
     end
 
     test "returns 404 for a missing feed", %{conn: conn} do

From 485697d96c6a45127af22b9a5f357c3802dba73c Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 10 Nov 2020 19:18:53 +0300
Subject: [PATCH 189/208] config descriptions for custom MRF policies

---
 config/description.exs                        | 302 ------------------
 lib/pleroma/docs/json.ex                      |   6 +-
 lib/pleroma/web/activity_pub/mrf.ex           |  58 +++-
 .../mrf/activity_expiration_policy.ex         |  18 ++
 .../web/activity_pub/mrf/hellthread_policy.ex |  27 ++
 .../web/activity_pub/mrf/keyword_policy.ex    |  42 +++
 .../web/activity_pub/mrf/mention_policy.ex    |  18 ++
 .../web/activity_pub/mrf/normalize_markup.ex  |  19 ++
 .../web/activity_pub/mrf/object_age_policy.ex |  28 ++
 .../web/activity_pub/mrf/reject_non_public.ex |  23 ++
 .../web/activity_pub/mrf/simple_policy.ex     |  74 +++++
 .../web/activity_pub/mrf/subchain_policy.ex   |  24 ++
 .../mrf/user_allow_list_policy.ex             |  14 +
 .../web/activity_pub/mrf/vocabulary_policy.ex |  28 ++
 test/fixtures/modules/good_mrf.ex             |  19 ++
 test/pleroma/web/activity_pub/mrf_test.exs    |  17 +
 16 files changed, 412 insertions(+), 305 deletions(-)
 create mode 100644 test/fixtures/modules/good_mrf.ex

diff --git a/config/description.exs b/config/description.exs
index 0b651696b..69072312c 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1,5 +1,4 @@
 use Mix.Config
-alias Pleroma.Docs.Generator
 
 websocket_config = [
   path: "/websocket",
@@ -1589,264 +1588,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_simple,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy",
-    label: "MRF Simple",
-    type: :group,
-    description: "Simple ingress policies",
-    children: [
-      %{
-        key: :media_removal,
-        type: {:list, :string},
-        description: "List of instances to strip media attachments from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :media_nsfw,
-        label: "Media NSFW",
-        type: {:list, :string},
-        description: "List of instances to tag all media as NSFW (sensitive) from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description:
-          "List of instances to remove from the Federated (aka The Whole Known Network) Timeline",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description: "List of instances to reject activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description: "List of instances to only accept activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :followers_only,
-        type: {:list, :string},
-        description: "Force posts from the given instances to be visible by followers only",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :report_removal,
-        type: {:list, :string},
-        description: "List of instances to reject reports from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :avatar_removal,
-        type: {:list, :string},
-        description: "List of instances to strip avatars from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :banner_removal,
-        type: {:list, :string},
-        description: "List of instances to strip banners from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject_deletes,
-        type: {:list, :string},
-        description: "List of instances to reject deletions from",
-        suggestions: ["example.com", "*.example.com"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_activity_expiration,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
-    label: "MRF Activity Expiration Policy",
-    type: :group,
-    description: "Adds automatic expiration to all local activities",
-    children: [
-      %{
-        key: :days,
-        type: :integer,
-        description: "Default global expiration time for all local activities (in days)",
-        suggestions: [90, 365]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_subchain,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
-    label: "MRF Subchain",
-    type: :group,
-    description:
-      "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
-        " All criteria are configured as a map of regular expressions to lists of policy modules.",
-    children: [
-      %{
-        key: :match_actor,
-        type: {:map, {:list, :string}},
-        description: "Matches a series of regular expressions against the actor field",
-        suggestions: [
-          %{
-            ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
-          }
-        ]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_rejectnonpublic,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic",
-    description: "RejectNonPublic drops posts with non-public visibility settings.",
-    label: "MRF Reject Non Public",
-    type: :group,
-    children: [
-      %{
-        key: :allow_followersonly,
-        label: "Allow followers-only",
-        type: :boolean,
-        description: "Whether to allow followers-only posts"
-      },
-      %{
-        key: :allow_direct,
-        type: :boolean,
-        description: "Whether to allow direct messages"
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_hellthread,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
-    label: "MRF Hellthread",
-    type: :group,
-    description: "Block messages with excessive user mentions",
-    children: [
-      %{
-        key: :delist_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the message gets removed from timelines and" <>
-            "disables notifications. Set to 0 to disable.",
-        suggestions: [10]
-      },
-      %{
-        key: :reject_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
-        suggestions: [20]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_keyword,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
-    label: "MRF Keyword",
-    type: :group,
-    description:
-      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
-    children: [
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description: """
-          A list of patterns which result in message being rejected.
-
-          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-        """,
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description: """
-          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
-
-          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-        """,
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :replace,
-        type: {:list, :tuple},
-        description: """
-          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-
-          **Replacement**: a string. Leaving the field empty is permitted.
-        """
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_mention,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
-    label: "MRF Mention",
-    type: :group,
-    description: "Block messages which mention a specific user",
-    children: [
-      %{
-        key: :actors,
-        type: {:list, :string},
-        description: "A list of actors for which any post mentioning them will be dropped",
-        suggestions: ["actor1", "actor2"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_vocabulary,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
-    label: "MRF Vocabulary",
-    type: :group,
-    description: "Filter messages which belong to certain activity vocabularies",
-    children: [
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      }
-    ]
-  },
-  # %{
-  #   group: :pleroma,
-  #   key: :mrf_user_allowlist,
-  #   tab: :mrf,
-  #   related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
-  #   type: :map,
-  #   description:
-  #     "The keys in this section are the domain names that the policy should apply to." <>
-  #       " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
-  #     suggestions: [
-  #       %{"example.org" => ["https://example.org/users/admin"]}
-  #     ]
-  #   ]
-  # },
   %{
     group: :pleroma,
     key: :media_proxy,
@@ -3159,22 +2900,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_normalize_markup,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
-    label: "MRF Normalize Markup",
-    description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
-    type: :group,
-    children: [
-      %{
-        key: :scrub_policy,
-        type: :module,
-        suggestions: [Pleroma.HTML.Scrubber.Default]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.User,
@@ -3364,33 +3089,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_object_age,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
-    label: "MRF Object Age",
-    type: :group,
-    description:
-      "Rejects or delists posts based on their timestamp deviance from your server's clock.",
-    children: [
-      %{
-        key: :threshold,
-        type: :integer,
-        description: "Required age (in seconds) of a post before actions are taken.",
-        suggestions: [172_800]
-      },
-      %{
-        key: :actions,
-        type: {:list, :atom},
-        description:
-          "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
-            "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
-            "`:reject` rejects the message entirely",
-        suggestions: [:delist, :strip_followers, :reject]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: :modules,
diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex
index 13618b509..a583e2a5b 100644
--- a/lib/pleroma/docs/json.ex
+++ b/lib/pleroma/docs/json.ex
@@ -11,7 +11,11 @@ defmodule Pleroma.Docs.JSON do
 
   @spec compile :: :ok
   def compile do
-    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions))
+    descriptions =
+      Pleroma.Web.ActivityPub.MRF.config_descriptions()
+      |> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end)
+
+    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions))
   end
 
   @spec compiled_descriptions :: Map.t()
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 5e5361082..656e4c7ca 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -3,7 +3,26 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF do
+  require Logger
+
+  @default_description %{
+    label: "",
+    description: "",
+    children: []
+  }
+
+  @required_description_keys [:key, :related_policy]
+
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
+  @callback describe() :: {:ok | :error, Map.t()}
+  @callback config_description() :: %{
+              optional(:children) => [map()],
+              key: atom(),
+              related_policy: String.t(),
+              label: String.t(),
+              description: String.t()
+            }
+  @optional_callbacks config_description: 0
 
   def filter(policies, %{} = message) do
     policies
@@ -51,8 +70,6 @@ def subdomain_match?(domains, host) do
     Enum.any?(domains, fn domain -> Regex.match?(domain, host) end)
   end
 
-  @callback describe() :: {:ok | :error, Map.t()}
-
   def describe(policies) do
     {:ok, policy_configs} =
       policies
@@ -82,4 +99,41 @@ def describe(policies) do
   end
 
   def describe, do: get_policies() |> describe()
+
+  def config_descriptions do
+    Pleroma.Web.ActivityPub.MRF
+    |> Pleroma.Docs.Generator.list_behaviour_implementations()
+    |> config_descriptions()
+  end
+
+  def config_descriptions(policies) do
+    Enum.reduce(policies, [], fn policy, acc ->
+      if function_exported?(policy, :config_description, 0) do
+        description =
+          @default_description
+          |> Map.merge(policy.config_description)
+          |> Map.put(:group, :pleroma)
+          |> Map.put(:tab, :mrf)
+          |> Map.put(:type, :group)
+
+        if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do
+          [description | acc]
+        else
+          Logger.warn(
+            "#{policy} config description doesn't have one or all required keys #{
+              inspect(@required_description_keys)
+            }"
+          )
+
+          acc
+        end
+      else
+        Logger.info(
+          "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method."
+        )
+
+        acc
+      end
+    end)
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index bee47b4ed..655a2ced0 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -40,4 +40,22 @@ defp maybe_add_expiration(activity) do
       _ -> Map.put(activity, "expires_at", expires_at)
     end
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
index 9ba07b4e3..3fd5c1e0a 100644
--- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
@@ -97,4 +97,31 @@ def filter(message), do: {:ok, message}
   @impl true
   def describe,
     do: {:ok, %{mrf_hellthread: Pleroma.Config.get(:mrf_hellthread) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_hellthread,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
+      label: "MRF Hellthread",
+      description: "Block messages with excessive user mentions",
+      children: [
+        %{
+          key: :delist_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the message gets removed from timelines and" <>
+              "disables notifications. Set to 0 to disable.",
+          suggestions: [10]
+        },
+        %{
+          key: :reject_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
+          suggestions: [20]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
index db66cfa3e..ded0fe7f2 100644
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@@ -126,4 +126,46 @@ def describe do
 
     {:ok, %{mrf_keyword: mrf_keyword}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_keyword,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
+      label: "MRF Keyword",
+      description:
+        "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
+      children: [
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being rejected.
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :replace,
+          type: {:list, :tuple},
+          description: """
+            **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+            **Replacement**: a string. Leaving the field empty is permitted.
+          """
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
index 7910ca131..9c096712a 100644
--- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
@@ -25,4 +25,22 @@ def filter(message), do: {:ok, message}
 
   @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_mention,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
+      label: "MRF Mention",
+      description: "Block messages which mention a specific user",
+      children: [
+        %{
+          key: :actors,
+          type: {:list, :string},
+          description: "A list of actors for which any post mentioning them will be dropped",
+          suggestions: ["actor1", "actor2"]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
index 7abae37ae..e00575c2a 100644
--- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
+++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @impl true
   def filter(%{"type" => "Create", "object" => child_object} = object) do
     scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
 
@@ -22,5 +23,23 @@ def filter(%{"type" => "Create", "object" => child_object} = object) do
 
   def filter(object), do: {:ok, object}
 
+  @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_normalize_markup,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
+      label: "MRF Normalize Markup",
+      description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
+      children: [
+        %{
+          key: :scrub_policy,
+          type: :module,
+          suggestions: [Pleroma.HTML.Scrubber.Default]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
index d45d2d7e3..eb0481f20 100644
--- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
@@ -106,4 +106,32 @@ def describe do
 
     {:ok, %{mrf_object_age: mrf_object_age}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_object_age,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
+      label: "MRF Object Age",
+      description:
+        "Rejects or delists posts based on their timestamp deviance from your server's clock.",
+      children: [
+        %{
+          key: :threshold,
+          type: :integer,
+          description: "Required age (in seconds) of a post before actions are taken.",
+          suggestions: [172_800]
+        },
+        %{
+          key: :actions,
+          type: {:list, :atom},
+          description:
+            "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
+              "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
+              "`:reject` rejects the message entirely",
+          suggestions: [:delist, :strip_followers, :reject]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
index 0b9ed2224..cd7665e31 100644
--- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
+++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
@@ -48,4 +48,27 @@ def filter(object), do: {:ok, object}
   @impl true
   def describe,
     do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_rejectnonpublic,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic",
+      description: "RejectNonPublic drops posts with non-public visibility settings.",
+      label: "MRF Reject Non Public",
+      children: [
+        %{
+          key: :allow_followersonly,
+          label: "Allow followers-only",
+          type: :boolean,
+          description: "Whether to allow followers-only posts"
+        },
+        %{
+          key: :allow_direct,
+          type: :boolean,
+          description: "Whether to allow direct messages"
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 161177727..6cd91826d 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -244,4 +244,78 @@ def describe do
 
     {:ok, %{mrf_simple: mrf_simple}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_simple,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy",
+      label: "MRF Simple",
+      description: "Simple ingress policies",
+      children: [
+        %{
+          key: :media_removal,
+          type: {:list, :string},
+          description: "List of instances to strip media attachments from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :media_nsfw,
+          label: "Media NSFW",
+          type: {:list, :string},
+          description: "List of instances to tag all media as NSFW (sensitive) from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description:
+            "List of instances to remove from the Federated (aka The Whole Known Network) Timeline",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: "List of instances to reject activities from (except deletes)",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :accept,
+          type: {:list, :string},
+          description: "List of instances to only accept activities from (except deletes)",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :followers_only,
+          type: {:list, :string},
+          description: "Force posts from the given instances to be visible by followers only",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :report_removal,
+          type: {:list, :string},
+          description: "List of instances to reject reports from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :avatar_removal,
+          type: {:list, :string},
+          description: "List of instances to strip avatars from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :banner_removal,
+          type: {:list, :string},
+          description: "List of instances to strip banners from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :reject_deletes,
+          type: {:list, :string},
+          description: "List of instances to reject deletions from",
+          suggestions: ["example.com", "*.example.com"]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
index 048052da6..2ec45260a 100644
--- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
@@ -39,4 +39,28 @@ def filter(message), do: {:ok, message}
 
   @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_subchain,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
+      label: "MRF Subchain",
+      description:
+        "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
+          " All criteria are configured as a map of regular expressions to lists of policy modules.",
+      children: [
+        %{
+          key: :match_actor,
+          type: {:map, {:list, :string}},
+          description: "Matches a series of regular expressions against the actor field",
+          suggestions: [
+            %{
+              ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
+            }
+          ]
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index 1a28f2ba2..885bcca6f 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -41,4 +41,18 @@ def describe do
 
     {:ok, %{mrf_user_allowlist: mrf_user_allowlist}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_user_allowlist,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
+      description:
+        "The keys in this section are the domain names that the policy should apply to." <>
+          " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
+      suggestions: [
+        %{"example.org" => ["https://example.org/users/admin"]}
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
index a6c545570..f325cb680 100644
--- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @impl true
   def filter(%{"type" => "Undo", "object" => child_message} = message) do
     with {:ok, _} <- filter(child_message) do
       {:ok, message}
@@ -36,6 +37,33 @@ def filter(%{"type" => message_type} = message) do
 
   def filter(message), do: {:ok, message}
 
+  @impl true
   def describe,
     do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_vocabulary,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
+      label: "MRF Vocabulary",
+      description: "Filter messages which belong to certain activity vocabularies",
+      children: [
+        %{
+          key: :accept,
+          type: {:list, :string},
+          description:
+            "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
+          suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
+        },
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description:
+            "A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
+          suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
+        }
+      ]
+    }
+  end
 end
diff --git a/test/fixtures/modules/good_mrf.ex b/test/fixtures/modules/good_mrf.ex
new file mode 100644
index 000000000..39d0f14ec
--- /dev/null
+++ b/test/fixtures/modules/good_mrf.ex
@@ -0,0 +1,19 @@
+defmodule Fixtures.Modules.GoodMRF do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(a), do: {:ok, a}
+
+  @impl true
+  def describe, do: %{}
+
+  @impl true
+  def config_description do
+    %{
+      key: :good_mrf,
+      related_policy: "Fixtures.Modules.GoodMRF",
+      label: "Good MRF",
+      description: "Some description"
+    }
+  end
+end
diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs
index e8cdde2e1..17aef2e16 100644
--- a/test/pleroma/web/activity_pub/mrf_test.exs
+++ b/test/pleroma/web/activity_pub/mrf_test.exs
@@ -87,4 +87,21 @@ test "it works as expected with mock policy" do
       {:ok, ^expected} = MRF.describe()
     end
   end
+
+  test "config_descriptions/0" do
+    descriptions = MRF.config_descriptions()
+
+    good_mrf = Enum.find(descriptions, fn %{key: key} -> key == :good_mrf end)
+
+    assert good_mrf == %{
+             key: :good_mrf,
+             related_policy: "Fixtures.Modules.GoodMRF",
+             label: "Good MRF",
+             description: "Some description",
+             children: [],
+             group: :pleroma,
+             tab: :mrf,
+             type: :group
+           }
+  end
 end

From 2933658446f4172bb77b4d294d032fa4e2d93237 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 10 Nov 2020 16:44:00 +0000
Subject: [PATCH 190/208] Apply 1 suggestion(s) to 1 file(s)

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 545392f8f..1753620c9 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -27,7 +27,7 @@ Check your OS documentation to adopt a similar strategy on other platforms.
 
 ### Virtual Machine and/or few CPU cores
 
-Disable the busy-waiting
+Disable the busy-waiting. This should generally only be done if you're on a platform that does burst scheduling, like AWS.
 
 **vm.args:**
 

From 952a8c213e37b38c48890813f80e8792cb0cebe2 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 10 Nov 2020 16:44:08 +0000
Subject: [PATCH 191/208] Apply 1 suggestion(s) to 1 file(s)

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index 1753620c9..ecfae86d9 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -39,7 +39,7 @@ Disable the busy-waiting. This should generally only be done if you're on a plat
 
 ### Dedicated Hardware
 
-Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports
+Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports. You can use this if you run on dedicated hardware, but it is not necessary.
 
 **vm.args:**
 

From 776067a9a32fd2b536c6782a0aa7c2dfd8444280 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 10 Nov 2020 16:44:17 +0000
Subject: [PATCH 192/208] Apply 1 suggestion(s) to 1 file(s)

---
 docs/configuration/optimizing_beam.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index ecfae86d9..a79d188ca 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -6,6 +6,8 @@ This strategy is very successful in making a performant and responsive applicati
 
 More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
 
+Please only change these settings if you are experiencing issues or really know what you are doing. In general, there's no need to change these settings.
+
 ## VPS Provider Recommendations
 
 ### Good

From 7681b4c5cd683bcc8c91b1d296261e7e2b11fd88 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 10 Nov 2020 16:44:23 +0000
Subject: [PATCH 193/208] Apply 1 suggestion(s) to 1 file(s)

---
 docs/configuration/optimizing_beam.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md
index a79d188ca..e336bd36c 100644
--- a/docs/configuration/optimizing_beam.md
+++ b/docs/configuration/optimizing_beam.md
@@ -12,7 +12,7 @@ Please only change these settings if you are experiencing issues or really know
 
 ### Good
 
-* ????
+* Hetzner Cloud
 
 ### Bad
 

From efc27f64643f24e97c604efa02c15bfe3210cf3b Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 11 Nov 2020 10:10:57 +0300
Subject: [PATCH 194/208] fix for adminFE

- revert UserAllowPolicy description
- MRF descriptions order
---
 config/description.exs                        | 34 ---------------
 lib/pleroma/web/activity_pub/mrf.ex           | 42 +++++++++++++++++--
 .../mrf/user_allow_list_policy.ex             | 32 ++++++++------
 3 files changed, 58 insertions(+), 50 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 69072312c..0552b37e0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1554,40 +1554,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf,
-    tab: :mrf,
-    label: "MRF",
-    type: :group,
-    description: "General MRF settings",
-    children: [
-      %{
-        key: :policies,
-        type: [:module, {:list, :module}],
-        description:
-          "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
-        suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
-      },
-      %{
-        key: :transparency,
-        label: "MRF transparency",
-        type: :boolean,
-        description:
-          "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
-      },
-      %{
-        key: :transparency_exclusions,
-        label: "MRF transparency exclusions",
-        type: {:list, :string},
-        description:
-          "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
-        suggestions: [
-          "exclusion.com"
-        ]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: :media_proxy,
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 656e4c7ca..ce125a696 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -5,10 +5,46 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   require Logger
 
+  @mrf_config_descriptions [
+    %{
+      group: :pleroma,
+      key: :mrf,
+      tab: :mrf,
+      label: "MRF",
+      type: :group,
+      description: "General MRF settings",
+      children: [
+        %{
+          key: :policies,
+          type: [:module, {:list, :module}],
+          description:
+            "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
+          suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
+        },
+        %{
+          key: :transparency,
+          label: "MRF transparency",
+          type: :boolean,
+          description:
+            "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
+        },
+        %{
+          key: :transparency_exclusions,
+          label: "MRF transparency exclusions",
+          type: {:list, :string},
+          description:
+            "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
+          suggestions: [
+            "exclusion.com"
+          ]
+        }
+      ]
+    }
+  ]
+
   @default_description %{
     label: "",
-    description: "",
-    children: []
+    description: ""
   }
 
   @required_description_keys [:key, :related_policy]
@@ -107,7 +143,7 @@ def config_descriptions do
   end
 
   def config_descriptions(policies) do
-    Enum.reduce(policies, [], fn policy, acc ->
+    Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc ->
       if function_exported?(policy, :config_description, 0) do
         description =
           @default_description
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index 885bcca6f..f2859abde 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -42,17 +42,23 @@ def describe do
     {:ok, %{mrf_user_allowlist: mrf_user_allowlist}}
   end
 
-  @impl true
-  def config_description do
-    %{
-      key: :mrf_user_allowlist,
-      related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
-      description:
-        "The keys in this section are the domain names that the policy should apply to." <>
-          " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
-      suggestions: [
-        %{"example.org" => ["https://example.org/users/admin"]}
-      ]
-    }
-  end
+  # TODO: change way of getting settings on `lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex:18` to use `hosts` subkey
+  # @impl true
+  # def config_description do
+  #   %{
+  #     key: :mrf_user_allowlist,
+  #     related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
+  #     description: "Accept-list of users from specified instances",
+  #     children: [
+  #       %{
+  #         key: :hosts,
+  #         type: :map,
+  #         description:
+  #           "The keys in this section are the domain names that the policy should apply to." <>
+  #             " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
+  #         suggestions: [%{"example.org" => ["https://example.org/users/admin"]}]
+  #       }
+  #     ]
+  #   }
+  # end
 end

From f97f24b067b6d0205f093b04aeb08c3d56faa7b4 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 11 Nov 2020 10:48:03 +0300
Subject: [PATCH 195/208] making credo happy and test fix

---
 lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex | 3 ++-
 test/pleroma/web/activity_pub/mrf_test.exs                 | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index f2859abde..e9d0d0503 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -55,7 +55,8 @@ def describe do
   #         type: :map,
   #         description:
   #           "The keys in this section are the domain names that the policy should apply to." <>
-  #             " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
+  #             " Each key should be assigned a list of users that should be allowed " <>
+  #             "through by their ActivityPub ID",
   #         suggestions: [%{"example.org" => ["https://example.org/users/admin"]}]
   #       }
   #     ]
diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs
index 17aef2e16..44a9cf086 100644
--- a/test/pleroma/web/activity_pub/mrf_test.exs
+++ b/test/pleroma/web/activity_pub/mrf_test.exs
@@ -98,7 +98,6 @@ test "config_descriptions/0" do
              related_policy: "Fixtures.Modules.GoodMRF",
              label: "Good MRF",
              description: "Some description",
-             children: [],
              group: :pleroma,
              tab: :mrf,
              type: :group

From 8da9f919f82ac45c4519910a7e24cac2b797061f Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 11 Nov 2020 18:49:15 +0300
Subject: [PATCH 196/208] little changes for MRF config descriptions

- log level reduction, if policy doesn't implement config_description method
- docs in dev.md
---
 docs/dev.md                         | 23 +++++++++++++++++++++++
 lib/pleroma/web/activity_pub/mrf.ex |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/docs/dev.md b/docs/dev.md
index 22e0691f1..aa89a941f 100644
--- a/docs/dev.md
+++ b/docs/dev.md
@@ -21,3 +21,26 @@ This document contains notes and guidelines for Pleroma developers.
 ## Auth-related configuration, OAuth consumer mode etc.
 
 See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication).
+
+## MRF policies descriptions
+
+If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns map with special structure.
+
+Example:
+
+```elixir
+%{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+```
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index ce125a696..6e73b2f22 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -164,7 +164,7 @@ def config_descriptions(policies) do
           acc
         end
       else
-        Logger.info(
+        Logger.debug(
           "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method."
         )
 

From 631def2df228ceb0ec8921c63b90867758e0c308 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 11 Nov 2020 17:10:59 +0100
Subject: [PATCH 197/208] RedirectController: Don't replace title, but inject
 into the meta

---
 lib/pleroma/web/fallback/redirect_controller.ex |  6 ++----
 test/pleroma/web/fallback_test.exs              | 16 +++++++++++++++-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex
index 712991c18..1ac1319f8 100644
--- a/lib/pleroma/web/fallback/redirect_controller.ex
+++ b/lib/pleroma/web/fallback/redirect_controller.ex
@@ -41,8 +41,7 @@ def redirector_with_meta(conn, params) do
 
     response =
       index_content
-      |> String.replace(~r/<title>.+?<\/title>/, title)
-      |> String.replace("<!--server-generated-meta-->", tags <> preloads)
+      |> String.replace("<!--server-generated-meta-->", tags <> preloads <> title)
 
     conn
     |> put_resp_content_type("text/html")
@@ -60,8 +59,7 @@ def redirector_with_preload(conn, params) do
 
     response =
       index_content
-      |> String.replace(~r/<title>.+?<\/title>/, title)
-      |> String.replace("<!--server-generated-meta-->", preloads)
+      |> String.replace("<!--server-generated-meta-->", preloads <> title)
 
     conn
     |> put_resp_content_type("text/html")
diff --git a/test/pleroma/web/fallback_test.exs b/test/pleroma/web/fallback_test.exs
index a65865860..46c7bad1c 100644
--- a/test/pleroma/web/fallback_test.exs
+++ b/test/pleroma/web/fallback_test.exs
@@ -20,15 +20,26 @@ test "GET /*path", %{conn: conn} do
     end
   end
 
+  test "GET /*path adds a title", %{conn: conn} do
+    clear_config([:instance, :name], "a cool title")
+
+    assert conn
+           |> get("/")
+           |> html_response(200) =~ "<title>a cool title</title>"
+  end
+
   describe "preloaded data and metadata attached to" do
     test "GET /:maybe_nickname_or_id", %{conn: conn} do
+      clear_config([:instance, :name], "a cool title")
+
       user = insert(:user)
       user_missing = get(conn, "/foo")
       user_present = get(conn, "/#{user.nickname}")
 
-      assert(html_response(user_missing, 200) =~ "<!--server-generated-meta-->")
+      assert html_response(user_missing, 200) =~ "<!--server-generated-meta-->"
       refute html_response(user_present, 200) =~ "<!--server-generated-meta-->"
       assert html_response(user_present, 200) =~ "initial-results"
+      assert html_response(user_present, 200) =~ "<title>a cool title</title>"
     end
 
     test "GET /*path", %{conn: conn} do
@@ -44,10 +55,13 @@ test "GET /*path", %{conn: conn} do
 
   describe "preloaded data is attached to" do
     test "GET /main/public", %{conn: conn} do
+      clear_config([:instance, :name], "a cool title")
+
       public_page = get(conn, "/main/public")
 
       refute html_response(public_page, 200) =~ "<!--server-generated-meta-->"
       assert html_response(public_page, 200) =~ "initial-results"
+      assert html_response(public_page, 200) =~ "<title>a cool title</title>"
     end
 
     test "GET /main/all", %{conn: conn} do

From 25bd64b03ac78857dc4d560be0c75ea096080c33 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 11 Nov 2020 17:17:41 +0100
Subject: [PATCH 198/208] Bundled FE: Remove title tag

---
 priv/static/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/priv/static/index.html b/priv/static/index.html
index f5690a8d6..e848c5f8c 100644
--- a/priv/static/index.html
+++ b/priv/static/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.77b1644622e3bae24b6b.css rel=stylesheet><link href=/static/fontello.1600365488745.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.90c4af83c1ae68f4cd95.js></script><script type=text/javascript src=/static/js/app.826c44232e0a76bbd9ba.js></script></body></html>
\ No newline at end of file
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.77b1644622e3bae24b6b.css rel=stylesheet><link href=/static/fontello.1600365488745.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.90c4af83c1ae68f4cd95.js></script><script type=text/javascript src=/static/js/app.826c44232e0a76bbd9ba.js></script></body></html>

From b0e4e0cf2a151900d29da56424f596f3defa23e3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 11 Nov 2020 17:19:09 +0100
Subject: [PATCH 199/208] Changelog: Add info about title injection

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b15ddb943..b619bd891 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Account backup
 - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 - Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`
+- The site title is now injected as a `title` tag like preloads or metadata.
 
 ### Changed
 

From 6fd72e9e8526680836e1bf34c58e10b66dcfee8c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 11 Nov 2020 12:27:51 -0600
Subject: [PATCH 200/208] Ingest blurhash for attachments if they were
 federated

---
 .../activity_pub/object_validators/attachment_validator.ex    | 3 ++-
 .../object_validators/attachment_validator_test.exs           | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index df102a134..f96fd54bf 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:type, :string)
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
+    field(:blurhash, :string)
 
     embeds_many :url, UrlObjectValidator, primary_key: false do
       field(:type, :string)
@@ -41,7 +42,7 @@ def changeset(struct, data) do
       |> fix_url()
 
     struct
-    |> cast(data, [:type, :mediaType, :name])
+    |> cast(data, [:type, :mediaType, :name, :blurhash])
     |> cast_embed(:url, with: &url_changeset/2)
     |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
     |> validate_required([:type, :mediaType, :url])
diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
index 760388e80..2e1975a79 100644
--- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
+++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs
@@ -33,7 +33,8 @@ test "it turns mastodon attachments into our attachments" do
           "http://mastodon.example.org/system/media_attachments/files/000/000/002/original/334ce029e7bfb920.jpg",
         "type" => "Document",
         "name" => nil,
-        "mediaType" => "image/jpeg"
+        "mediaType" => "image/jpeg",
+        "blurhash" => "UD9jJz~VSbR#xT$~%KtQX9R,WAs9RjWBs:of"
       }
 
       {:ok, attachment} =
@@ -50,6 +51,7 @@ test "it turns mastodon attachments into our attachments" do
              ] = attachment.url
 
       assert attachment.mediaType == "image/jpeg"
+      assert attachment.blurhash == "UD9jJz~VSbR#xT$~%KtQX9R,WAs9RjWBs:of"
     end
 
     test "it handles our own uploads" do

From 2254e5e5958803beef94e2d01bdb04647a1f82c9 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 11 Nov 2020 12:51:13 -0600
Subject: [PATCH 201/208] Render blurhashes in Mastodon API

---
 lib/pleroma/web/mastodon_api/views/status_view.ex        | 3 ++-
 test/pleroma/web/mastodon_api/views/status_view_test.exs | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 435bcde15..7cbbd3750 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -435,7 +435,8 @@ def render("attachment.json", %{attachment: attachment}) do
       text_url: href,
       type: type,
       description: attachment["name"],
-      pleroma: %{mime_type: media_type}
+      pleroma: %{mime_type: media_type},
+      blurhash: attachment["blurhash"]
     }
   end
 
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index 70d829979..665199f97 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -420,6 +420,7 @@ test "attachments" do
           "href" => "someurl"
         }
       ],
+      "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn",
       "uuid" => 6
     }
 
@@ -431,7 +432,8 @@ test "attachments" do
       preview_url: "someurl",
       text_url: "someurl",
       description: nil,
-      pleroma: %{mime_type: "image/png"}
+      pleroma: %{mime_type: "image/png"},
+      blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn"
     }
 
     api_spec = Pleroma.Web.ApiSpec.spec()

From 2156de2feef373eecdf8e16f21ac7c4b1ee99995 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 11 Nov 2020 13:39:02 -0600
Subject: [PATCH 202/208] Ingest blurhash field during transmogrification

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 39c8f7e39..0bcd1db22 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -252,6 +252,7 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm
           }
           |> Maps.put_if_present("mediaType", media_type)
           |> Maps.put_if_present("name", data["name"])
+          |> Maps.put_if_present("blurhash", data["blurhash"])
         else
           nil
         end

From 966663c3f89519ad0e3b434e611dcceed6f99822 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 11 Nov 2020 16:17:35 -0600
Subject: [PATCH 203/208] Fix tests for other attachment types

---
 .../web/activity_pub/transmogrifier/audio_handling_test.exs     | 1 +
 .../web/activity_pub/transmogrifier/video_handling_test.exs     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
index 0636d00c5..eef7ab4c6 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
@@ -69,6 +69,7 @@ test "Funkwhale Audio object" do
                "mediaType" => "audio/ogg",
                "type" => "Link",
                "name" => nil,
+               "blurhash" => nil,
                "url" => [
                  %{
                    "href" =>
diff --git a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs
index 69c953a2e..57411fafa 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs
@@ -54,6 +54,7 @@ test "it remaps video URLs as attachments if necessary" do
                "type" => "Link",
                "mediaType" => "video/mp4",
                "name" => nil,
+               "blurhash" => nil,
                "url" => [
                  %{
                    "href" =>
@@ -76,6 +77,7 @@ test "it remaps video URLs as attachments if necessary" do
                "type" => "Link",
                "mediaType" => "video/mp4",
                "name" => nil,
+               "blurhash" => nil,
                "url" => [
                  %{
                    "href" =>

From 6ca709816f74f1171423c7bc040619fca57a2087 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 28 Oct 2020 18:08:23 +0300
Subject: [PATCH 204/208] Fix object spoofing vulnerability in attachments

Validate the content-type of the response when fetching an object,
according to https://www.w3.org/TR/activitypub/#x3-2-retrieving-objects.

content-type headers had to be added to many mocks in order to support
this, some of this was done with a regex. While I did go over the
resulting files to check I didn't modify anything unrelated, there is a
 possibility I missed something.

Closes pleroma#1948
---
 lib/pleroma/object/fetcher.ex                 |  20 +-
 test/fixtures/spoofed-object.json             |  26 +++
 test/pleroma/object/fetcher_test.exs          |  27 ++-
 test/pleroma/object_test.exs                  |  15 +-
 .../web/activity_pub/activity_pub_test.exs    |  24 ++-
 .../transmogrifier/announce_handling_test.exs |   6 +-
 .../transmogrifier/article_handling_test.exs  |  15 +-
 .../transmogrifier/audio_handling_test.exs    |   3 +-
 .../transmogrifier/event_handling_test.exs    |   6 +-
 test/support/http_request_mock.ex             | 190 +++++++++++++-----
 10 files changed, 253 insertions(+), 79 deletions(-)
 create mode 100644 test/fixtures/spoofed-object.json

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 169298b34..ae4301738 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -232,8 +232,24 @@ defp get_object_http(id) do
       |> sign_fetch(id, date)
 
     case HTTP.get(id, headers) do
-      {:ok, %{body: body, status: code}} when code in 200..299 ->
-        {:ok, body}
+      {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
+        case List.keyfind(headers, "content-type", 0) do
+          {_, content_type} ->
+            case Plug.Conn.Utils.media_type(content_type) do
+              {:ok, "application", "activity+json", _} ->
+                {:ok, body}
+
+              {:ok, "application", "ld+json",
+               %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
+                {:ok, body}
+
+              _ ->
+                {:error, {:content_type, content_type}}
+            end
+
+          _ ->
+            {:error, {:content_type, nil}}
+        end
 
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
diff --git a/test/fixtures/spoofed-object.json b/test/fixtures/spoofed-object.json
new file mode 100644
index 000000000..91e34307d
--- /dev/null
+++ b/test/fixtures/spoofed-object.json
@@ -0,0 +1,26 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://patch.cx/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "actor": "https://patch.cx/users/rin",
+  "attachment": [],
+  "attributedTo": "https://patch.cx/users/rin",
+  "cc": [
+    "https://patch.cx/users/rin/followers"
+  ],
+  "content": "Oracle Corporation (NYSE: ORCL) today announced that it has signed a definitive merger agreement to acquire Pleroma AG (FRA: PLA), for $26.50 per share (approximately $10.3 billion). The transaction has been approved by the boards of directors of both companies and should close by early January.",
+  "context": "https://patch.cx/contexts/spoof",
+  "id": "https://patch.cx/objects/spoof",
+  "published": "2020-10-23T18:02:06.038856Z",
+  "sensitive": false,
+  "summary": "Oracle buys Pleroma",
+  "tag": [],
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "type": "Note"
+}
diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs
index 14d2c645f..7df6af7fe 100644
--- a/test/pleroma/object/fetcher_test.exs
+++ b/test/pleroma/object/fetcher_test.exs
@@ -21,6 +21,17 @@ defmodule Pleroma.Object.FetcherTest do
       %{method: :get, url: "https://mastodon.example.org/users/userisgone404"} ->
         %Tesla.Env{status: 404}
 
+      %{
+        method: :get,
+        url:
+          "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-type", "application/json"}],
+          body: File.read!("test/fixtures/spoofed-object.json")
+        }
+
       env ->
         apply(HttpRequestMock, :request, [env])
     end)
@@ -34,19 +45,22 @@ defmodule Pleroma.Object.FetcherTest do
         %{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} ->
           %Tesla.Env{
             status: 200,
-            body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json")
+            body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"),
+            headers: HttpRequestMock.activitypub_object_headers()
           }
 
         %{method: :get, url: "https://social.sakamoto.gq/users/eal"} ->
           %Tesla.Env{
             status: 200,
-            body: File.read!("test/fixtures/fetch_mocks/eal.json")
+            body: File.read!("test/fixtures/fetch_mocks/eal.json"),
+            headers: HttpRequestMock.activitypub_object_headers()
           }
 
         %{method: :get, url: "https://busshi.moe/users/tuxcrafting/statuses/104410921027210069"} ->
           %Tesla.Env{
             status: 200,
-            body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json")
+            body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json"),
+            headers: HttpRequestMock.activitypub_object_headers()
           }
 
         %{method: :get, url: "https://busshi.moe/users/tuxcrafting"} ->
@@ -132,6 +146,13 @@ test "Return MRF reason when fetched status is rejected by one" do
                  "http://mastodon.example.org/@admin/99541947525187367"
                )
     end
+
+    test "it does not fetch a spoofed object uploaded on an instance as an attachment" do
+      assert {:error, _} =
+               Fetcher.fetch_object_from_id(
+                 "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
+               )
+    end
   end
 
   describe "implementation quirks" do
diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs
index 99caba336..5d4e6fb84 100644
--- a/test/pleroma/object_test.exs
+++ b/test/pleroma/object_test.exs
@@ -281,7 +281,11 @@ test "does not fetch unknown objects when fetch_remote is false" do
     setup do
       mock(fn
         %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
-          %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")}
+          %Tesla.Env{
+            status: 200,
+            body: File.read!("test/fixtures/tesla_mock/poll_original.json"),
+            headers: HttpRequestMock.activitypub_object_headers()
+          }
 
         env ->
           apply(HttpRequestMock, :request, [env])
@@ -315,7 +319,8 @@ test "refetches if the time since the last refetch is greater than the interval"
 
       mock_modified.(%Tesla.Env{
         status: 200,
-        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json"),
+        headers: HttpRequestMock.activitypub_object_headers()
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
@@ -359,7 +364,8 @@ test "does not refetch if the time since the last refetch is greater than the in
 
       mock_modified.(%Tesla.Env{
         status: 200,
-        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json"),
+        headers: HttpRequestMock.activitypub_object_headers()
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
@@ -387,7 +393,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
 
       mock_modified.(%Tesla.Env{
         status: 200,
-        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json"),
+        headers: HttpRequestMock.activitypub_object_headers()
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 43bd14ee6..3eeb0f735 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -1426,19 +1426,25 @@ test "doesn't crash when follower and following counters are hidden" do
       mock(fn env ->
         case env.url do
           "http://localhost:4001/users/masto_hidden_counters/following" ->
-            json(%{
-              "@context" => "https://www.w3.org/ns/activitystreams",
-              "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
-            })
+            json(
+              %{
+                "@context" => "https://www.w3.org/ns/activitystreams",
+                "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
+              },
+              headers: HttpRequestMock.activitypub_object_headers()
+            )
 
           "http://localhost:4001/users/masto_hidden_counters/following?page=1" ->
             %Tesla.Env{status: 403, body: ""}
 
           "http://localhost:4001/users/masto_hidden_counters/followers" ->
-            json(%{
-              "@context" => "https://www.w3.org/ns/activitystreams",
-              "id" => "http://localhost:4001/users/masto_hidden_counters/following"
-            })
+            json(
+              %{
+                "@context" => "https://www.w3.org/ns/activitystreams",
+                "id" => "http://localhost:4001/users/masto_hidden_counters/following"
+              },
+              headers: HttpRequestMock.activitypub_object_headers()
+            )
 
           "http://localhost:4001/users/masto_hidden_counters/followers?page=1" ->
             %Tesla.Env{status: 403, body: ""}
@@ -2278,7 +2284,7 @@ test "allow fetching of accounts with an empty string name field" do
     Tesla.Mock.mock(fn
       %{method: :get, url: "https://princess.cat/users/mewmew"} ->
         file = File.read!("test/fixtures/mewmew_no_name.json")
-        %Tesla.Env{status: 200, body: file}
+        %Tesla.Env{status: 200, body: file, headers: HttpRequestMock.activitypub_object_headers()}
     end)
 
     {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew")
diff --git a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
index 54335acdb..99c296c74 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs
@@ -60,7 +60,11 @@ test "it works for incoming announces, fetching the announced object" do
 
     Tesla.Mock.mock(fn
       %{method: :get} ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/mastodon-note-object.json")}
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/mastodon-note-object.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
+        }
     end)
 
     _user = insert(:user, local: false, ap_id: data["actor"])
diff --git a/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs
index 9b12a470a..b0ae804c5 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs
@@ -13,7 +13,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do
 
   test "Pterotype (Wordpress Plugin) Article" do
     Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} ->
-      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")}
+      %Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json"),
+        headers: HttpRequestMock.activitypub_object_headers()
+      }
     end)
 
     data =
@@ -36,13 +40,15 @@ test "Plume Article" do
       %{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} ->
         %Tesla.Env{
           status: 200,
-          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
         }
 
       %{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} ->
         %Tesla.Env{
           status: 200,
-          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
         }
     end)
 
@@ -61,7 +67,8 @@ test "Prismo Article" do
     Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} ->
       %Tesla.Env{
         status: 200,
-        body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
+        body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json"),
+        headers: HttpRequestMock.activitypub_object_headers()
       }
     end)
 
diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
index eef7ab4c6..6eeb1c863 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
@@ -48,7 +48,8 @@ test "Funkwhale Audio object" do
       %{url: "https://channels.tests.funkwhale.audio/federation/actors/compositions"} ->
         %Tesla.Env{
           status: 200,
-          body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json")
+          body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
         }
     end)
 
diff --git a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs
index 7f1ef2cbd..d7c55cfbe 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs
@@ -13,13 +13,15 @@ test "Mobilizon Event object" do
       %{url: "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"} ->
         %Tesla.Env{
           status: 200,
-          body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json")
+          body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
         }
 
       %{url: "https://mobilizon.org/@tcit"} ->
         %Tesla.Env{
           status: 200,
-          body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json")
+          body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json"),
+          headers: HttpRequestMock.activitypub_object_headers()
         }
     end)
 
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index cb022333f..93464ebff 100644
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -5,6 +5,8 @@
 defmodule HttpRequestMock do
   require Logger
 
+  def activitypub_object_headers, do: [{"content-type", "application/activity+json"}]
+
   def request(
         %Tesla.Env{
           url: url,
@@ -34,7 +36,8 @@ def get("https://osada.macgirvin.com/channel/mike", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json")
+       body: File.read!("test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -42,7 +45,8 @@ def get("https://shitposter.club/users/moonman", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/moonman@shitposter.club.json")
+       body: File.read!("test/fixtures/tesla_mock/moonman@shitposter.club.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -50,7 +54,8 @@ def get("https://mastodon.social/users/emelie/statuses/101849165031453009", _, _
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/status.emelie.json")
+       body: File.read!("test/fixtures/tesla_mock/status.emelie.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -66,7 +71,8 @@ def get("https://mastodon.social/users/emelie", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/emelie.json")
+       body: File.read!("test/fixtures/tesla_mock/emelie.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -78,7 +84,8 @@ def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/rinpatch.json")
+       body: File.read!("test/fixtures/tesla_mock/rinpatch.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -86,7 +93,8 @@ def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/poll_attachment.json")
+       body: File.read!("test/fixtures/tesla_mock/poll_attachment.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -99,7 +107,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json")
+       body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -112,7 +121,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/mike@osada.macgirvin.com.json")
+       body: File.read!("test/fixtures/tesla_mock/mike@osada.macgirvin.com.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -190,7 +200,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/lucifermysticus.json")
+       body: File.read!("test/fixtures/tesla_mock/lucifermysticus.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -198,7 +209,8 @@ def get("https://prismo.news/@mxb", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
+       body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -211,7 +223,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json")
+       body: File.read!("test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -219,7 +232,8 @@ def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+jso
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/rye.json")
+       body: File.read!("test/fixtures/tesla_mock/rye.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -227,7 +241,8 @@ def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+jso
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/rye.json")
+       body: File.read!("test/fixtures/tesla_mock/rye.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -246,7 +261,8 @@ def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/puckipedia.com.json")
+       body: File.read!("test/fixtures/tesla_mock/puckipedia.com.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -254,7 +270,8 @@ def get("https://peertube.moe/accounts/7even", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/7even.json")
+       body: File.read!("test/fixtures/tesla_mock/7even.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -262,7 +279,8 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/peertube.moe-vid.json")
+       body: File.read!("test/fixtures/tesla_mock/peertube.moe-vid.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -270,7 +288,8 @@ def get("https://framatube.org/accounts/framasoft", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https___framatube.org_accounts_framasoft.json")
+       body: File.read!("test/fixtures/tesla_mock/https___framatube.org_accounts_framasoft.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -278,7 +297,8 @@ def get("https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/framatube.org-video.json")
+       body: File.read!("test/fixtures/tesla_mock/framatube.org-video.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -286,7 +306,8 @@ def get("https://peertube.social/accounts/craigmaloney", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/craigmaloney.json")
+       body: File.read!("test/fixtures/tesla_mock/craigmaloney.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -294,7 +315,8 @@ def get("https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/peertube-social.json")
+       body: File.read!("test/fixtures/tesla_mock/peertube-social.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -304,7 +326,8 @@ def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _,
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json")
+       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -312,7 +335,8 @@ def get("https://mobilizon.org/@tcit", _, _, [{"accept", "application/activity+j
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json")
+       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -320,7 +344,8 @@ def get("https://baptiste.gelez.xyz/@/BaptisteGelez", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
+       body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -328,7 +353,8 @@ def get("https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
+       body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -336,7 +362,8 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/object/85810", _, _, _) d
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/wedistribute-article.json")
+       body: File.read!("test/fixtures/tesla_mock/wedistribute-article.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -344,7 +371,8 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/actor/-blog", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")
+       body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -352,7 +380,8 @@ def get("http://mastodon.example.org/users/admin", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/admin@mastdon.example.org.json")
+       body: File.read!("test/fixtures/tesla_mock/admin@mastdon.example.org.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -362,7 +391,8 @@ def get("http://mastodon.example.org/users/relay", _, _, [
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/relay@mastdon.example.org.json")
+       body: File.read!("test/fixtures/tesla_mock/relay@mastdon.example.org.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -482,7 +512,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json")
+       body: File.read!("test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -543,7 +574,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/mastodon-note-object.json")
+       body: File.read!("test/fixtures/mastodon-note-object.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -567,7 +599,8 @@ def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activ
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/mayumayu.json")
+       body: File.read!("test/fixtures/tesla_mock/mayumayu.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -580,7 +613,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/mayumayupost.json")
+       body: File.read!("test/fixtures/tesla_mock/mayumayupost.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -795,7 +829,8 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/winterdienst_webfinger.json")
+       body: File.read!("test/fixtures/tesla_mock/winterdienst_webfinger.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -867,12 +902,21 @@ def get("https://social.heldscal.la/.well-known/host-meta", _, _, _) do
   end
 
   def get("https://mastodon.social/users/lambadalambda", _, _, _) do
-    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}}
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/lambadalambda.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("https://apfed.club/channel/indio", _, _, _) do
     {:ok,
-     %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}}
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do
@@ -895,7 +939,8 @@ def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/masto_closed_followers.json")
+       body: File.read!("test/fixtures/users_mock/masto_closed_followers.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -903,7 +948,8 @@ def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json")
+       body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -911,7 +957,8 @@ def get("http://localhost:4001/users/masto_closed/following", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/masto_closed_following.json")
+       body: File.read!("test/fixtures/users_mock/masto_closed_following.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -919,7 +966,8 @@ def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json")
+       body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -927,7 +975,8 @@ def get("http://localhost:8080/followers/fuser3", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/friendica_followers.json")
+       body: File.read!("test/fixtures/users_mock/friendica_followers.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -935,7 +984,8 @@ def get("http://localhost:8080/following/fuser3", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/friendica_following.json")
+       body: File.read!("test/fixtures/users_mock/friendica_following.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -943,7 +993,8 @@ def get("http://localhost:4001/users/fuser2/followers", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/pleroma_followers.json")
+       body: File.read!("test/fixtures/users_mock/pleroma_followers.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -951,7 +1002,8 @@ def get("http://localhost:4001/users/fuser2/following", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/users_mock/pleroma_following.json")
+       body: File.read!("test/fixtures/users_mock/pleroma_following.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -1049,7 +1101,8 @@ def get("https://info.pleroma.site/activity.json", _, _, [
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity.json")
+       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -1063,7 +1116,8 @@ def get("https://info.pleroma.site/activity2.json", _, _, [
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json")
+       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -1077,7 +1131,8 @@ def get("https://info.pleroma.site/activity3.json", _, _, [
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json")
+       body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -1110,7 +1165,12 @@ def get("https://www.patreon.com/posts/mastodon-2-9-and-28121681", _, _, _) do
   end
 
   def get("http://mastodon.example.org/@admin/99541947525187367", _, _, _) do
-    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/mastodon-post-activity.json")}}
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/mastodon-post-activity.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("https://info.pleroma.site/activity4.json", _, _, _) do
@@ -1137,7 +1197,8 @@ def get("https://skippers-bin.com/notes/7x9tmrp97i", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
-       body: File.read!("test/fixtures/tesla_mock/misskey_poll_no_end_date.json")
+       body: File.read!("test/fixtures/tesla_mock/misskey_poll_no_end_date.json"),
+       headers: activitypub_object_headers()
      }}
   end
 
@@ -1146,11 +1207,21 @@ def get("https://example.org/emoji/firedfox.png", _, _, _) do
   end
 
   def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do
-    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/sjw.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("https://patch.cx/users/rin", _, _, _) do
-    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}}
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/rin.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get(
@@ -1160,12 +1231,20 @@ def get(
         _
       ) do
     {:ok,
-     %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/funkwhale_audio.json")}}
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/funkwhale_audio.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("https://channels.tests.funkwhale.audio/federation/actors/compositions", _, _, _) do
     {:ok,
-     %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json")}}
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("http://example.com/rel_me/error", _, _, _) do
@@ -1173,7 +1252,12 @@ def get("http://example.com/rel_me/error", _, _, _) do
   end
 
   def get("https://relay.mastodon.host/actor", _, _, _) do
-    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/relay/relay.json")}}
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/relay/relay.json"),
+       headers: activitypub_object_headers()
+     }}
   end
 
   def get("http://localhost:4001/", _, "", [{"accept", "text/html"}]) do

From c09813193a3429bbab3a3310d9c4b9b75efd6154 Mon Sep 17 00:00:00 2001
From: Michael Walker <mike@barrucadu.co.uk>
Date: Thu, 12 Nov 2020 22:20:17 +0000
Subject: [PATCH 205/208] Install file-dev in Dockerfile build stage

This is required by the majic, added in #2534.
---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c210cf79c..4e7c01c5d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,7 +4,7 @@ COPY . .
 
 ENV MIX_ENV=prod
 
-RUN apk add git gcc g++ musl-dev make cmake &&\
+RUN apk add git gcc g++ musl-dev make cmake file-dev &&\
 	echo "import Mix.Config" > config/prod.secret.exs &&\
 	mix local.hex --force &&\
 	mix local.rebar --force &&\

From 10528344c7c56c70dcfc09a72ff5f109f7b952fe Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 13 Nov 2020 09:07:08 +0300
Subject: [PATCH 206/208] remove PurgeExpiredActivity from Oban db config

---
 ...pired_activity_worker_from_oban_config.exs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs

diff --git a/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs b/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs
new file mode 100644
index 000000000..fe31f4442
--- /dev/null
+++ b/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs
@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.RemovePurgeExpiredActivityWorkerFromObanConfig do
+  use Ecto.Migration
+
+  def change do
+    with %Pleroma.ConfigDB{} = config <-
+           Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}),
+         crontab when is_list(crontab) <- config.value[:crontab],
+         index when is_integer(index) <-
+           Enum.find_index(crontab, fn {_, worker} ->
+             worker == Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker
+           end) do
+      updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index))
+
+      config
+      |> Ecto.Changeset.change(value: updated_value)
+      |> Pleroma.Repo.update()
+    end
+  end
+end

From 70e4b86250da9ef97a836f497510c36bf22fa905 Mon Sep 17 00:00:00 2001
From: Ilja <pleroma@spectraltheorem.be>
Date: Fri, 13 Nov 2020 13:35:46 +0000
Subject: [PATCH 207/208] Make notifs view work for reports

* These are the first small steps for issue 2034 "Reports should send a notification to admins".
* I added a new type of notification "pleroma:report" to the the database manually (a migration will need to be written later)
* I added the new type to the notification_controller
* I made the view return the notification. It doesn't include the report itself (yet)
---
 CHANGELOG.md                                  |  1 +
 docs/API/differences_in_mastoapi_responses.md | 20 +++++++-
 lib/pleroma/notification.ex                   | 12 ++++-
 .../operations/notification_operation.ex      |  3 ++
 .../mastodon_api/views/notification_view.ex   | 11 +++++
 ...eroma_report_to_enum_for_notifications.exs | 48 +++++++++++++++++++
 test/pleroma/notification_test.exs            | 13 +++++
 .../notification_controller_test.exs          | 28 +++++++++++
 .../views/notification_view_test.exs          | 22 +++++++++
 9 files changed, 155 insertions(+), 3 deletions(-)
 create mode 100644 priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f0b90ff40..c963972c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
 - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Reports now generate notifications for admins and mods.
 - Pleroma API: Importing the mutes users from CSV files.
 - Experimental websocket-based federation between Pleroma instances.
 - Support pagination of blocks and mutes
diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md
index 3075b6b86..ba48a2ca1 100644
--- a/docs/API/differences_in_mastoapi_responses.md
+++ b/docs/API/differences_in_mastoapi_responses.md
@@ -129,12 +129,30 @@ The `type` value is `pleroma:emoji_reaction`. Has these fields:
 - `account`: The account of the user who reacted
 - `status`: The status that was reacted on
 
+### ChatMention Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:chat_mention`
+
+- `account`: The account who sent the message
+- `chat_message`: The chat message
+
+### Report Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:report`
+
+- `account`: The account who reported
+- `report`: The report
+
 ## GET `/api/v1/notifications`
 
 Accepts additional parameters:
 
 - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
-- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
+- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
 
 ## DELETE `/api/v1/notifications/destroy_multiple`
 
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 8868a910e..dd7a1c824 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -70,6 +70,7 @@ def unread_notifications_count(%User{id: user_id}) do
     move
     pleroma:chat_mention
     pleroma:emoji_reaction
+    pleroma:report
     reblog
   }
 
@@ -367,7 +368,7 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
-      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
     do_create_notifications(activity, options)
   end
 
@@ -410,6 +411,9 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do
       "EmojiReact" ->
         "pleroma:emoji_reaction"
 
+      "Flag" ->
+        "pleroma:report"
+
       # Compatibility with old reactions
       "EmojiReaction" ->
         "pleroma:emoji_reaction"
@@ -467,7 +471,7 @@ def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true)
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
-      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
+      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
     potential_receivers =
@@ -503,6 +507,10 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => obje
     [object_id]
   end
 
+  def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag"}}) do
+    User.all_superusers() |> Enum.map(fn user -> user.ap_id end)
+  end
+
   def get_potential_receiver_ap_ids(activity) do
     []
     |> Utils.maybe_notify_to_recipients(activity)
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index f09be64cb..264a530d2 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -193,6 +193,7 @@ defp notification_type do
         "mention",
         "pleroma:emoji_reaction",
         "pleroma:chat_mention",
+        "pleroma:report",
         "move",
         "follow_request"
       ],
@@ -206,6 +207,8 @@ defp notification_type do
       - `poll` - A poll you have voted in or created has ended
       - `move` - Someone moved their account
       - `pleroma:emoji_reaction` - Someone reacted with emoji to your status
+      - `pleroma:chat_mention` - Someone mentioned you in a chat message
+      - `pleroma:report` - Someone was reported
       """
     }
   end
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index c97e6d32f..5b06a6b51 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.UserRelationship
+  alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
@@ -118,11 +120,20 @@ def render(
       "pleroma:chat_mention" ->
         put_chat_message(response, activity, reading_user, status_render_opts)
 
+      "pleroma:report" ->
+        put_report(response, activity)
+
       type when type in ["follow", "follow_request"] ->
         response
     end
   end
 
+  defp put_report(response, activity) do
+    report_render = ReportView.render("show.json", Report.extract_report_info(activity))
+
+    Map.put(response, :report, report_render)
+  end
+
   defp put_emoji(response, activity) do
     Map.put(response, :emoji, activity.data["content"])
   end
diff --git a/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs b/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs
new file mode 100644
index 000000000..01fb90459
--- /dev/null
+++ b/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs
@@ -0,0 +1,48 @@
+defmodule Pleroma.Repo.Migrations.AddPleromaReportTypeToEnumForNotifications do
+  use Ecto.Migration
+
+  @disable_ddl_transaction true
+
+  def up do
+    """
+    alter type notification_type add value 'pleroma:report'
+    """
+    |> execute()
+  end
+
+  def down do
+    alter table(:notifications) do
+      modify(:type, :string)
+    end
+
+    """
+    delete from notifications where type = 'pleroma:report'
+    """
+    |> execute()
+
+    """
+    drop type if exists notification_type
+    """
+    |> execute()
+
+    """
+    create type notification_type as enum (
+      'follow',
+      'follow_request',
+      'mention',
+      'move',
+      'pleroma:emoji_reaction',
+      'pleroma:chat_mention',
+      'reblog',
+      'favourite'
+    )
+    """
+    |> execute()
+
+    """
+    alter table notifications 
+    alter column type type notification_type using (type::notification_type)
+    """
+    |> execute()
+  end
+end
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
index 92c0bc8b6..ed2cd219d 100644
--- a/test/pleroma/notification_test.exs
+++ b/test/pleroma/notification_test.exs
@@ -32,6 +32,19 @@ test "never returns nil" do
       refute {:ok, [nil]} == Notification.create_notifications(activity)
     end
 
+    test "creates a notification for a report" do
+      reporting_user = insert(:user)
+      reported_user = insert(:user)
+      {:ok, moderator_user} = insert(:user) |> User.admin_api_update(%{is_moderator: true})
+
+      {:ok, activity} = CommonAPI.report(reporting_user, %{account_id: reported_user.id})
+
+      {:ok, [notification]} = Notification.create_notifications(activity)
+
+      assert notification.user_id == moderator_user.id
+      assert notification.type == "pleroma:report"
+    end
+
     test "creates a notification for an emoji reaction" do
       user = insert(:user)
       other_user = insert(:user)
diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs
index 5fd518c60..9ac8488f6 100644
--- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs
@@ -75,6 +75,34 @@ test "by default, does not contain pleroma:chat_mention" do
     assert [_] = result
   end
 
+  test "by default, does not contain pleroma:report" do
+    %{user: user, conn: conn} = oauth_access(["read:notifications"])
+    other_user = insert(:user)
+    third_user = insert(:user)
+
+    user
+    |> User.admin_api_update(%{is_moderator: true})
+
+    {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
+
+    {:ok, _report} =
+      CommonAPI.report(third_user, %{account_id: other_user.id, status_ids: [activity.id]})
+
+    result =
+      conn
+      |> get("/api/v1/notifications")
+      |> json_response_and_validate_schema(200)
+
+    assert [] == result
+
+    result =
+      conn
+      |> get("/api/v1/notifications?include_types[]=pleroma:report")
+      |> json_response_and_validate_schema(200)
+
+    assert [_] = result
+  end
+
   test "getting a single notification" do
     %{user: user, conn: conn} = oauth_access(["read:notifications"])
     other_user = insert(:user)
diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
index 2f6a808f1..9de11a87e 100644
--- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs
@@ -12,6 +12,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
@@ -207,6 +209,26 @@ test "EmojiReact notification" do
     test_notifications_rendering([notification], user, [expected])
   end
 
+  test "Report notification" do
+    reporting_user = insert(:user)
+    reported_user = insert(:user)
+    {:ok, moderator_user} = insert(:user) |> User.admin_api_update(%{is_moderator: true})
+
+    {:ok, activity} = CommonAPI.report(reporting_user, %{account_id: reported_user.id})
+    {:ok, [notification]} = Notification.create_notifications(activity)
+
+    expected = %{
+      id: to_string(notification.id),
+      pleroma: %{is_seen: false, is_muted: false},
+      type: "pleroma:report",
+      account: AccountView.render("show.json", %{user: reporting_user, for: moderator_user}),
+      created_at: Utils.to_masto_date(notification.inserted_at),
+      report: ReportView.render("show.json", Report.extract_report_info(activity))
+    }
+
+    test_notifications_rendering([notification], moderator_user, [expected])
+  end
+
   test "muted notification" do
     user = insert(:user)
     another_user = insert(:user)

From e2f573d68baa8b161b92459ffacf534054082422 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 14 Nov 2020 22:27:13 +0100
Subject: [PATCH 208/208] pleroma.instance: Fix Exiftool module name

---
 CHANGELOG.md                             | 3 +++
 lib/mix/tasks/pleroma/instance.ex        | 2 +-
 test/mix/tasks/pleroma/instance_test.exs | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b619bd891..ab73de0ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -72,6 +72,9 @@ switched to a new configuration mechanism, however it was not officially removed
 ### Changed
 - API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
 
+### Fixes
+- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`
+
 ## [2.1.2] - 2020-09-17
 
 ### Security
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index fc21ae062..ac8688424 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -284,7 +284,7 @@ defp write_robots_txt(static_dir, indexable, template_dir) do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip do
-        [Pleroma.Upload.Filter.ExifTool]
+        [Pleroma.Upload.Filter.Exiftool]
       else
         []
       end
diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs
index 8a02710ee..6580fc932 100644
--- a/test/mix/tasks/pleroma/instance_test.exs
+++ b/test/mix/tasks/pleroma/instance_test.exs
@@ -88,7 +88,7 @@ test "running gen" do
     assert generated_config =~ "password: \"dbpass\""
     assert generated_config =~ "configurable_from_database: true"
     assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
-    assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]"
+    assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]"
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
   end