From 01c45ddc9ead715131b3c583caa14fcf20845354 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 11 May 2019 11:26:46 +0200
Subject: [PATCH 01/35] Search: Use RUM index.

---
 .../mastodon_api/mastodon_api_controller.ex   |  6 ++--
 ...510135645_add_fts_index_to_objects_two.exs | 33 +++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 956736780..32677df95 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1019,12 +1019,12 @@ def status_search(user, query) do
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-            o.data,
+            "? @@ plainto_tsquery('english', ?)",
+            o.fts_content,
             ^query
           ),
         limit: 20,
-        order_by: [desc: :id]
+        order_by: [fragment("? <=> now()::date", o.inserted_at)]
       )
 
     Repo.all(q) ++ fetched
diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
new file mode 100644
index 000000000..14b964847
--- /dev/null
+++ b/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
@@ -0,0 +1,33 @@
+defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
+  use Ecto.Migration
+
+  def up do
+    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    alter table(:objects) do
+      add(:fts_content, :tsvector)
+    end
+
+    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+    begin
+      new.fts_content := to_tsvector('english', new.data->>'content');
+      return new;
+    end
+    $$ LANGUAGE plpgsql")
+    execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+    execute("UPDATE objects SET updated_at = NOW()")
+  end
+
+  def down do
+    execute "drop index objects_fts"
+    execute "drop trigger tsvectorupdate on objects"
+    execute "drop function objects_fts_update()"
+    alter table(:objects) do
+      remove(:fts_content, :tsvector)
+    end
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

From f1e67bdc312ba16a37916024244d6cb9d4417c9e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 15 May 2019 15:28:01 +0200
Subject: [PATCH 02/35] Search: Add optional rum indexing / searching.

---
 config/config.exs                             |  2 +
 docs/config.md                                | 15 +++++++
 .../mastodon_api/mastodon_api_controller.ex   | 43 ++++++++++++++-----
 mix.exs                                       |  5 ++-
 mix.lock                                      | 10 ++---
 ...510135645_add_fts_index_to_objects_two.exs |  3 ++
 6 files changed, 62 insertions(+), 16 deletions(-)
 rename priv/repo/{migrations => optional_migrations/rum_indexing}/20190510135645_add_fts_index_to_objects_two.exs (91%)

diff --git a/config/config.exs b/config/config.exs
index 1e64b79a7..42e4cb4ce 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -476,6 +476,8 @@
   token_expires_in: 600,
   issue_new_refresh_token: true
 
+config :pleroma, :database, rum_enabled: false
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/config.md b/docs/config.md
index 43ea24d80..99cee25cd 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -537,3 +537,18 @@ Configure OAuth 2 provider capabilities:
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+
+## Database options
+
+### RUM indexing for full text search
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 32677df95..fc0100b1e 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1000,6 +1000,30 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     end
   end
 
+  def status_search_query_with_gin(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^query
+        ),
+      order_by: [desc: :id]
+    )
+  end
+
+  def status_search_query_with_rum(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
@@ -1013,20 +1037,19 @@ def status_search(user, query) do
       end || []
 
     q =
-      from(
-        [a, o] in Activity.with_preloaded_object(Activity),
+      from([a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        where:
-          fragment(
-            "? @@ plainto_tsquery('english', ?)",
-            o.fts_content,
-            ^query
-          ),
-        limit: 20,
-        order_by: [fragment("? <=> now()::date", o.inserted_at)]
+        limit: 20
       )
 
+    q =
+      if Pleroma.Config.get([:database, :rum_enabled]) do
+        status_search_query_with_rum(q, query)
+      else
+        status_search_query_with_gin(q, query)
+      end
+
     Repo.all(q) ++ fetched
   end
 
diff --git a/mix.exs b/mix.exs
index fae21f18d..f7955c6ca 100644
--- a/mix.exs
+++ b/mix.exs
@@ -65,7 +65,10 @@ defp deps do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql, "~>3.0.5"},
+      {:ecto_sql,
+       git: "https://github.com/elixir-ecto/ecto_sql",
+       ref: "e839a9a327b632d73533ac8105ba360bc831cf83",
+       override: true},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index 624c0fb35..d223803ab 100644
--- a/mix.lock
+++ b/mix.lock
@@ -16,12 +16,12 @@
   "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
-  "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
+  "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
-  "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
@@ -61,7 +61,7 @@
   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
-  "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
   "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
   "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
@@ -74,7 +74,7 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
-  "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"},
+  "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
diff --git a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
similarity index 91%
rename from priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
rename to priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 14b964847..09e6cbfb1 100644
--- a/priv/repo/migrations/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -2,6 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
   use Ecto.Migration
 
   def up do
+    execute("create extension if not exists rum")
     drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
     alter table(:objects) do
       add(:fts_content, :tsvector)
@@ -19,6 +20,7 @@ def up do
     FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
 
     execute("UPDATE objects SET updated_at = NOW()")
+    execute("vacuum analyze")
   end
 
   def down do
@@ -29,5 +31,6 @@ def down do
       remove(:fts_content, :tsvector)
     end
     create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    execute("vacuum analyze")
   end
 end

From 88cc423fa3c4baa698e99e1e62d251b6f81cc93d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 15 May 2019 16:45:26 +0200
Subject: [PATCH 03/35] Dependencies: Update Ecto SQL.

---
 mix.exs  |  5 ++++-
 mix.lock | 10 +++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/mix.exs b/mix.exs
index b7b9d534d..2535fcf98 100644
--- a/mix.exs
+++ b/mix.exs
@@ -65,7 +65,10 @@ defp deps do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql, "~>3.0.5"},
+      {:ecto_sql,
+       git: "https://github.com/elixir-ecto/ecto_sql",
+       ref: "e839a9a327b632d73533ac8105ba360bc831cf83",
+       override: true},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index 0b24818c5..5ee5634d3 100644
--- a/mix.lock
+++ b/mix.lock
@@ -16,12 +16,12 @@
   "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
-  "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
+  "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
-  "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
@@ -64,7 +64,7 @@
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
-  "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
   "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
   "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
@@ -77,7 +77,7 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
-  "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"},
+  "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},

From aa11fa4864bf6ebdd4acfdba837675abbdcffc83 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 16 May 2019 12:49:40 +0700
Subject: [PATCH 04/35] add report uri and report to

---
 CHANGELOG.md                            |   3 +-
 config/test.exs                         |   2 +
 docs/config.md                          |   3 +-
 lib/pleroma/plugs/http_security_plug.ex |  31 ++++--
 test/plugs/http_security_plug_test.exs  | 133 ++++++++++++++----------
 5 files changed, 107 insertions(+), 65 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b0e849285..ea1f29304 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `fetch_initial_posts` option
 - Configuration: `notify_email` option
 - Configuration: Media proxy `whitelist` option
+- Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
 - Admin API: Endpoints for listing/revoking invite tokens
@@ -98,7 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
 
 ## Removed
-- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` 
+- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`
 
 ## [0.9.9999] - 2019-04-05
 ### Security
diff --git a/config/test.exs b/config/test.exs
index a0c90c371..40db66170 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -61,6 +61,8 @@
 
 config :pleroma, :app_account_creation, max_requests: 5
 
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
 try do
   import_config "test.secret.exs"
 rescue
diff --git a/docs/config.md b/docs/config.md
index 470f71b7c..c2af5c012 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -286,7 +286,8 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``sts``: Whether to additionally send a `Strict-Transport-Security` header
 * ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent
 * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent
-* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
+* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`
+* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
 
 ## :mrf_user_allowlist
 
diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index a476f1d49..485ddfbc7 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -20,8 +20,9 @@ def call(conn, _options) do
 
   defp headers do
     referrer_policy = Config.get([:http_security, :referrer_policy])
+    report_uri = Config.get([:http_security, :report_uri])
 
-    [
+    headers = [
       {"x-xss-protection", "1; mode=block"},
       {"x-permitted-cross-domain-policies", "none"},
       {"x-frame-options", "DENY"},
@@ -30,12 +31,27 @@ defp headers do
       {"x-download-options", "noopen"},
       {"content-security-policy", csp_string() <> ";"}
     ]
+
+    if report_uri do
+      report_group = %{
+        "group" => "csp-endpoint",
+        "max-age" => 10_886_400,
+        "endpoints" => [
+          %{"url" => report_uri}
+        ]
+      }
+
+      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+    else
+      headers
+    end
   end
 
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
+    report_uri = Config.get([:http_security, :report_uri])
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
@@ -53,7 +69,7 @@ defp csp_string do
         "script-src 'self'"
       end
 
-    [
+    main_part = [
       "default-src 'none'",
       "base-uri 'self'",
       "frame-ancestors 'none'",
@@ -63,11 +79,14 @@ defp csp_string do
       "font-src 'self'",
       "manifest-src 'self'",
       connect_src,
-      script_src,
-      if scheme == "https" do
-        "upgrade-insecure-requests"
-      end
+      script_src
     ]
+
+    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
+
+    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
+
+    (main_part ++ report ++ insecure)
     |> Enum.join("; ")
   end
 
diff --git a/test/plugs/http_security_plug_test.exs b/test/plugs/http_security_plug_test.exs
index 0cbb7e4b1..7dfd50c1f 100644
--- a/test/plugs/http_security_plug_test.exs
+++ b/test/plugs/http_security_plug_test.exs
@@ -7,28 +7,89 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
   alias Pleroma.Config
   alias Plug.Conn
 
-  test "it sends CSP headers when enabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
+  describe "http security enabled" do
+    setup do
+      enabled = Config.get([:http_securiy, :enabled])
 
-    conn =
-      conn
-      |> get("/api/v1/instance")
+      Config.put([:http_security, :enabled], true)
 
-    refute Conn.get_resp_header(conn, "x-xss-protection") == []
-    refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
-    refute Conn.get_resp_header(conn, "x-frame-options") == []
-    refute Conn.get_resp_header(conn, "x-content-type-options") == []
-    refute Conn.get_resp_header(conn, "x-download-options") == []
-    refute Conn.get_resp_header(conn, "referrer-policy") == []
-    refute Conn.get_resp_header(conn, "content-security-policy") == []
+      on_exit(fn ->
+        Config.put([:http_security, :enabled], enabled)
+      end)
+
+      :ok
+    end
+
+    test "it sends CSP headers when enabled", %{conn: conn} do
+      conn = get(conn, "/api/v1/instance")
+
+      refute Conn.get_resp_header(conn, "x-xss-protection") == []
+      refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
+      refute Conn.get_resp_header(conn, "x-frame-options") == []
+      refute Conn.get_resp_header(conn, "x-content-type-options") == []
+      refute Conn.get_resp_header(conn, "x-download-options") == []
+      refute Conn.get_resp_header(conn, "referrer-policy") == []
+      refute Conn.get_resp_header(conn, "content-security-policy") == []
+    end
+
+    test "it sends STS headers when enabled", %{conn: conn} do
+      Config.put([:http_security, :sts], true)
+
+      conn = get(conn, "/api/v1/instance")
+
+      refute Conn.get_resp_header(conn, "strict-transport-security") == []
+      refute Conn.get_resp_header(conn, "expect-ct") == []
+    end
+
+    test "it does not send STS headers when disabled", %{conn: conn} do
+      Config.put([:http_security, :sts], false)
+
+      conn = get(conn, "/api/v1/instance")
+
+      assert Conn.get_resp_header(conn, "strict-transport-security") == []
+      assert Conn.get_resp_header(conn, "expect-ct") == []
+    end
+
+    test "referrer-policy header reflects configured value", %{conn: conn} do
+      conn = get(conn, "/api/v1/instance")
+
+      assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"]
+
+      Config.put([:http_security, :referrer_policy], "no-referrer")
+
+      conn =
+        build_conn()
+        |> get("/api/v1/instance")
+
+      assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"]
+    end
+
+    test "it sends `report-to` & `report-uri` CSP response headers" do
+      conn =
+        build_conn()
+        |> get("/api/v1/instance")
+
+      [csp] = Conn.get_resp_header(conn, "content-security-policy")
+
+      assert csp =~ ~r|report-uri https://endpoint.com; report-to csp-endpoint;|
+
+      [reply_to] = Conn.get_resp_header(conn, "reply-to")
+
+      assert reply_to ==
+               "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}"
+    end
   end
 
   test "it does not send CSP headers when disabled", %{conn: conn} do
+    enabled = Config.get([:http_securiy, :enabled])
+
     Config.put([:http_security, :enabled], false)
 
-    conn =
-      conn
-      |> get("/api/v1/instance")
+    on_exit(fn ->
+      Config.put([:http_security, :enabled], enabled)
+    end)
+
+    conn = get(conn, "/api/v1/instance")
 
     assert Conn.get_resp_header(conn, "x-xss-protection") == []
     assert Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
@@ -38,46 +99,4 @@ test "it does not send CSP headers when disabled", %{conn: conn} do
     assert Conn.get_resp_header(conn, "referrer-policy") == []
     assert Conn.get_resp_header(conn, "content-security-policy") == []
   end
-
-  test "it sends STS headers when enabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
-    Config.put([:http_security, :sts], true)
-
-    conn =
-      conn
-      |> get("/api/v1/instance")
-
-    refute Conn.get_resp_header(conn, "strict-transport-security") == []
-    refute Conn.get_resp_header(conn, "expect-ct") == []
-  end
-
-  test "it does not send STS headers when disabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
-    Config.put([:http_security, :sts], false)
-
-    conn =
-      conn
-      |> get("/api/v1/instance")
-
-    assert Conn.get_resp_header(conn, "strict-transport-security") == []
-    assert Conn.get_resp_header(conn, "expect-ct") == []
-  end
-
-  test "referrer-policy header reflects configured value", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
-
-    conn =
-      conn
-      |> get("/api/v1/instance")
-
-    assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"]
-
-    Config.put([:http_security, :referrer_policy], "no-referrer")
-
-    conn =
-      build_conn()
-      |> get("/api/v1/instance")
-
-    assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"]
-  end
 end

From 8355d0607e986f5a2cc5addfb2bcb35f799f10cb Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 16 May 2019 17:03:12 +0700
Subject: [PATCH 05/35] trailing spaces

---
 docs/introduction.md | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/docs/introduction.md b/docs/introduction.md
index 4af0747fe..045dc7c05 100644
--- a/docs/introduction.md
+++ b/docs/introduction.md
@@ -1,30 +1,30 @@
 # Introduction to Pleroma
 ## What is Pleroma?
-Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.  
-It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.  
-It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.  
+Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.
+It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.
+It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.
 One account on a instance is enough to talk to the entire fediverse!
-  
+
 ## How can I use it?
 
-Pleroma instances are already widely deployed, a list can be found here:  
+Pleroma instances are already widely deployed, a list can be found here:
 http://distsn.org/pleroma-instances.html
 
-If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!  
-Installation instructions can be found here:  
+If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!
+Installation instructions can be found here:
 [main Pleroma wiki](/)
-  
+
 ## I got an account, now what?
-Great! Now you can explore the fediverse!  
-- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password.  
-(If you don't have one yet, click on Register) :slightly_smiling_face:  
+Great! Now you can explore the fediverse!
+- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password.
+(If you don't have one yet, click on Register) :slightly_smiling_face:
 
 At this point you will have two columns in front of you.
 
 ### Left column
-- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers).  
-Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000).  
-If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile:   
+- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers).
+Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000).
+If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile:
 To post your status, simply press Submit.
 
 - second block: Here you can switch between the different timelines:
@@ -38,7 +38,7 @@ To post your status, simply press Submit.
 - fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses.
 
 ### Right column
-This is where the interesting stuff happens! :slight_smile:   
+This is where the interesting stuff happens! :slight_smile:
 Depending on the timeline you will see different statuses, but each status has a standard structure:
 - Icon + name + link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the replied-to status).
 - A + button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime!
@@ -47,9 +47,9 @@ Depending on the timeline you will see different statuses, but each status has a
 - Four buttons (left to right): Reply, Repeat, Favorite, Delete.
 
 ## Mastodon interface
-If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile:  
-Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks:  
-For more information on the Mastodon interface, please look here:  
+If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile:
+Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks:
+For more information on the Mastodon interface, please look here:
 https://github.com/tootsuite/documentation/blob/master/Using-Mastodon/User-guide.md
 
 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

From 4711d8151c8cb7ce2024f378e17b3a113a3aac1c Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 16 May 2019 17:35:07 +0700
Subject: [PATCH 06/35] try this

---
 test/web/common_api/common_api_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 8d4f401ee..b0c441a37 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -279,14 +279,14 @@ test "creates a report" do
     test "add a reblog mute", %{muter: muter, muted: muted} do
       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
 
-      assert Pleroma.User.showing_reblogs?(muter, muted) == false
+      assert User.showing_reblogs?(muter, muted) == false
     end
 
     test "remove a reblog mute", %{muter: muter, muted: muted} do
       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
       {:ok, muter} = CommonAPI.show_reblogs(muter, muted)
 
-      assert Pleroma.User.showing_reblogs?(muter, muted) == true
+      assert User.showing_reblogs?(muter, muted) == true
     end
   end
 end

From 2c303afc8b25ca92eaa229df3e9e4737856e890c Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Thu, 16 May 2019 20:04:08 +0000
Subject: [PATCH 07/35] Remove duplicated entries in users' following lists

---
 CHANGELOG.md                      |  1 +
 lib/mix/tasks/pleroma/database.ex | 17 +++++++++--
 lib/pleroma/user.ex               | 14 ++++++++-
 test/tasks/database_test.exs      | 49 +++++++++++++++++++++++++++++++
 test/tasks/user_test.exs          |  1 +
 test/user_test.exs                | 31 +++++++++++++++++++
 6 files changed, 110 insertions(+), 3 deletions(-)
 create mode 100644 test/tasks/database_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc09d4f6b..4d314817b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for Mastodon's remote interaction
 - Mix Tasks: `mix pleroma.database bump_all_conversations`
 - Mix Tasks: `mix pleroma.database remove_embedded_objects`
+- Mix Tasks: `mix pleroma.database update_users_following_followers_counts`
 - Mix Tasks: `mix pleroma.user toggle_confirmed`
 - Federation: Support for reports
 - Configuration: `safe_dm_mentions` option
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 42753a1a4..f650b447d 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -5,6 +5,8 @@
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Conversation
+  alias Pleroma.Repo
+  alias Pleroma.User
   require Logger
   use Mix.Task
 
@@ -25,6 +27,9 @@ defmodule Mix.Tasks.Pleroma.Database do
 
       mix pleroma.database bump_all_conversations
 
+  ## Remove duplicated items from following and update followers count for all users
+
+      mix pleroma.database update_users_following_followers_counts
   """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
@@ -38,7 +43,7 @@ def run(["remove_embedded_objects" | args]) do
     Common.start_pleroma()
     Logger.info("Removing embedded objects")
 
-    Pleroma.Repo.query!(
+    Repo.query!(
       "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
@@ -47,7 +52,7 @@ def run(["remove_embedded_objects" | args]) do
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")
 
-      Pleroma.Repo.query!(
+      Repo.query!(
         "vacuum full;",
         [],
         timeout: :infinity
@@ -59,4 +64,12 @@ def run(["bump_all_conversations"]) do
     Common.start_pleroma()
     Conversation.bump_for_all_activities()
   end
+
+  def run(["update_users_following_followers_counts"]) do
+    Common.start_pleroma()
+
+    users = Repo.all(User)
+    Enum.each(users, &User.remove_duplicated_following/1)
+    Enum.each(users, &User.update_follower_count/1)
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 1aa966dfc..9ffb61300 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -166,7 +166,7 @@ def remote_user_creation(params) do
 
   def update_changeset(struct, params \\ %{}) do
     struct
-    |> cast(params, [:bio, :name, :avatar])
+    |> cast(params, [:bio, :name, :avatar, :following])
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: 5000)
@@ -709,6 +709,18 @@ def update_follower_count(%User{} = user) do
     end
   end
 
+  def remove_duplicated_following(%User{following: following} = user) do
+    uniq_following = Enum.uniq(following)
+
+    if length(following) == length(uniq_following) do
+      {:ok, user}
+    else
+      user
+      |> update_changeset(%{following: uniq_following})
+      |> update_and_set_cache()
+    end
+  end
+
   @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
   def get_users_from_set(ap_ids, local_only \\ true) do
     criteria = %{ap_id: ap_ids, deactivated: false}
diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs
new file mode 100644
index 000000000..579130b05
--- /dev/null
+++ b/test/tasks/database_test.exs
@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.DatabaseTest do
+  alias Pleroma.Repo
+  alias Pleroma.User
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  setup_all do
+    Mix.shell(Mix.Shell.Process)
+
+    on_exit(fn ->
+      Mix.shell(Mix.Shell.IO)
+    end)
+
+    :ok
+  end
+
+  describe "running update_users_following_followers_counts" do
+    test "following and followers count are updated" do
+      [user, user2] = insert_pair(:user)
+      {:ok, %User{following: following, info: info} = user} = User.follow(user, user2)
+
+      assert length(following) == 2
+      assert info.follower_count == 0
+
+      info_cng = Ecto.Changeset.change(info, %{follower_count: 3})
+
+      {:ok, user} =
+        user
+        |> Ecto.Changeset.change(%{following: following ++ following})
+        |> Ecto.Changeset.put_embed(:info, info_cng)
+        |> Repo.update()
+
+      assert length(user.following) == 4
+      assert user.info.follower_count == 3
+
+      assert :ok == Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"])
+
+      user = User.get_by_id(user.id)
+
+      assert length(user.following) == 2
+      assert user.info.follower_count == 0
+    end
+  end
+end
diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs
index 1f97740be..260ce0d95 100644
--- a/test/tasks/user_test.exs
+++ b/test/tasks/user_test.exs
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.UserTest do
+  alias Pleroma.Repo
   alias Pleroma.User
   use Pleroma.DataCase
 
diff --git a/test/user_test.exs b/test/user_test.exs
index 16a014f2f..721b65693 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -626,6 +626,37 @@ test "it sets the info->follower_count property" do
     end
   end
 
+  describe "remove duplicates from following list" do
+    test "it removes duplicates" do
+      user = insert(:user)
+      follower = insert(:user)
+
+      {:ok, %User{following: following} = follower} = User.follow(follower, user)
+      assert length(following) == 2
+
+      {:ok, follower} =
+        follower
+        |> User.update_changeset(%{following: following ++ following})
+        |> Repo.update()
+
+      assert length(follower.following) == 4
+
+      {:ok, follower} = User.remove_duplicated_following(follower)
+      assert length(follower.following) == 2
+    end
+
+    test "it does nothing when following is uniq" do
+      user = insert(:user)
+      follower = insert(:user)
+
+      {:ok, follower} = User.follow(follower, user)
+      assert length(follower.following) == 2
+
+      {:ok, follower} = User.remove_duplicated_following(follower)
+      assert length(follower.following) == 2
+    end
+  end
+
   describe "follow_import" do
     test "it imports user followings from list" do
       [user1, user2, user3] = insert_list(3, :user)

From 70235ce840a4ed834080ab6afc3000185590c6cb Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Fri, 17 May 2019 09:02:04 +0800
Subject: [PATCH 08/35] Fix typo: s/"tags"/"tag"/g

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +-
 lib/pleroma/web/activity_pub/mrf/tag_policy.ex    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 2f105700b..50426e920 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -55,7 +55,7 @@ defp check_media_nsfw(
     object =
       if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
         tags = (child_object["tag"] || []) ++ ["nsfw"]
-        child_object = Map.put(child_object, "tags", tags)
+        child_object = Map.put(child_object, "tag", tags)
         child_object = Map.put(child_object, "sensitive", true)
         Map.put(object, "object", child_object)
       else
diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
index b52be30e7..6683b8d8e 100644
--- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
@@ -31,7 +31,7 @@ defp process_tag(
 
     object =
       object
-      |> Map.put("tags", tags)
+      |> Map.put("tag", tags)
       |> Map.put("sensitive", true)
 
     message = Map.put(message, "object", object)

From 443cd68a4c37cc8e3a3260feff25791bd5ebabf4 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Fri, 17 May 2019 08:11:17 +0800
Subject: [PATCH 09/35] Add tests for MRF.SimplePolicy

---
 .../activity_pub/mrf/simple_policy_test.exs   | 192 ++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 test/web/activity_pub/mrf/simple_policy_test.exs

diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
new file mode 100644
index 000000000..1e0511975
--- /dev/null
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -0,0 +1,192 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Config
+  alias Pleroma.Web.ActivityPub.MRF.SimplePolicy
+
+  setup do
+    orig = Config.get!(:mrf_simple)
+
+    Config.put(:mrf_simple,
+      media_removal: [],
+      media_nsfw: [],
+      federated_timeline_removal: [],
+      reject: [],
+      accept: []
+    )
+
+    on_exit(fn ->
+      Config.put(:mrf_simple, orig)
+    end)
+  end
+
+  describe "when :media_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :media_removal], [])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) == {:ok, media_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :media_removal], ["remote.instance"])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) ==
+               {:ok,
+                media_message
+                |> Map.put("object", Map.delete(media_message["object"], "attachment"))}
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  describe "when :media_nsfw" do
+    test "is empty" do
+      Config.put([:mrf_simple, :media_nsfw], [])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) == {:ok, media_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :media_nsfw], ["remote.instance"])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) ==
+               {:ok,
+                media_message
+                |> put_in(["object", "tag"], ["foo", "nsfw"])
+                |> put_in(["object", "sensitive"], true)}
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_media_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Create",
+      "object" => %{
+        "attachment" => [%{}],
+        "tag" => ["foo"],
+        "sensitive" => false
+      }
+    }
+  end
+
+  describe "when :federated_timeline_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :federated_timeline_removal], [])
+      {_, ftl_message} = build_ftl_actor_and_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      {actor, ftl_message} = build_ftl_actor_and_message()
+
+      ftl_message_actor_host =
+        ftl_message
+        |> Map.fetch!("actor")
+        |> URI.parse()
+        |> Map.fetch!(:host)
+
+      Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host])
+      local_message = build_local_message()
+
+      assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message)
+      assert actor.follower_address in ftl_message["to"]
+      refute actor.follower_address in ftl_message["cc"]
+      refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"]
+      assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"]
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_ftl_actor_and_message do
+    actor = insert(:user)
+
+    {actor,
+     %{
+       "actor" => actor.ap_id,
+       "to" => ["https://www.w3.org/ns/activitystreams#Public", "http://foo.bar/baz"],
+       "cc" => [actor.follower_address, "http://foo.bar/qux"]
+     }}
+  end
+
+  describe "when :reject" do
+    test "is empty" do
+      Config.put([:mrf_simple, :reject], [])
+
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :reject], ["remote.instance"])
+
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(remote_message) == {:reject, nil}
+    end
+  end
+
+  describe "when :accept" do
+    test "is empty" do
+      Config.put([:mrf_simple, :accept], [])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :accept], ["non.matching.remote"])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:reject, nil}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :accept], ["remote.instance"])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+  end
+
+  defp build_local_message do
+    %{
+      "actor" => "#{Pleroma.Web.base_url()}/users/alice",
+      "to" => [],
+      "cc" => []
+    }
+  end
+
+  defp build_remote_message do
+    %{"actor" => "https://remote.instance/users/bob"}
+  end
+end

From 68c75a08e8593d41f496a362ab1be43d054c09d5 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 17 May 2019 07:34:52 +0200
Subject: [PATCH 10/35] MastoAPI account_view.ex: requested is false when
 following is true

Closes: https://git.pleroma.social/pleroma/pleroma/issues/903
---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 779b9a382..606a0f2ed 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -40,7 +40,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target
     follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target)
 
     requested =
-      if follow_activity do
+      if follow_activity and !User.following?(target, user) do
         follow_activity.data["state"] == "pending"
       else
         false

From 7ed682213faf88edc67b4c2a24b277ab1d992813 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 17 May 2019 07:25:20 +0000
Subject: [PATCH 11/35] Fix/902 random compile failing

---
 lib/pleroma/user.ex                                    |  6 +++---
 lib/pleroma/web/activity_pub/transmogrifier.ex         |  1 -
 lib/pleroma/web/common_api/common_api.ex               |  6 +++---
 lib/pleroma/web/federator/publisher.ex                 |  8 ++++----
 .../web/mastodon_api/mastodon_api_controller.ex        |  2 +-
 lib/pleroma/web/oauth/authorization.ex                 |  2 +-
 lib/pleroma/web/oauth/token.ex                         |  2 +-
 lib/pleroma/web/web_finger/web_finger.ex               |  2 +-
 test/formatter_test.exs                                |  2 +-
 test/plugs/legacy_authentication_plug_test.exs         |  2 +-
 test/repo_test.exs                                     |  7 ++++---
 test/support/factory.ex                                |  9 +++++----
 test/user_test.exs                                     | 10 +++++-----
 test/web/mastodon_api/mastodon_api_controller_test.exs |  8 ++++----
 test/web/ostatus/ostatus_test.exs                      |  4 ++--
 15 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 9ffb61300..9607c1f03 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -55,7 +55,7 @@ defmodule Pleroma.User do
     field(:last_refreshed_at, :naive_datetime_usec)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
-    embeds_one(:info, Pleroma.User.Info)
+    embeds_one(:info, User.Info)
 
     timestamps()
   end
@@ -233,7 +233,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
       |> validate_confirmation(:password)
       |> unique_constraint(:email)
       |> unique_constraint(:nickname)
-      |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames]))
+      |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames]))
       |> validate_format(:nickname, local_nickname_regex())
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
@@ -278,7 +278,7 @@ def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset),
          {:ok, user} <- autofollow_users(user),
          {:ok, user} <- set_cache(user),
-         {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user),
+         {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user),
          {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 508f3532f..5edd8ccc7 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 208c12c7b..a599ffee5 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -200,7 +200,7 @@ def update(user) do
     user =
       with emoji <- emoji_from_profile(user),
            source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji),
-           info_cng <- Pleroma.User.Info.set_source_data(user.info, source_data),
+           info_cng <- User.Info.set_source_data(user.info, source_data),
            change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
            {:ok, user} <- User.update_and_set_cache(change) do
         user
@@ -233,7 +233,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
          } = activity <- get_by_id_or_ap_id(id_or_ap_id),
          true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+           User.Info.add_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do
@@ -250,7 +250,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
   def unpin(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+           User.Info.remove_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do
diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex
index fb4e8548d..70f870244 100644
--- a/lib/pleroma/web/federator/publisher.ex
+++ b/lib/pleroma/web/federator/publisher.ex
@@ -52,9 +52,9 @@ def perform(type, _, _) do
   @doc """
   Relays an activity to all specified peers.
   """
-  @callback publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok | {:error, any()}
+  @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()}
 
-  @spec publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok
+  @spec publish(User.t(), Activity.t()) :: :ok
   def publish(%User{} = user, %Activity{} = activity) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.each(fn module ->
@@ -70,9 +70,9 @@ def publish(%User{} = user, %Activity{} = activity) do
   @doc """
   Gathers links used by an outgoing federation module for WebFinger output.
   """
-  @callback gather_webfinger_links(Pleroma.User.t()) :: list()
+  @callback gather_webfinger_links(User.t()) :: list()
 
-  @spec gather_webfinger_links(Pleroma.User.t()) :: list()
+  @spec gather_webfinger_links(User.t()) :: list()
   def gather_webfinger_links(%User{} = user) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.reduce([], fn module, links ->
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 66056a846..1b776fbca 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1222,7 +1222,7 @@ def remove_from_list(%{assigns: %{user: user}} = conn, %{"id" => id, "account_id
     accounts
     |> Enum.each(fn account_id ->
       with %Pleroma.List{} = list <- Pleroma.List.get(id, user),
-           %User{} = followed <- Pleroma.User.get_cached_by_id(account_id) do
+           %User{} = followed <- User.get_cached_by_id(account_id) do
         Pleroma.List.unfollow(list, followed)
       end
     end)
diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex
index b47688de1..18973413e 100644
--- a/lib/pleroma/web/oauth/authorization.ex
+++ b/lib/pleroma/web/oauth/authorization.ex
@@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index ef047d565..66c95c2e9 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:refresh_token, :string)
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 3a3b98a10..1239b962a 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -99,7 +99,7 @@ def ensure_keys_present(user) do
 
       info_cng =
         info
-        |> Pleroma.User.Info.set_keys(pem)
+        |> User.Info.set_keys(pem)
 
       cng =
         Ecto.Changeset.change(user)
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index 06f4f6e50..5e7011160 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -125,7 +125,7 @@ test "gives a replacement for user links, using local nicknames in user links te
       archaeme =
         insert(:user, %{
           nickname: "archa_eme_",
-          info: %Pleroma.User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}}
+          info: %User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}}
         })
 
       archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs
index 8b0b06772..02f530058 100644
--- a/test/plugs/legacy_authentication_plug_test.exs
+++ b/test/plugs/legacy_authentication_plug_test.exs
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do
-  use Pleroma.Web.ConnCase, async: true
+  use Pleroma.Web.ConnCase
 
   alias Pleroma.Plugs.LegacyAuthenticationPlug
   alias Pleroma.User
diff --git a/test/repo_test.exs b/test/repo_test.exs
index 5382289c7..85085a1fa 100644
--- a/test/repo_test.exs
+++ b/test/repo_test.exs
@@ -1,23 +1,24 @@
 defmodule Pleroma.RepoTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  alias Pleroma.User
 
   describe "find_resource/1" do
     test "returns user" do
       user = insert(:user)
-      query = from(t in Pleroma.User, where: t.id == ^user.id)
+      query = from(t in User, where: t.id == ^user.id)
       assert Repo.find_resource(query) == {:ok, user}
     end
 
     test "returns not_found" do
-      query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
+      query = from(t in User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
       assert Repo.find_resource(query) == {:error, :not_found}
     end
   end
 
   describe "get_assoc/2" do
     test "get assoc from preloaded data" do
-      user = %Pleroma.User{name: "Agent Smith"}
+      user = %User{name: "Agent Smith"}
       token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user}
       assert Repo.get_assoc(token, :user) == {:ok, user}
     end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 90c7d80f2..be6247ca4 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Factory do
   use ExMachina.Ecto, repo: Pleroma.Repo
+  alias Pleroma.User
 
   def participation_factory do
     conversation = insert(:conversation)
@@ -23,7 +24,7 @@ def conversation_factory do
   end
 
   def user_factory do
-    user = %Pleroma.User{
+    user = %User{
       name: sequence(:name, &"Test テスト User #{&1}"),
       email: sequence(:email, &"user#{&1}@example.com"),
       nickname: sequence(:nickname, &"nick#{&1}"),
@@ -34,9 +35,9 @@ def user_factory do
 
     %{
       user
-      | ap_id: Pleroma.User.ap_id(user),
-        follower_address: Pleroma.User.ap_followers(user),
-        following: [Pleroma.User.ap_id(user)]
+      | ap_id: User.ap_id(user),
+        follower_address: User.ap_followers(user),
+        following: [User.ap_id(user)]
     }
   end
 
diff --git a/test/user_test.exs b/test/user_test.exs
index 721b65693..10e463ff8 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -277,7 +277,7 @@ test "it requires an email, name, nickname and password, bio is optional" do
     end
 
     test "it restricts certain nicknames" do
-      [restricted_name | _] = Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
+      [restricted_name | _] = Pleroma.Config.get([User, :restricted_nicknames])
 
       assert is_bitstring(restricted_name)
 
@@ -1223,11 +1223,11 @@ test "follower count is updated when a follower is blocked" do
     follower2 = insert(:user)
     follower3 = insert(:user)
 
-    {:ok, follower} = Pleroma.User.follow(follower, user)
-    {:ok, _follower2} = Pleroma.User.follow(follower2, user)
-    {:ok, _follower3} = Pleroma.User.follow(follower3, user)
+    {:ok, follower} = User.follow(follower, user)
+    {:ok, _follower2} = User.follow(follower2, user)
+    {:ok, _follower3} = User.follow(follower3, user)
 
-    {:ok, _} = Pleroma.User.block(user, follower)
+    {:ok, _} = User.block(user, follower)
 
     user_show = Pleroma.Web.TwitterAPI.UserView.render("show.json", %{user: user})
 
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 90d67a55f..cbff141c8 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -446,7 +446,7 @@ test "verify_credentials", %{conn: conn} do
   end
 
   test "verify_credentials default scope unlisted", %{conn: conn} do
-    user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "unlisted"}})
+    user = insert(:user, %{info: %User.Info{default_scope: "unlisted"}})
 
     conn =
       conn
@@ -1322,7 +1322,7 @@ test "returns the relationships for the current user", %{conn: conn} do
 
   describe "locked accounts" do
     test "/api/v1/follow_requests works" do
-      user = insert(:user, %{info: %Pleroma.User.Info{locked: true}})
+      user = insert(:user, %{info: %User.Info{locked: true}})
       other_user = insert(:user)
 
       {:ok, _activity} = ActivityPub.follow(other_user, user)
@@ -1367,7 +1367,7 @@ test "/api/v1/follow_requests/:id/authorize works" do
     end
 
     test "verify_credentials", %{conn: conn} do
-      user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "private"}})
+      user = insert(:user, %{info: %User.Info{default_scope: "private"}})
 
       conn =
         conn
@@ -1379,7 +1379,7 @@ test "verify_credentials", %{conn: conn} do
     end
 
     test "/api/v1/follow_requests/:id/reject works" do
-      user = insert(:user, %{info: %Pleroma.User.Info{locked: true}})
+      user = insert(:user, %{info: %User.Info{locked: true}})
       other_user = insert(:user)
 
       {:ok, _activity} = ActivityPub.follow(other_user, user)
diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs
index 2916caf8d..3516a13fa 100644
--- a/test/web/ostatus/ostatus_test.exs
+++ b/test/web/ostatus/ostatus_test.exs
@@ -355,7 +355,7 @@ test "tries to use the information in poco fields" do
 
       {:ok, user} = OStatus.find_or_make_user(uri)
 
-      user = Pleroma.User.get_cached_by_id(user.id)
+      user = User.get_cached_by_id(user.id)
       assert user.name == "Constance Variable"
       assert user.nickname == "lambadalambda@social.heldscal.la"
       assert user.local == false
@@ -374,7 +374,7 @@ test "find_or_make_user sets all the nessary input fields" do
       {:ok, user} = OStatus.find_or_make_user(uri)
 
       assert user.info ==
-               %Pleroma.User.Info{
+               %User.Info{
                  id: user.info.id,
                  ap_enabled: false,
                  background: %{},

From 718b59a2631af6245570a073e3c5b2464b810a27 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 11:19:30 +0200
Subject: [PATCH 12/35] Dependencies: Update ecto sql once more.

---
 mix.exs  | 2 +-
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index 2535fcf98..2888b69e2 100644
--- a/mix.exs
+++ b/mix.exs
@@ -67,7 +67,7 @@ defp deps do
       {:phoenix_ecto, "~> 4.0"},
       {:ecto_sql,
        git: "https://github.com/elixir-ecto/ecto_sql",
-       ref: "e839a9a327b632d73533ac8105ba360bc831cf83",
+       ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
        override: true},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
diff --git a/mix.lock b/mix.lock
index 5ee5634d3..714a8a2b1 100644
--- a/mix.lock
+++ b/mix.lock
@@ -21,7 +21,7 @@
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]},
+  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},

From e39c190f046bb345e80a2195278bb59ac81b8002 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 11:30:35 +0200
Subject: [PATCH 13/35] Configuration: disable migration lock.

---
 config/config.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 3d11c3a87..92d3e0dcd 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -48,7 +48,8 @@
 
 config :pleroma, Pleroma.Repo,
   types: Pleroma.PostgresTypes,
-  telemetry_event: [Pleroma.Repo.Instrumenter]
+  telemetry_event: [Pleroma.Repo.Instrumenter],
+  migration_lock: nil
 
 config :pleroma, Pleroma.Captcha,
   enabled: false,

From 249b31ffe53115ad187235890c405138ba190e14 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 11:49:13 +0200
Subject: [PATCH 14/35] Fix specs.

---
 lib/pleroma/user.ex               | 1 -
 test/web/ostatus/ostatus_test.exs | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 9607c1f03..28da310ee 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1144,7 +1144,6 @@ def delete_user_activities(%User{ap_id: ap_id} = user) do
     stream =
       ap_id
       |> Activity.query_by_actor()
-      |> Activity.with_preloaded_object()
       |> Repo.stream()
 
     Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs
index 3516a13fa..f6be16862 100644
--- a/test/web/ostatus/ostatus_test.exs
+++ b/test/web/ostatus/ostatus_test.exs
@@ -407,7 +407,7 @@ test "find_make_or_update_user takes an author element and returns an updated us
       {:ok, user} = OStatus.find_or_make_user(uri)
       old_name = user.name
       old_bio = user.bio
-      change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil})
+      change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, name: nil})
 
       {:ok, user} = Repo.update(change)
       refute user.avatar

From 5743124c7727af3fbcd8243f1f8e71de91c9b470 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 11:51:50 +0200
Subject: [PATCH 15/35] Dependencies: Update tzdata.

This fixes the bug that tz data could not be loaded. Closes #910.
---
 mix.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.lock b/mix.lock
index cd2d2370a..1dc19532d 100644
--- a/mix.lock
+++ b/mix.lock
@@ -83,7 +83,7 @@
   "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
-  "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
+  "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
   "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"},

From 46684db84d0583cb5d88f8fcbd6b970ef95c84de Mon Sep 17 00:00:00 2001
From: lambda <lain@soykaf.club>
Date: Fri, 17 May 2019 09:53:44 +0000
Subject: [PATCH 16/35] Update account_view.ex

---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 606a0f2ed..134c07b7e 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -40,7 +40,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target
     follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target)
 
     requested =
-      if follow_activity and !User.following?(target, user) do
+      if follow_activity && !User.following?(target, user) do
         follow_activity.data["state"] == "pending"
       else
         false

From a3fc7294da43a7e2d17626200e8d3bf5c05808d3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 12:47:38 +0200
Subject: [PATCH 17/35] CI: Add rum variant testing.

---
 .gitlab-ci.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f9745122a..3ea275127 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,6 +15,7 @@ cache:
 stages:
   - build
   - test
+  - test-rum
   - deploy
 
 before_script:
@@ -45,7 +46,7 @@ docs-build:
 unit-testing:
   stage: test
   services:
-  - name: postgres:9.6.2
+  - name: lainsoykaf/postgres-with-rum
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
@@ -54,6 +55,20 @@ unit-testing:
     - mix test --trace --preload-modules
     - mix coveralls
 
+unit-testing-rum:
+  stage: test-rum
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  script:
+    - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs"
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
+    - mix test --trace --preload-modules
+    - mix coveralls
+
 lint:
   stage: test
   script:
@@ -65,7 +80,6 @@ analysis:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
-
 docs-deploy:
   stage: deploy
   image: alpine:3.9

From 022e6e4b44229be70f0ec6720a66610a0e2a403a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 14:59:01 +0200
Subject: [PATCH 18/35] RUM: Remove vacuum analyze from migration

Can't be run in a trnasaction.
---
 .../20190510135645_add_fts_index_to_objects_two.exs             | 2 --
 1 file changed, 2 deletions(-)

diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 09e6cbfb1..b6a24441a 100644
--- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -20,7 +20,6 @@ def up do
     FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
 
     execute("UPDATE objects SET updated_at = NOW()")
-    execute("vacuum analyze")
   end
 
   def down do
@@ -31,6 +30,5 @@ def down do
       remove(:fts_content, :tsvector)
     end
     create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
-    execute("vacuum analyze")
   end
 end

From 8784a7d1b40394cb1b4d5ce2d04b8cd057a57ceb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 15:06:51 +0200
Subject: [PATCH 19/35] RUM: Set rum status by the environment.

---
 config/test.exs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/config/test.exs b/config/test.exs
index 40db66170..e1785d10f 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -63,6 +63,10 @@
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
 try do
   import_config "test.secret.exs"
 rescue

From ef63cf70883f4c7e3814ebac907f000da835ea10 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 May 2019 15:21:29 +0200
Subject: [PATCH 20/35] CI: Use the correct image with the correct hostname.

---
 .gitlab-ci.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3ea275127..8b5131dc3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,6 @@ cache:
 stages:
   - build
   - test
-  - test-rum
   - deploy
 
 before_script:
@@ -47,6 +46,7 @@ unit-testing:
   stage: test
   services:
   - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
@@ -56,18 +56,19 @@ unit-testing:
     - mix coveralls
 
 unit-testing-rum:
-  stage: test-rum
+  stage: test
   services:
   - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  variables:
+    RUM_ENABLED: "true"
   script:
-    - "echo 'config :pleroma, :database, rum_enabled: true' >> config/test.exs"
     - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
     - mix test --trace --preload-modules
-    - mix coveralls
 
 lint:
   stage: test

From 78588dbd80580fbef53819dd87ee8fcc26cb09e9 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 18:49:10 +0000
Subject: [PATCH 21/35] mrf: simple policy: mark all posts instead of posts
 with media as sensitive if they match media_nsfw

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 50426e920..9627c3400 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -48,10 +48,9 @@ defp check_media_nsfw(
          %{host: actor_host} = _actor_info,
          %{
            "type" => "Create",
-           "object" => %{"attachment" => child_attachment} = child_object
+           "object" => child_object
          } = object
-       )
-       when length(child_attachment) > 0 do
+       ) do
     object =
       if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
         tags = (child_object["tag"] || []) ++ ["nsfw"]

From 0da1233e8e093ff7c69994f9e81d58611be60507 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 18:49:43 +0000
Subject: [PATCH 22/35] rich media: suppress link previews if post is marked as
 sensitive

---
 lib/pleroma/web/rich_media/helpers.ex |  1 +
 test/web/rich_media/helpers_test.exs  | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 0162a5be9..9bc8f2559 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -24,6 +24,7 @@ defp validate_page_url(_), do: :error
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
          %Object{} = object <- Object.normalize(activity),
+         false <- object.data["sensitive"] || false,
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
index 60d93768f..6e23392ca 100644
--- a/test/web/rich_media/helpers_test.exs
+++ b/test/web/rich_media/helpers_test.exs
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.RichMedia.HelpersTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
@@ -59,4 +60,24 @@ test "crawls valid, complete URLs" do
 
     Pleroma.Config.put([:rich_media, :enabled], false)
   end
+
+  test "refuses to crawl URLs from posts marked sensitive" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp",
+        "spoiler_text" => "."
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
 end

From d3b8cd342ff6d1fe87bcfe6305424faf49666252 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 19:03:19 +0000
Subject: [PATCH 23/35] http: request builder: send user-agent when making
 requests

---
 lib/pleroma/http/request_builder.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex
index 5f2cff2c0..522728da1 100644
--- a/lib/pleroma/http/request_builder.ex
+++ b/lib/pleroma/http/request_builder.ex
@@ -45,8 +45,9 @@ def url(request, u) do
   Add headers to the request
   """
   @spec headers(map(), list(tuple)) :: map()
-  def headers(request, h) do
-    Map.put_new(request, :headers, h)
+  def headers(request, header_list) do
+    header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+    Map.put_new(request, :headers, header_list)
   end
 
   @doc """

From 290f5b2cfe91dd2acba56f79ef137f15c68a0db0 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:28:58 +0000
Subject: [PATCH 24/35] config: make sending the user agent configurable,
 disable sending the user agent in tests

---
 config/config.exs                   | 1 +
 config/test.exs                     | 2 ++
 lib/pleroma/http/request_builder.ex | 8 +++++++-
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index e82f08e07..9a10b0ff7 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -192,6 +192,7 @@
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
+  send_user_agent: true,
   adapter: [
     ssl_options: [
       # We don't support TLS v1.3 yet
diff --git a/config/test.exs b/config/test.exs
index e1785d10f..6100989c4 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -63,6 +63,8 @@
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
+config :pleroma, :http, send_user_agent: false
+
 rum_enabled = System.get_env("RUM_ENABLED") == "true"
 config :pleroma, :database, rum_enabled: rum_enabled
 IO.puts("RUM enabled: #{rum_enabled}")
diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex
index 522728da1..e23457999 100644
--- a/lib/pleroma/http/request_builder.ex
+++ b/lib/pleroma/http/request_builder.ex
@@ -46,7 +46,13 @@ def url(request, u) do
   """
   @spec headers(map(), list(tuple)) :: map()
   def headers(request, header_list) do
-    header_list = header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+    header_list =
+      if Pleroma.Config.get([:http, :send_user_agent]) do
+        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+      else
+        header_list
+      end
+
     Map.put_new(request, :headers, header_list)
   end
 

From c234ce546a769747f436c19fee99bed2a7a58f3b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:31:39 +0000
Subject: [PATCH 25/35] add CHANGELOG entry

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..5de9ae292 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- User-Agent is now sent correctly for all HTTP requests.
 
 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`

From dc081595385084fe6b382e4b38c17cb51cf0a611 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:42:51 +0000
Subject: [PATCH 26/35] also suppress link previews from posts marked #nsfw

---
 lib/pleroma/web/common_api/common_api.ex |  4 +++-
 lib/pleroma/web/common_api/utils.ex      |  4 +++-
 test/web/rich_media/helpers_test.exs     | 21 ++++++++++++++++++++-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index a599ffee5..5a312d673 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -157,6 +157,7 @@ def post(user, %{"status" => status} = data) do
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
          cw <- data["spoiler_text"] || "",
+         sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
          full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
@@ -169,7 +170,8 @@ def post(user, %{"status" => status} = data) do
              in_reply_to,
              tags,
              cw,
-             cc
+             cc,
+             sensitive
            ),
          object <-
            Map.put(
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index bee2fd159..8d6160976 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -223,7 +223,8 @@ def make_note_data(
         in_reply_to,
         tags,
         cw \\ nil,
-        cc \\ []
+        cc \\ [],
+        sensitive \\ false
       ) do
     object = %{
       "type" => "Note",
@@ -231,6 +232,7 @@ def make_note_data(
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
+      "sensitive" => sensitive,
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
index 6e23392ca..53b0596f5 100644
--- a/test/web/rich_media/helpers_test.exs
+++ b/test/web/rich_media/helpers_test.exs
@@ -67,7 +67,26 @@ test "refuses to crawl URLs from posts marked sensitive" do
     {:ok, activity} =
       CommonAPI.post(user, %{
         "status" => "http://example.com/ogp",
-        "spoiler_text" => "."
+        "sensitive" => true
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
+
+  test "refuses to crawl URLs from posts tagged NSFW" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp #nsfw"
       })
 
     %Object{} = object = Object.normalize(activity)

From c4a55e167afcfd25cf4c1efb46886f2defe72c22 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 17 May 2019 20:43:31 +0000
Subject: [PATCH 27/35] add Changelog entry

---
 CHANGELOG.md                        | 1 +
 lib/pleroma/web/common_api/utils.ex | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12c439135..a73bf47d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
+- Posts which are marked sensitive or tagged nsfw no longer have link previews.
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 8d6160976..d93c0d46e 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -232,7 +232,7 @@ def make_note_data(
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
-      "sensitive" => sensitive,
+      "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive),
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,

From 2375e9a95ba9042958ff7e8f75830df4ab53fed2 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Mon, 20 May 2019 06:02:50 +0800
Subject: [PATCH 28/35] Add report filtering to MRF.SimplePolicy

---
 config/config.exs                             |  1 +
 .../web/activity_pub/mrf/simple_policy.ex     | 13 ++++++++-
 .../activity_pub/mrf/simple_policy_test.exs   | 28 +++++++++++++++++++
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index 9a10b0ff7..bab47a8a2 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -298,6 +298,7 @@
   media_removal: [],
   media_nsfw: [],
   federated_timeline_removal: [],
+  report_removal: [],
   reject: [],
   accept: []
 
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 9627c3400..7190652d2 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -94,6 +94,16 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
     {:ok, object}
   end
 
+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(object) do
     actor_info = URI.parse(object["actor"])
@@ -102,7 +112,8 @@ def filter(object) do
          {:ok, object} <- check_reject(actor_info, object),
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
       _e -> {:reject, nil}
diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs
index 1e0511975..74af7dcde 100644
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
       media_removal: [],
       media_nsfw: [],
       federated_timeline_removal: [],
+      report_removal: [],
       reject: [],
       accept: []
     )
@@ -85,6 +86,33 @@ defp build_media_message do
     }
   end
 
+  describe "when :report_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :report_removal], [])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:ok, report_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :report_removal], ["remote.instance"])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:reject, nil}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_report_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Flag"
+    }
+  end
+
   describe "when :federated_timeline_removal" do
     test "is empty" do
       Config.put([:mrf_simple, :federated_timeline_removal], [])

From 54e9cb5c2db580bc12441f3651fa87a7b976137d Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:39:23 +0100
Subject: [PATCH 29/35] Add API endpoints for a custom user mascot

---
 docs/api/pleroma_api.md                       | 39 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 21 ++++++++++
 .../mastodon_api/mastodon_api_controller.ex   | 36 ++++++++++++++++-
 lib/pleroma/web/router.ex                     |  3 ++
 4 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md
index dd0b6ca73..4d99a2d2b 100644
--- a/docs/api/pleroma_api.md
+++ b/docs/api/pleroma_api.md
@@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```
 
+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 5f0cefc00..ffcd06e3e 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,6 +43,19 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+
+    field(:mascot, :map,
+      default: %{
+        id: "pleromatan",
+        url: "/images/pleroma-fox-tan-smol.png",
+        type: "image",
+        preview_url: "/images/pleroma-fox-tan-smol.png",
+        pleroma: %{
+          mime_type: "image/png"
+        }
+      }
+    )
+
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
@@ -248,6 +261,14 @@ def mastodon_flavour_update(info, flavour) do
     |> validate_required([:flavour])
   end
 
+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
   def set_source_data(info, source_data) do
     params = %{source_data: source_data}
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 1051861ff..67f363859 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -707,6 +707,40 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     end
   end
 
+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    %{info: %{mascot: mascot}} = user
+
+    conn
+    |> json(mascot)
+  end
+
   def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
@@ -1329,7 +1363,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 6a4e4a1d4..4c29b24eb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do
 
       post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour)
 
+      get("/pleroma/mascot", MastodonAPIController, :get_mascot)
+      put("/pleroma/mascot", MastodonAPIController, :set_mascot)
+
       post("/reports", MastodonAPIController, :reports)
     end
 

From e81f0fc6d45249dd70656c58af926c21c70c482f Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:06 +0100
Subject: [PATCH 30/35] Add mascot get/set tests

---
 .../mastodon_api/mastodon_api_controller.ex   |   1 +
 test/fixtures/sound.mp3                       | Bin 0 -> 521 bytes
 .../mastodon_api_controller_test.exs          |  65 ++++++++++++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 test/fixtures/sound.mp3

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 67f363859..d7f095a1f 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -729,6 +729,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
         |> json(rendered)
       else
         conn
+        |> put_resp_content_type("application/json")
         |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
       end
     end
diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3
new file mode 100644
index 0000000000000000000000000000000000000000..9f0f661a3bc34b6366304e9fd1d4fe8513411083
GIT binary patch
literal 521
zcmezWd%_V0bP$o5mkt!;2VzDB1}091|Fj1{y8?V1eO-<93=IrecEX$_s-VK;;K;!E
q0OXGES_-dW5+jBF|6AY)1M>j}#w9=>D=;vaG%zr*zym6jY5)LcXO3V1

literal 0
HcmV?d00001

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index cbff141c8..87e1c105d 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1455,6 +1455,71 @@ test "media upload", %{conn: conn} do
     assert object.data["actor"] == User.ap_id(user)
   end
 
+  test "mascot upload", %{conn: conn} do
+    user = insert(:user)
+
+    non_image_file = %Plug.Upload{
+      content_type: "audio/mpeg",
+      path: Path.absname("test/fixtures/sound.mp3"),
+      filename: "sound.mp3"
+    }
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file})
+
+    assert json_response(conn, 415)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert %{"id" => _, "type" => image} = json_response(conn, 200)
+  end
+
+  test "mascot retrieving", %{conn: conn} do
+    user = insert(:user)
+    # When user hasn't set a mascot, we should just get pleroma tan back
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url} = json_response(conn, 200)
+    assert url =~ "pleroma-fox-tan-smol"
+
+    # When a user sets their mascot, we should get that back
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn = 
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+    assert json_response(conn, 200)
+
+    user = User.get_cached_by_id(user.id)
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url, "type" => "image"} = json_response(conn, 200)
+    assert url =~ "an_image"
+  end
+
   test "hashtag timeline", %{conn: conn} do
     following = insert(:user)
 

From dc916ba15f0fd77afa015084849f082065ed6f74 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 12:58:17 +0100
Subject: [PATCH 31/35] Format mascot tests

---
 test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 87e1c105d..1d9f5a816 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1503,10 +1503,11 @@ test "mascot retrieving", %{conn: conn} do
       filename: "an_image.jpg"
     }
 
-    conn = 
+    conn =
       build_conn()
       |> assign(:user, user)
       |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
     assert json_response(conn, 200)
 
     user = User.get_cached_by_id(user.id)

From 3d0d9e7a5680bcb1b79e5f4aab0e8c514fc9e5ff Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 13:10:04 +0100
Subject: [PATCH 32/35] Use string map for default mascot

---
 lib/pleroma/user/info.ex | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index ffcd06e3e..e76d04d7f 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -46,12 +46,12 @@ defmodule Pleroma.User.Info do
 
     field(:mascot, :map,
       default: %{
-        id: "pleromatan",
-        url: "/images/pleroma-fox-tan-smol.png",
-        type: "image",
-        preview_url: "/images/pleroma-fox-tan-smol.png",
-        pleroma: %{
-          mime_type: "image/png"
+        "id" => "pleromatan",
+        "url" => "/images/pleroma-fox-tan-smol.png",
+        "type" => "image",
+        "preview_url" => "/images/pleroma-fox-tan-smol.png",
+        "pleroma" => %{
+          "mime_type" => "image/png"
         }
       }
     )

From d835810610e5e7660a56d305bc8509e38c642942 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 14:19:42 +0100
Subject: [PATCH 33/35] Add changelog entry for mascot config

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 87101db30..07e6bce31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
 - Admin API: added filters (role, tags, email, name) for users endpoint

From daeae8e2e7c506b72c66dea6ac790408f948ec16 Mon Sep 17 00:00:00 2001
From: Sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Mon, 20 May 2019 16:12:55 +0100
Subject: [PATCH 34/35] Move default mascot configuration to `config/`

---
 config/config.exs                             | 13 ++++++++++++
 docs/config.md                                | 10 ++++++++++
 lib/pleroma/user.ex                           | 20 +++++++++++++++++++
 lib/pleroma/user/info.ex                      | 14 +------------
 .../mastodon_api/mastodon_api_controller.ex   |  4 ++--
 5 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index bab47a8a2..72908266d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -276,6 +276,19 @@
     showInstanceSpecificPanel: true
   }
 
+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
diff --git a/docs/config.md b/docs/config.md
index 450d73fda..197326bbd 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)
 
+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 28da310ee..05fe58f7c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1402,4 +1402,24 @@ def toggle_confirmation(%User{} = user) do
     |> put_embed(:info, info_changeset)
     |> update_and_set_cache()
   end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
 end
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index e76d04d7f..6397e2737 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -43,19 +43,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
-
-    field(:mascot, :map,
-      default: %{
-        "id" => "pleromatan",
-        "url" => "/images/pleroma-fox-tan-smol.png",
-        "type" => "image",
-        "preview_url" => "/images/pleroma-fox-tan-smol.png",
-        "pleroma" => %{
-          "mime_type" => "image/png"
-        }
-      }
-    )
-
+    field(:mascot, :map, default: nil)
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index d7f095a1f..1ec0f30a1 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -736,7 +736,7 @@ def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
   end
 
   def get_mascot(%{assigns: %{user: user}} = conn, _params) do
-    %{info: %{mascot: mascot}} = user
+    mascot = User.get_mascot(user)
 
     conn
     |> json(mascot)
@@ -1364,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: Map.get(user.info.mascot, "url", "/images/pleroma-fox-tan-smol.png")
+            mascot: User.get_mascot(user)["url"]
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),

From 5aa107d9126c484f0ae06eddec6995b535623b32 Mon Sep 17 00:00:00 2001
From: Aaron Tinio <aptinio@gmail.com>
Date: Tue, 21 May 2019 00:59:12 +0800
Subject: [PATCH 35/35] Document MRF.Simple :report_removal

---
 docs/config/mrf.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/config/mrf.md b/docs/config/mrf.md
index 2cc16cef0..45be18fc5 100644
--- a/docs/config/mrf.md
+++ b/docs/config/mrf.md
@@ -5,11 +5,12 @@ Possible uses include:
 
 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance
 
-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.  
+The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
 It is possible to use multiple, active MRF policies at the same time.
 
 ## Quarantine Instances
@@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.
 
 Servers should be configured as lists.
 
 ### Example
 
-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```
 config :pleroma, :instance,
@@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
   media_nsfw: ["porn.biz", "porn.business"],
   reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]
 
 ```