From 241a3d744ae4e9d040247ad0aeb6287156acf920 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 11 Feb 2020 13:53:24 +0400
Subject: [PATCH 001/283] Add ActivityExpirationPolicy

---
 config/config.exs                             |  2 +
 lib/pleroma/web/activity_pub/mrf.ex           |  7 +---
 .../mrf/activity_expiration_policy.ex         | 35 +++++++++++++++++
 .../mrf/activity_expiration_policy_test.exs   | 38 +++++++++++++++++++
 4 files changed, 77 insertions(+), 5 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
 create mode 100644 test/web/activity_pub/mrf/activity_expiration_policy_test.exs

diff --git a/config/config.exs b/config/config.exs
index 41c1ff637..d5b298c16 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -361,6 +361,8 @@
 
 config :pleroma, :mrf_subchain, match_actor: %{}
 
+config :pleroma, :mrf_activity_expiration, days: 365
+
 config :pleroma, :mrf_vocabulary,
   accept: [],
   reject: []
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 263ed11af..b6e737de5 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -8,11 +8,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do
   def filter(policies, %{} = object) do
     policies
     |> Enum.reduce({:ok, object}, fn
-      policy, {:ok, object} ->
-        policy.filter(object)
-
-      _, error ->
-        error
+      policy, {:ok, object} -> policy.filter(object)
+      _, error -> error
     end)
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
new file mode 100644
index 000000000..1b8860161
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -0,0 +1,35 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
+  @moduledoc "Adds expiration to all local activities"
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(%{"id" => id} = activity) do
+    activity =
+      if String.starts_with?(id, Pleroma.Web.Endpoint.url()) do
+        maybe_add_expiration(activity)
+      else
+        activity
+      end
+
+    {:ok, activity}
+  end
+
+  @impl true
+  def describe, do: {:ok, %{}}
+
+  defp maybe_add_expiration(activity) do
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+    expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days)
+
+    with %{"expires_at" => existing_expires_at} <- activity,
+         :lt <- NaiveDateTime.compare(existing_expires_at, expires_at) do
+      activity
+    else
+      _ -> Map.put(activity, "expires_at", expires_at)
+    end
+  end
+end
diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
new file mode 100644
index 000000000..2e65048c0
--- /dev/null
+++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
+  use ExUnit.Case, async: true
+  alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
+
+  @id Pleroma.Web.Endpoint.url() <> "/activities/cofe"
+
+  test "adds `expires_at` property" do
+    assert {:ok, %{"expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{"id" => @id})
+
+    assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
+  end
+
+  test "keeps existing `expires_at` if it less than the config setting" do
+    expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: 1)
+
+    assert {:ok, %{"expires_at" => ^expires_at}} =
+             ActivityExpirationPolicy.filter(%{"id" => @id, "expires_at" => expires_at})
+  end
+
+  test "owerwrites existing `expires_at` if it greater than the config setting" do
+    too_distant_future = NaiveDateTime.utc_now() |> Timex.shift(years: 2)
+
+    assert {:ok, %{"expires_at" => expires_at}} =
+             ActivityExpirationPolicy.filter(%{"id" => @id, "expires_at" => too_distant_future})
+
+    assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
+  end
+
+  test "ignores remote activities" do
+    assert {:ok, activity} = ActivityExpirationPolicy.filter(%{"id" => "https://example.com/123"})
+
+    refute Map.has_key?(activity, "expires_at")
+  end
+end

From 4d459b0e9906b2ebc0280b36c92007b2e680671f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 12 Feb 2020 22:51:26 +0400
Subject: [PATCH 002/283] Move ActivityExpiration creation from
 CommonApi.post/2 to ActivityPub.insert/4

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 17 ++++++++++++++---
 lib/pleroma/web/common_api/activity_draft.ex |  9 ++++++++-
 lib/pleroma/web/common_api/common_api.ex     | 12 +-----------
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5c436941a..408f6c966 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1,10 +1,11 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Config
   alias Pleroma.Conversation
   alias Pleroma.Conversation.Participation
@@ -135,12 +136,14 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
          {:containment, :ok} <- {:containment, Containment.contain_child(map)},
          {:ok, map, object} <- insert_full_object(map) do
       {:ok, activity} =
-        Repo.insert(%Activity{
+        %Activity{
           data: map,
           local: local,
           actor: map["actor"],
           recipients: recipients
-        })
+        }
+        |> Repo.insert()
+        |> maybe_create_activity_expiration()
 
       # Splice in the child object if we have one.
       activity =
@@ -180,6 +183,14 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
     end
   end
 
+  defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do
+    with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do
+      {:ok, activity}
+    end
+  end
+
+  defp maybe_create_activity_expiration(result), do: result
+
   defp create_or_bump_conversation(activity, actor) do
     with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
          %User{} = user <- User.get_cached_by_ap_id(actor),
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index f7da81b34..7a83cad9c 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -193,6 +193,13 @@ defp preview?(draft) do
 
   defp changes(draft) do
     direct? = draft.visibility == "direct"
+    additional = %{"cc" => draft.cc, "directMessage" => direct?}
+
+    additional =
+      case draft.expires_at do
+        %NaiveDateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at)
+        _ -> additional
+      end
 
     changes =
       %{
@@ -200,7 +207,7 @@ defp changes(draft) do
         actor: draft.user,
         context: draft.context,
         object: draft.object,
-        additional: %{"cc" => draft.cc, "directMessage" => direct?}
+        additional: additional
       }
       |> Utils.maybe_add_list_data(draft.user, draft.visibility)
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 2a348dcf6..03921de27 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -277,20 +277,10 @@ def listen(user, %{"title" => _} = data) do
 
   def post(user, %{"status" => _} = data) do
     with {:ok, draft} <- Pleroma.Web.CommonAPI.ActivityDraft.create(user, data) do
-      draft.changes
-      |> ActivityPub.create(draft.preview?)
-      |> maybe_create_activity_expiration(draft.expires_at)
+      ActivityPub.create(draft.changes, draft.preview?)
     end
   end
 
-  defp maybe_create_activity_expiration({:ok, activity}, %NaiveDateTime{} = expires_at) do
-    with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do
-      {:ok, activity}
-    end
-  end
-
-  defp maybe_create_activity_expiration(result, _), do: result
-
   # Updates the emojis for a user based on their profile
   def update(user) do
     emoji = emoji_from_profile(user)

From e2d358f1fb0babbdd2a318bad863e27afecbb3d1 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 14 Feb 2020 15:19:23 +0400
Subject: [PATCH 003/283] Fix typo

---
 test/web/activity_pub/mrf/activity_expiration_policy_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
index 2e65048c0..2f2f90b44 100644
--- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
+++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
@@ -21,7 +21,7 @@ test "keeps existing `expires_at` if it less than the config setting" do
              ActivityExpirationPolicy.filter(%{"id" => @id, "expires_at" => expires_at})
   end
 
-  test "owerwrites existing `expires_at` if it greater than the config setting" do
+  test "overwrites existing `expires_at` if it greater than the config setting" do
     too_distant_future = NaiveDateTime.utc_now() |> Timex.shift(years: 2)
 
     assert {:ok, %{"expires_at" => expires_at}} =

From 57878f870879995f53227bb7a24b810531dd4217 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 14 Feb 2020 15:50:31 +0400
Subject: [PATCH 004/283] Improve readability

---
 .../web/activity_pub/mrf/activity_expiration_policy.ex    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index 1b8860161..5d823f2c7 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -7,9 +7,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true
-  def filter(%{"id" => id} = activity) do
+  def filter(activity) do
     activity =
-      if String.starts_with?(id, Pleroma.Web.Endpoint.url()) do
+      if local?(activity) do
         maybe_add_expiration(activity)
       else
         activity
@@ -21,6 +21,10 @@ def filter(%{"id" => id} = activity) do
   @impl true
   def describe, do: {:ok, %{}}
 
+  defp local?(%{"id" => id}) do
+    String.starts_with?(id, Pleroma.Web.Endpoint.url())
+  end
+
   defp maybe_add_expiration(activity) do
     days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
     expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days)

From 3732b0ba729bb7443e338b5f6bcc7e018983aa4c Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 14 Feb 2020 16:39:02 +0400
Subject: [PATCH 005/283] Update CHANGELOG

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 150fd27cd..e4a641a7e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Rate limiter is now disabled for localhost/socket (unless remoteip plug is enabled)
 - Logger: default log level changed from `warn` to `info`.
 - Config mix task `migrate_to_db` truncates `config` table before migrating the config file.
+- MFR policy to set global expiration for every local activity
+
 <details>
   <summary>API Changes</summary>
 

From 0ddcd67d32eb40cb6cb2a3dfee4c55e930e7f37c Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 14 Feb 2020 16:53:53 +0400
Subject: [PATCH 006/283] Update `cheatsheet.md` and `config/description.exs`

---
 config/description.exs           | 15 +++++++++++++++
 docs/configuration/cheatsheet.md |  9 +++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index e5bac9b3f..d86a4ccca 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1346,6 +1346,21 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :mrf_activity_expiration,
+    label: "MRF Activity Expiration Policy",
+    type: :group,
+    description: "Adds expiration to all local activities",
+    children: [
+      %{
+        key: :days,
+        type: :integer,
+        description: "Default global expiration time for all local activities (in days)",
+        suggestions: [90, 365]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :mrf_subchain,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 2bd935983..bd03aec66 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -33,7 +33,7 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic
 * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
     * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default).
     * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production.
-    * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See [`:mrf_simple`](#mrf_simple)).
+    * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certain instances (See [`:mrf_simple`](#mrf_simple)).
     * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive).
     * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (See [`:mrf_subchain`](#mrf_subchain)).
     * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See [`:mrf_rejectnonpublic`](#mrf_rejectnonpublic)).
@@ -43,7 +43,8 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
-* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Adds expiration to all local activities (see [`:mrf_activity_expiration`](#mrf_activity_expiration)).
+* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
 * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
 * `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
@@ -142,6 +143,10 @@ config :pleroma, :mrf_user_allowlist,
   * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
   * `:reject` rejects the message entirely
 
+#### :mrf_activity_expiration
+
+* `days`: Default global expiration time for all local activities (in days)
+
 ### :activitypub
 * ``unfollow_blocked``: Whether blocks result in people getting unfollowed
 * ``outgoing_blocks``: Whether to federate blocks to other instances

From 819cd467170cb6dd1334cde0a0c79dbb785a22b6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 20 Feb 2020 22:04:02 +0400
Subject: [PATCH 007/283] Auto-expire Create activities only

---
 .../mrf/activity_expiration_policy.ex         |  2 +-
 test/web/activity_pub/activity_pub_test.exs   | 16 +++++++++
 .../mrf/activity_expiration_policy_test.exs   | 35 +++++++++++++++----
 .../purge_expired_activities_worker_test.exs  | 30 ++++++++++++++++
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index 5d823f2c7..274bb9a5c 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
   @impl true
   def filter(activity) do
     activity =
-      if local?(activity) do
+      if activity["type"] == "Create" && local?(activity) do
         maybe_add_expiration(activity)
       else
         activity
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index ce68e7d0e..2cd908a87 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1784,4 +1784,20 @@ test "old user must be in the new user's `also_known_as` list" do
                ActivityPub.move(old_user, new_user)
     end
   end
+
+  describe "global activity expiration" do
+    clear_config([:instance, :rewrite_policy])
+
+    test "creates an activity expiration for local Create activities" do
+      Pleroma.Config.put(
+        [:instance, :rewrite_policy],
+        Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
+      )
+
+      {:ok, %{id: id_create}} = ActivityBuilder.insert(%{"type" => "Create", "context" => "3hu"})
+      {:ok, _follow} = ActivityBuilder.insert(%{"type" => "Follow", "context" => "3hu"})
+
+      assert [%{activity_id: ^id_create}] = Pleroma.ActivityExpiration |> Repo.all()
+    end
+  end
 end
diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
index 2f2f90b44..0d3bcc457 100644
--- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
+++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
@@ -9,7 +9,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
   @id Pleroma.Web.Endpoint.url() <> "/activities/cofe"
 
   test "adds `expires_at` property" do
-    assert {:ok, %{"expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{"id" => @id})
+    assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
+             ActivityExpirationPolicy.filter(%{"id" => @id, "type" => "Create"})
 
     assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
   end
@@ -17,21 +18,43 @@ test "adds `expires_at` property" do
   test "keeps existing `expires_at` if it less than the config setting" do
     expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: 1)
 
-    assert {:ok, %{"expires_at" => ^expires_at}} =
-             ActivityExpirationPolicy.filter(%{"id" => @id, "expires_at" => expires_at})
+    assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} =
+             ActivityExpirationPolicy.filter(%{
+               "id" => @id,
+               "type" => "Create",
+               "expires_at" => expires_at
+             })
   end
 
   test "overwrites existing `expires_at` if it greater than the config setting" do
     too_distant_future = NaiveDateTime.utc_now() |> Timex.shift(years: 2)
 
-    assert {:ok, %{"expires_at" => expires_at}} =
-             ActivityExpirationPolicy.filter(%{"id" => @id, "expires_at" => too_distant_future})
+    assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
+             ActivityExpirationPolicy.filter(%{
+               "id" => @id,
+               "type" => "Create",
+               "expires_at" => too_distant_future
+             })
 
     assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
   end
 
   test "ignores remote activities" do
-    assert {:ok, activity} = ActivityExpirationPolicy.filter(%{"id" => "https://example.com/123"})
+    assert {:ok, activity} =
+             ActivityExpirationPolicy.filter(%{
+               "id" => "https://example.com/123",
+               "type" => "Create"
+             })
+
+    refute Map.has_key?(activity, "expires_at")
+  end
+
+  test "ignores non-Create activities" do
+    assert {:ok, activity} =
+             ActivityExpirationPolicy.filter(%{
+               "id" => "https://example.com/123",
+               "type" => "Follow"
+             })
 
     refute Map.has_key?(activity, "expires_at")
   end
diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs
index c2561683e..c6c7ff388 100644
--- a/test/workers/cron/purge_expired_activities_worker_test.exs
+++ b/test/workers/cron/purge_expired_activities_worker_test.exs
@@ -12,6 +12,7 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do
   import ExUnit.CaptureLog
 
   clear_config([ActivityExpiration, :enabled])
+  clear_config([:instance, :rewrite_policy])
 
   test "deletes an expiration activity" do
     Pleroma.Config.put([ActivityExpiration, :enabled], true)
@@ -36,6 +37,35 @@ test "deletes an expiration activity" do
     refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id)
   end
 
+  test "works with ActivityExpirationPolicy" do
+    Pleroma.Config.put([ActivityExpiration, :enabled], true)
+
+    Pleroma.Config.put(
+      [:instance, :rewrite_policy],
+      Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
+    )
+
+    user = insert(:user)
+
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+
+    {:ok, %{id: id} = activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "cofe"})
+
+    past_date =
+      NaiveDateTime.utc_now() |> Timex.shift(days: -days) |> NaiveDateTime.truncate(:second)
+
+    activity
+    |> Repo.preload(:expiration)
+    |> Map.get(:expiration)
+    |> Ecto.Changeset.change(%{scheduled_at: past_date})
+    |> Repo.update!()
+
+    Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid)
+
+    assert [%{data: %{"type" => "Delete", "deleted_activity_id" => ^id}}] =
+             Pleroma.Repo.all(Pleroma.Activity)
+  end
+
   describe "delete_activity/1" do
     test "adds log message if activity isn't find" do
       assert capture_log([level: :error], fn ->

From 011ede45361096f55dda938078e24574cdf33b2b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 21 Feb 2020 14:42:43 +0400
Subject: [PATCH 008/283] Update documentation

---
 CHANGELOG.md                                                  | 2 +-
 config/description.exs                                        | 4 ++--
 docs/configuration/cheatsheet.md                              | 4 ++--
 .../web/activity_pub/mrf/activity_expiration_policy.ex        | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e4a641a7e..c5558e0c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,7 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Rate limiter is now disabled for localhost/socket (unless remoteip plug is enabled)
 - Logger: default log level changed from `warn` to `info`.
 - Config mix task `migrate_to_db` truncates `config` table before migrating the config file.
-- MFR policy to set global expiration for every local activity
+- MFR policy to set global expiration for all local Create activities
 
 <details>
   <summary>API Changes</summary>
diff --git a/config/description.exs b/config/description.exs
index d86a4ccca..f0c6e3377 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1351,12 +1351,12 @@
     key: :mrf_activity_expiration,
     label: "MRF Activity Expiration Policy",
     type: :group,
-    description: "Adds expiration to all local activities",
+    description: "Adds expiration to all local Create activities",
     children: [
       %{
         key: :days,
         type: :integer,
-        description: "Default global expiration time for all local activities (in days)",
+        description: "Default global expiration time for all local Create activities (in days)",
         suggestions: [90, 365]
       }
     ]
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index bd03aec66..f50c8bab7 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -43,7 +43,7 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
-    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Adds expiration to all local activities (see [`:mrf_activity_expiration`](#mrf_activity_expiration)).
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Adds expiration to all local Create activities (see [`:mrf_activity_expiration`](#mrf_activity_expiration)).
 * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
 * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
 * `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``.
@@ -145,7 +145,7 @@ config :pleroma, :mrf_user_allowlist,
 
 #### :mrf_activity_expiration
 
-* `days`: Default global expiration time for all local activities (in days)
+* `days`: Default global expiration time for all local Create activities (in days)
 
 ### :activitypub
 * ``unfollow_blocked``: Whether blocks result in people getting unfollowed
diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index 274bb9a5c..a9bdf3b69 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
-  @moduledoc "Adds expiration to all local activities"
+  @moduledoc "Adds expiration to all local Create activities"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

From fc2eb1fbd6a5b38a3cf72e557cce1029d6b7f16f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 20 Mar 2020 22:16:57 +0400
Subject: [PATCH 009/283] Fix formatter warnings

---
 test/workers/cron/purge_expired_activities_worker_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs
index 85ae1e5ef..beac55fb2 100644
--- a/test/workers/cron/purge_expired_activities_worker_test.exs
+++ b/test/workers/cron/purge_expired_activities_worker_test.exs
@@ -11,8 +11,8 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do
   import Pleroma.Factory
   import ExUnit.CaptureLog
 
-  setup do 
-  	clear_config([ActivityExpiration, :enabled])
+  setup do
+    clear_config([ActivityExpiration, :enabled])
     clear_config([:instance, :rewrite_policy])
   end
 

From fd97b0e634d30dec3217efcf3d67610d1b54bf8b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 9 Mar 2020 17:00:16 +0100
Subject: [PATCH 010/283] Chats: Basic implementation.

---
 lib/pleroma/chat.ex                           | 41 +++++++++++++++++
 .../20200309123730_create_chats.exs           | 16 +++++++
 test/chat_test.exs                            | 44 +++++++++++++++++++
 3 files changed, 101 insertions(+)
 create mode 100644 lib/pleroma/chat.ex
 create mode 100644 priv/repo/migrations/20200309123730_create_chats.exs
 create mode 100644 test/chat_test.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
new file mode 100644
index 000000000..e2a8b8eba
--- /dev/null
+++ b/lib/pleroma/chat.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Chat do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  @moduledoc """
+  Chat keeps a reference to DirectMessage conversations between a user and an recipient. The recipient can be a user (for now) or a group (not implemented yet).
+
+  It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
+  """
+
+  schema "chats" do
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+    field(:recipient, :string)
+    field(:unread, :integer, default: 0)
+
+    timestamps()
+  end
+
+  def creation_cng(struct, params) do
+    struct
+    |> cast(params, [:user_id, :recipient])
+    |> validate_required([:user_id, :recipient])
+    |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
+  end
+
+  def get_or_create(user_id, recipient) do
+    %__MODULE__{}
+    |> creation_cng(%{user_id: user_id, recipient: recipient})
+    |> Repo.insert(
+      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
+      conflict_target: [:user_id, :recipient]
+    )
+  end
+end
diff --git a/priv/repo/migrations/20200309123730_create_chats.exs b/priv/repo/migrations/20200309123730_create_chats.exs
new file mode 100644
index 000000000..715d798ea
--- /dev/null
+++ b/priv/repo/migrations/20200309123730_create_chats.exs
@@ -0,0 +1,16 @@
+defmodule Pleroma.Repo.Migrations.CreateChats do
+  use Ecto.Migration
+
+  def change do
+    create table(:chats) do
+      add(:user_id, references(:users, type: :uuid))
+      # Recipient is an ActivityPub id, to future-proof for group support.
+      add(:recipient, :string)
+      add(:unread, :integer, default: 0)
+      timestamps()
+    end
+
+    # There's only one chat between a user and a recipient.
+    create(index(:chats, [:user_id, :recipient], unique: true))
+  end
+end
diff --git a/test/chat_test.exs b/test/chat_test.exs
new file mode 100644
index 000000000..ca9206802
--- /dev/null
+++ b/test/chat_test.exs
@@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ChatTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Chat
+
+  import Pleroma.Factory
+
+  describe "creation and getting" do
+    test "it creates a chat for a user and recipient" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      assert chat.id
+    end
+
+    test "it returns a chat for a user and recipient if it already exists" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat_two} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      assert chat.id == chat_two.id
+    end
+
+    test "a returning chat will have an updated `update_at` field" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      :timer.sleep(1500)
+      {:ok, chat_two} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      assert chat.id == chat_two.id
+      assert chat.updated_at != chat_two.updated_at
+    end
+  end
+end

From 3775683a04e9b819f88bfba533b755bbd5b3c2df Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 8 Apr 2020 15:55:43 +0200
Subject: [PATCH 011/283] ChatMessage: Basic incoming handling.

---
 lib/pleroma/chat.ex                           |  2 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  |  1 +
 .../web/activity_pub/object_validator.ex      | 30 +++++++++-
 .../chat_message_validator.ex                 | 58 +++++++++++++++++++
 .../create_chat_message_validator.ex          | 35 +++++++++++
 ..._validator.ex => create_note_validator.ex} |  0
 .../object_validators/types/recipients.ex     | 23 ++++++++
 .../web/activity_pub/transmogrifier.ex        |  7 +++
 .../transmogrifier/chat_message_handling.ex   | 30 ++++++++++
 test/fixtures/create-chat-message.json        | 19 ++++++
 .../types/recipients_test.exs                 | 15 +++++
 .../transmogrifier/chat_message_test.exs      | 32 ++++++++++
 12 files changed, 250 insertions(+), 2 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
 rename lib/pleroma/web/activity_pub/object_validators/{create_validator.ex => create_note_validator.ex} (100%)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
 create mode 100644 lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
 create mode 100644 test/fixtures/create-chat-message.json
 create mode 100644 test/web/activity_pub/object_validators/types/recipients_test.exs
 create mode 100644 test/web/activity_pub/transmogrifier/chat_message_test.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index e2a8b8eba..07ad62b97 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Chat do
   alias Pleroma.Repo
 
   @moduledoc """
-  Chat keeps a reference to DirectMessage conversations between a user and an recipient. The recipient can be a user (for now) or a group (not implemented yet).
+  Chat keeps a reference to ChatMessage conversations between a user and an recipient. The recipient can be a user (for now) or a group (not implemented yet).
 
   It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
   """
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 19286fd01..0b4892501 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -397,6 +397,7 @@ defp do_unreact_with_emoji(user, reaction_id, options) do
     end
   end
 
+  # TODO: Is this even used now?
   # TODO: This is weird, maybe we shouldn't check here if we can make the activity.
   @spec like(User.t(), Object.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t(), Object.t()} | {:error, any()}
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index dc4bce059..49cc72561 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -12,18 +12,46 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
 
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)
 
   def validate(%{"type" => "Like"} = object, meta) do
     with {:ok, object} <-
-           object |> LikeValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do
+           object
+           |> LikeValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object |> Map.from_struct())
       {:ok, object, meta}
     end
   end
 
+  def validate(%{"type" => "ChatMessage"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> ChatMessageValidator.cast_and_apply() do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Create"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> CreateChatMessageValidator.cast_and_apply() do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def stringify_keys(%{__struct__: _} = object) do
+    object
+    |> Map.from_struct()
+    |> stringify_keys
+  end
+
   def stringify_keys(object) do
     object
     |> Map.new(fn {key, val} -> {to_string(key), val} end)
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
new file mode 100644
index 000000000..ab5be3596
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -0,0 +1,58 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:to, Types.Recipients, default: [])
+    field(:type, :string)
+    field(:content, :string)
+    field(:actor, Types.ObjectID)
+    field(:published, Types.DateTime)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def fix(data) do
+    data
+    |> Map.put_new("actor", data["attributedTo"])
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["ChatMessage"])
+    |> validate_required([:id, :actor, :to, :type, :content])
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
new file mode 100644
index 000000000..659311480
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -0,0 +1,35 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# NOTES
+# - Can probably be a generic create validator
+# - doesn't embed, will only get the object id
+# - object has to be validated first, maybe with some meta info from the surrounding create
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:actor, Types.ObjectID)
+    field(:type, :string)
+    field(:to, Types.Recipients, default: [])
+    field(:object, Types.ObjectID)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_data(data) do
+    cast(%__MODULE__{}, data, __schema__(:fields))
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
similarity index 100%
rename from lib/pleroma/web/activity_pub/object_validators/create_validator.ex
rename to lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
new file mode 100644
index 000000000..5a3040842
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
@@ -0,0 +1,23 @@
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do
+  use Ecto.Type
+
+  def type, do: {:array, :string}
+
+  def cast(object) when is_binary(object) do
+    cast([object])
+  end
+
+  def cast([_ | _] = data), do: {:ok, data}
+
+  def cast(_) do
+    :error
+  end
+
+  def dump(data) do
+    {:ok, data}
+  end
+
+  def load(data) do
+    {:ok, data}
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 0a8ad62ad..becc35ea3 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.Pipeline
+  alias Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
@@ -612,6 +613,12 @@ def handle_incoming(
     |> handle_incoming(options)
   end
 
+  def handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
+        options
+      ),
+      do: ChatMessageHandling.handle_incoming(data, options)
+
   def handle_incoming(%{"type" => "Like"} = data, _options) do
     with {_, {:ok, cast_data_sym}} <-
            {:casting_data,
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
new file mode 100644
index 000000000..b5843736f
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling do
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
+  alias Pleroma.Web.ActivityPub.Pipeline
+
+  def handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object_data} = data,
+        _options
+      ) do
+    with {_, {:ok, cast_data_sym}} <-
+           {:casting_data, data |> CreateChatMessageValidator.cast_and_apply()},
+         cast_data = ObjectValidator.stringify_keys(cast_data_sym),
+         {_, {:ok, object_cast_data_sym}} <-
+           {:casting_object_data, object_data |> ChatMessageValidator.cast_and_apply()},
+         object_cast_data = ObjectValidator.stringify_keys(object_cast_data_sym),
+         {_, {:ok, validated_object, _meta}} <-
+           {:validate_object, ObjectValidator.validate(object_cast_data, %{})},
+         {_, {:ok, _created_object}} <- {:persist_object, Object.create(validated_object)},
+         {_, {:ok, activity, _meta}} <-
+           {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do
+      {:ok, activity}
+    end
+  end
+end
diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json
new file mode 100644
index 000000000..4aa17f4a5
--- /dev/null
+++ b/test/fixtures/create-chat-message.json
@@ -0,0 +1,19 @@
+{
+    "actor": "http://2hu.gensokyo/users/raymoo",
+    "id": "http://2hu.gensokyo/objects/1",
+    "object": {
+        "attributedTo": "http://2hu.gensokyo/users/raymoo",
+        "content": "You expected a cute girl? Too bad.",
+        "id": "http://2hu.gensokyo/objects/2",
+        "published": "2020-02-12T14:08:20Z",
+        "to": [
+	    "http://2hu.gensokyo/users/marisa"
+        ],
+        "type": "ChatMessage"
+    },
+    "published": "2018-02-12T14:08:20Z",
+    "to": [
+        "http://2hu.gensokyo/users/marisa"
+    ],
+    "type": "Create"
+}
diff --git a/test/web/activity_pub/object_validators/types/recipients_test.exs b/test/web/activity_pub/object_validators/types/recipients_test.exs
new file mode 100644
index 000000000..2f9218774
--- /dev/null
+++ b/test/web/activity_pub/object_validators/types/recipients_test.exs
@@ -0,0 +1,15 @@
+defmodule Pleroma.Web.ObjectValidators.Types.RecipientsTest do
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients
+  use Pleroma.DataCase
+
+  test "it works with a list" do
+    list = ["https://lain.com/users/lain"]
+    assert {:ok, list} == Recipients.cast(list)
+  end
+
+  test "it turns a single string into a list" do
+    recipient = "https://lain.com/users/lain"
+
+    assert {:ok, [recipient]} == Recipients.cast(recipient)
+  end
+end
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
new file mode 100644
index 000000000..aed62c520
--- /dev/null
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  describe "handle_incoming" do
+    test "it insert it" do
+      data =
+        File.read!("test/fixtures/create-chat-message.json")
+        |> Poison.decode!()
+
+      author = insert(:user, ap_id: data["actor"], local: false)
+      recipient = insert(:user, ap_id: List.first(data["to"]), local: false)
+
+      {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data)
+
+      assert activity.actor == author.ap_id
+      assert activity.recipients == [recipient.ap_id, author.ap_id]
+
+      %Object{} = object = Object.get_by_ap_id(activity.data["object"])
+      assert object
+    end
+  end
+end

From 2e78686686f04726ad73749ee744b8a9df91ffb8 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 12:44:20 +0200
Subject: [PATCH 012/283] SideEffects: Handle ChatMessage creation.

---
 lib/pleroma/chat.ex                          | 15 ++++++----
 lib/pleroma/web/activity_pub/builder.ex      | 22 +++++++++++++++
 lib/pleroma/web/activity_pub/side_effects.ex | 29 ++++++++++++++++++++
 test/chat_test.exs                           | 14 ++++++----
 test/web/activity_pub/side_effects_test.exs  | 26 ++++++++++++++++++
 5 files changed, 95 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 07ad62b97..b61bc4c0e 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -18,23 +18,28 @@ defmodule Pleroma.Chat do
   schema "chats" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     field(:recipient, :string)
-    field(:unread, :integer, default: 0)
+    field(:unread, :integer, default: 0, read_after_writes: true)
 
     timestamps()
   end
 
   def creation_cng(struct, params) do
     struct
-    |> cast(params, [:user_id, :recipient])
+    |> cast(params, [:user_id, :recipient, :unread])
     |> validate_required([:user_id, :recipient])
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
-  def get_or_create(user_id, recipient) do
+  def get(user_id, recipient) do
+    __MODULE__
+    |> Repo.get_by(user_id: user_id, recipient: recipient)
+  end
+
+  def bump_or_create(user_id, recipient) do
     %__MODULE__{}
-    |> creation_cng(%{user_id: user_id, recipient: recipient})
+    |> creation_cng(%{user_id: user_id, recipient: recipient, unread: 1})
     |> Repo.insert(
-      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
+      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()], inc: [unread: 1]],
       conflict_target: [:user_id, :recipient]
     )
   end
diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index 429a510b8..f0a6c1e1b 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -10,6 +10,28 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
 
+  def create(actor, object_id, recipients) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "to" => recipients,
+       "object" => object_id,
+       "type" => "Create"
+     }, []}
+  end
+
+  def chat_message(actor, recipient, content) do
+    {:ok,
+     %{
+       "id" => Utils.generate_object_id(),
+       "actor" => actor.ap_id,
+       "type" => "ChatMessage",
+       "to" => [recipient],
+       "content" => content
+     }, []}
+  end
+
   @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()}
   def like(actor, object) do
     object_actor = User.get_cached_by_ap_id(object.data["actor"])
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 666a4e310..594f32700 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -5,8 +5,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   liked object, a `Follow` activity will add the user to the follower
   collection, and so on.
   """
+  alias Pleroma.Chat
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Utils
 
   def handle(object, meta \\ [])
@@ -21,8 +23,35 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
     {:ok, object, meta}
   end
 
+  def handle(%{data: %{"type" => "Create", "object" => object_id}} = activity, meta) do
+    object = Object.get_by_ap_id(object_id)
+
+    {:ok, _object} = handle_object_creation(object)
+
+    {:ok, activity, meta}
+  end
+
   # Nothing to do
   def handle(object, meta) do
     {:ok, object, meta}
   end
+
+  def handle_object_creation(%{data: %{"type" => "ChatMessage"}} = object) do
+    actor = User.get_cached_by_ap_id(object.data["actor"])
+    recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
+
+    [[actor, recipient], [recipient, actor]]
+    |> Enum.each(fn [user, other_user] ->
+      if user.local do
+        Chat.bump_or_create(user.id, other_user.ap_id)
+      end
+    end)
+
+    {:ok, object}
+  end
+
+  # Nothing to do
+  def handle_object_creation(object) do
+    {:ok, object}
+  end
 end
diff --git a/test/chat_test.exs b/test/chat_test.exs
index ca9206802..bb2b46d51 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -14,7 +14,7 @@ test "it creates a chat for a user and recipient" do
       user = insert(:user)
       other_user = insert(:user)
 
-      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
 
       assert chat.id
     end
@@ -23,19 +23,21 @@ test "it returns a chat for a user and recipient if it already exists" do
       user = insert(:user)
       other_user = insert(:user)
 
-      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
-      {:ok, chat_two} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+      {:ok, chat_two} = Chat.bump_or_create(user.id, other_user.ap_id)
 
       assert chat.id == chat_two.id
     end
 
-    test "a returning chat will have an updated `update_at` field" do
+    test "a returning chat will have an updated `update_at` field and an incremented unread count" do
       user = insert(:user)
       other_user = insert(:user)
 
-      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+      assert chat.unread == 1
       :timer.sleep(1500)
-      {:ok, chat_two} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat_two} = Chat.bump_or_create(user.id, other_user.ap_id)
+      assert chat_two.unread == 2
 
       assert chat.id == chat_two.id
       assert chat.updated_at != chat_two.updated_at
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index b67bd14b3..5fd8372b5 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Chat
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
@@ -31,4 +32,29 @@ test "add the like to the original object", %{like: like, user: user} do
       assert user.ap_id in object.data["likes"]
     end
   end
+
+  describe "creation of ChatMessages" do
+    test "it creates a Chat for the local users and bumps the unread count" do
+      author = insert(:user, local: false)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+      {:ok, chat_message_object} = Object.create(chat_message_data)
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+
+      # The remote user won't get a chat
+      chat = Chat.get(author.id, recipient.ap_id)
+      refute chat
+
+      # The local user will get a chat
+      chat = Chat.get(recipient.id, author.ap_id)
+      assert chat
+    end
+  end
 end

From 4b047850718086a6d2edb5b2d94c6f888eba3016 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 12:46:33 +0200
Subject: [PATCH 013/283] SideEffects: Extend ChatMessage test.

---
 test/web/activity_pub/side_effects_test.exs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 5fd8372b5..b629d0d5d 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -55,6 +55,26 @@ test "it creates a Chat for the local users and bumps the unread count" do
       # The local user will get a chat
       chat = Chat.get(recipient.id, author.ap_id)
       assert chat
+
+      author = insert(:user, local: true)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+      {:ok, chat_message_object} = Object.create(chat_message_data)
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+
+      # Both users are local and get the chat
+      chat = Chat.get(author.id, recipient.ap_id)
+      assert chat
+
+      chat = Chat.get(recipient.id, author.ap_id)
+      assert chat
     end
   end
 end

From 8e637ae1a7b75fa08679ae9cf424650fc105de85 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 13:20:16 +0200
Subject: [PATCH 014/283] CommonAPI: Basic ChatMessage support.

---
 lib/pleroma/web/common_api/common_api.ex | 23 +++++++++++++++++++++++
 test/web/common_api/common_api_test.exs  | 21 +++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 636cf3301..39e15adbf 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Conversation.Participation
   alias Pleroma.FollowingRelationship
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -23,6 +24,28 @@ defmodule Pleroma.Web.CommonAPI do
   require Pleroma.Constants
   require Logger
 
+  def post_chat_message(user, recipient, content) do
+    transaction =
+      Repo.transaction(fn ->
+        with {_, {:ok, chat_message_data, _meta}} <-
+               {:build_object, Builder.chat_message(user, recipient.ap_id, content)},
+             {_, {:ok, chat_message_object}} <-
+               {:create_object, Object.create(chat_message_data)},
+             {_, {:ok, create_activity_data, _meta}} <-
+               {:build_create_activity,
+                Builder.create(user, chat_message_object.data["id"], [recipient.ap_id])},
+             {_, {:ok, %Activity{} = activity, _meta}} <-
+               {:common_pipeline, Pipeline.common_pipeline(create_activity_data, local: true)} do
+          {:ok, activity}
+        end
+      end)
+
+    case transaction do
+      {:ok, value} -> value
+      error -> error
+    end
+  end
+
   def follow(follower, followed) do
     timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout])
 
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index f46ad0272..1aea06d24 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.CommonAPITest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Chat
   alias Pleroma.Conversation.Participation
   alias Pleroma.Object
   alias Pleroma.User
@@ -21,6 +22,26 @@ defmodule Pleroma.Web.CommonAPITest do
   setup do: clear_config([:instance, :limit])
   setup do: clear_config([:instance, :max_pinned_statuses])
 
+  describe "posting chat messages" do
+    test "it posts a chat message" do
+      author = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, activity} = CommonAPI.post_chat_message(author, recipient, "a test message")
+
+      assert activity.data["type"] == "Create"
+      assert activity.local
+      object = Object.normalize(activity)
+
+      assert object.data["type"] == "ChatMessage"
+      assert object.data["to"] == [recipient.ap_id]
+      assert object.data["content"] == "a test message"
+
+      assert Chat.get(author.id, recipient.ap_id)
+      assert Chat.get(recipient.id, author.ap_id)
+    end
+  end
+
   test "when replying to a conversation / participation, it will set the correct context id even if no explicit reply_to is given" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"})

From 68abea313d0be49aa6b8d4b980aa361383f991a7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 15:13:55 +0200
Subject: [PATCH 015/283] ChatController: Add creation and return of chats.

---
 lib/pleroma/chat.ex                           | 10 ++++
 .../controllers/chat_controller.ex            | 47 ++++++++++++++++
 lib/pleroma/web/router.ex                     |  7 +++
 .../controllers/chat_controller_test.exs      | 55 +++++++++++++++++++
 4 files changed, 119 insertions(+)
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
 create mode 100644 test/web/pleroma_api/controllers/chat_controller_test.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index b61bc4c0e..2475019d1 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -35,6 +35,16 @@ def get(user_id, recipient) do
     |> Repo.get_by(user_id: user_id, recipient: recipient)
   end
 
+  def get_or_create(user_id, recipient) do
+    %__MODULE__{}
+    |> creation_cng(%{user_id: user_id, recipient: recipient})
+    |> Repo.insert(
+      on_conflict: :nothing,
+      returning: true,
+      conflict_target: [:user_id, :recipient]
+    )
+  end
+
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
     |> creation_cng(%{user_id: user_id, recipient: recipient, unread: 1})
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
new file mode 100644
index 000000000..0ee8bea33
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.Web.PleromaAPI.ChatController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Chat
+  alias Pleroma.Repo
+
+  import Ecto.Query
+
+  def index(%{assigns: %{user: %{id: user_id}}} = conn, _params) do
+    chats =
+      from(c in Chat,
+        where: c.user_id == ^user_id,
+        order_by: [desc: c.updated_at]
+      )
+      |> Repo.all()
+
+    represented_chats =
+      Enum.map(chats, fn chat ->
+        %{
+          id: chat.id,
+          recipient: chat.recipient,
+          unread: chat.unread
+        }
+      end)
+
+    conn
+    |> json(represented_chats)
+  end
+
+  def create(%{assigns: %{user: user}} = conn, params) do
+    recipient = params["ap_id"] |> URI.decode_www_form()
+
+    with {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
+      represented_chat = %{
+        id: chat.id,
+        recipient: chat.recipient,
+        unread: chat.unread
+      }
+
+      conn
+      |> json(represented_chat)
+    end
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 3ecd59cd1..18ce9ee4b 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -284,6 +284,13 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
+    scope [] do
+      pipe_through(:authenticated_api)
+
+      post("/chats/by-ap-id/:ap_id", ChatController, :create)
+      get("/chats", ChatController, :index)
+    end
+
     scope [] do
       pipe_through(:authenticated_api)
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
new file mode 100644
index 000000000..40c09d1cd
--- /dev/null
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
+  use Pleroma.Web.ConnCase, async: true
+
+  alias Pleroma.Chat
+
+  import Pleroma.Factory
+
+  describe "POST /api/v1/pleroma/chats/by-ap-id/:id" do
+    test "it creates or returns a chat", %{conn: conn} do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> post("/api/v1/pleroma/chats/by-ap-id/#{URI.encode_www_form(other_user.ap_id)}")
+        |> json_response(200)
+
+      assert result["id"]
+    end
+  end
+
+  describe "GET /api/v1/pleroma/chats" do
+    test "it return a list of chats the current user is participating in, in descending order of updates",
+         %{conn: conn} do
+      user = insert(:user)
+      har = insert(:user)
+      jafnhar = insert(:user)
+      tridi = insert(:user)
+
+      {:ok, chat_1} = Chat.get_or_create(user.id, har.ap_id)
+      :timer.sleep(1000)
+      {:ok, _chat_2} = Chat.get_or_create(user.id, jafnhar.ap_id)
+      :timer.sleep(1000)
+      {:ok, chat_3} = Chat.get_or_create(user.id, tridi.ap_id)
+      :timer.sleep(1000)
+
+      # bump the second one
+      {:ok, chat_2} = Chat.bump_or_create(user.id, jafnhar.ap_id)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats")
+        |> json_response(200)
+
+      ids = Enum.map(result, & &1["id"])
+
+      assert ids == [chat_2.id, chat_3.id, chat_1.id]
+    end
+  end
+end

From e8fd0dd689be0c7bbca006f7267955329279da98 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 16:59:49 +0200
Subject: [PATCH 016/283] ChatController: Basic support for returning messages.

---
 .../controllers/chat_controller.ex            | 40 +++++++++++++++++++
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 28 +++++++++++++
 3 files changed, 69 insertions(+)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 0ee8bea33..de23b9a22 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -5,10 +5,50 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Chat
+  alias Pleroma.Object
   alias Pleroma.Repo
 
   import Ecto.Query
 
+  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) do
+    with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
+      messages =
+        from(o in Object,
+          where: fragment("?->>'type' = ?", o.data, "ChatMessage"),
+          where:
+            fragment(
+              """
+              (?->>'actor' = ? and ?->'to' = ?) 
+              OR (?->>'actor' = ? and ?->'to' = ?) 
+              """,
+              o.data,
+              ^user.ap_id,
+              o.data,
+              ^[chat.recipient],
+              o.data,
+              ^chat.recipient,
+              o.data,
+              ^[user.ap_id]
+            ),
+          order_by: [desc: o.id]
+        )
+        |> Repo.all()
+
+      represented_messages =
+        messages
+        |> Enum.map(fn message ->
+          %{
+            actor: message.data["actor"],
+            id: message.id,
+            content: message.data["content"]
+          }
+        end)
+
+      conn
+      |> json(represented_messages)
+    end
+  end
+
   def index(%{assigns: %{user: %{id: user_id}}} = conn, _params) do
     chats =
       from(c in Chat,
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 18ce9ee4b..368e77d3e 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -289,6 +289,7 @@ defmodule Pleroma.Web.Router do
 
       post("/chats/by-ap-id/:ap_id", ChatController, :create)
       get("/chats", ChatController, :index)
+      get("/chats/:id/messages", ChatController, :messages)
     end
 
     scope [] do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 40c09d1cd..6b2db5064 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,9 +5,37 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
+  alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
+  describe "GET /api/v1/pleroma/chats/:id/messages" do
+    # TODO
+    # - Test that statuses don't show
+    # - Test the case where it's not the user's chat
+    # - Test the returned data
+    test "it returns the messages for a given chat", %{conn: conn} do
+      user = insert(:user)
+      other_user = insert(:user)
+      third_user = insert(:user)
+
+      {:ok, _} = CommonAPI.post_chat_message(user, other_user, "hey")
+      {:ok, _} = CommonAPI.post_chat_message(user, third_user, "hey")
+      {:ok, _} = CommonAPI.post_chat_message(user, other_user, "how are you?")
+      {:ok, _} = CommonAPI.post_chat_message(other_user, user, "fine, how about you?")
+
+      chat = Chat.get(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
+        |> json_response(200)
+
+      assert length(result) == 3
+    end
+  end
+
   describe "POST /api/v1/pleroma/chats/by-ap-id/:id" do
     test "it creates or returns a chat", %{conn: conn} do
       user = insert(:user)

From 2cc68414245805dc3b83c200798e424f139e71fc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 9 Apr 2020 17:18:31 +0200
Subject: [PATCH 017/283] ChatController: Basic message posting.

---
 .../controllers/chat_controller.ex            | 26 +++++++++++++++++++
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 17 ++++++++++++
 3 files changed, 44 insertions(+)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index de23b9a22..972330f4e 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -7,9 +7,35 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Chat
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
 
   import Ecto.Query
 
+  # TODO
+  # - Oauth stuff
+  # - Views / Representers
+  # - Error handling
+
+  def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
+        "id" => id,
+        "content" => content
+      }) do
+    with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
+         %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
+         {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, content),
+         message <- Object.normalize(activity) do
+      represented_message = %{
+        actor: message.data["actor"],
+        id: message.id,
+        content: message.data["content"]
+      }
+
+      conn
+      |> json(represented_message)
+    end
+  end
+
   def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       messages =
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 368e77d3e..ce69725dc 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -290,6 +290,7 @@ defmodule Pleroma.Web.Router do
       post("/chats/by-ap-id/:ap_id", ChatController, :create)
       get("/chats", ChatController, :index)
       get("/chats/:id/messages", ChatController, :messages)
+      post("/chats/:id/messages", ChatController, :post_chat_message)
     end
 
     scope [] do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 6b2db5064..b4230e5ad 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -9,6 +9,23 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   import Pleroma.Factory
 
+  describe "POST /api/v1/pleroma/chats/:id/messages" do
+    test "it posts a message to the chat", %{conn: conn} do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
+        |> json_response(200)
+
+      assert result["content"] == "Hallo!!"
+    end
+  end
+
   describe "GET /api/v1/pleroma/chats/:id/messages" do
     # TODO
     # - Test that statuses don't show

From 64c78581fe397b6d9356c52cf3f43becd2ff3b4e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 10 Apr 2020 14:47:56 +0200
Subject: [PATCH 018/283] Chat: Only create them for valid users for now.

---
 lib/pleroma/chat.ex | 7 +++++++
 test/chat_test.exs  | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 2475019d1..c2044881f 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -26,6 +26,13 @@ defmodule Pleroma.Chat do
   def creation_cng(struct, params) do
     struct
     |> cast(params, [:user_id, :recipient, :unread])
+    |> validate_change(:recipient, fn
+      :recipient, recipient ->
+        case User.get_cached_by_ap_id(recipient) do
+          nil -> [recipient: "must a an existing user"]
+          _ -> []
+        end
+    end)
     |> validate_required([:user_id, :recipient])
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
diff --git a/test/chat_test.exs b/test/chat_test.exs
index bb2b46d51..952598c87 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -10,6 +10,13 @@ defmodule Pleroma.ChatTest do
   import Pleroma.Factory
 
   describe "creation and getting" do
+    test "it only works if the recipient is a valid user (for now)" do
+      user = insert(:user)
+
+      assert {:error, _chat} = Chat.bump_or_create(user.id, "http://some/nonexisting/account")
+      assert {:error, _chat} = Chat.get_or_create(user.id, "http://some/nonexisting/account")
+    end
+
     test "it creates a chat for a user and recipient" do
       user = insert(:user)
       other_user = insert(:user)

From 6ace22b56a3ced833bd990de5715048d6bd32f80 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 15 Apr 2020 18:23:16 +0200
Subject: [PATCH 019/283] Chat: Add views, don't return them in timeline
 queries.

---
 lib/pleroma/web/activity_pub/activity_pub.ex  | 13 +++
 .../web/api_spec/operations/chat_operation.ex | 81 +++++++++++++++++++
 lib/pleroma/web/common_api/common_api.ex      |  2 +-
 .../controllers/chat_controller.ex            | 47 +++--------
 .../pleroma_api/views/chat_message_view.ex    | 28 +++++++
 .../web/pleroma_api/views/chat_view.ex        | 21 +++++
 .../controllers/timeline_controller_test.exs  |  3 +
 .../controllers/chat_controller_test.exs      |  8 +-
 .../views/chat_message_view_test.exs          | 42 ++++++++++
 9 files changed, 207 insertions(+), 38 deletions(-)
 create mode 100644 lib/pleroma/web/api_spec/operations/chat_operation.ex
 create mode 100644 lib/pleroma/web/pleroma_api/views/chat_message_view.ex
 create mode 100644 lib/pleroma/web/pleroma_api/views/chat_view.ex
 create mode 100644 test/web/pleroma_api/views/chat_message_view_test.exs

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 4a56beb73..b6ba91052 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1207,6 +1207,18 @@ defp exclude_poll_votes(query, _) do
     end
   end
 
+  defp exclude_chat_messages(query, %{"include_chat_messages" => true}), do: query
+
+  defp exclude_chat_messages(query, _) do
+    if has_named_binding?(query, :object) do
+      from([activity, object: o] in query,
+        where: fragment("not(?->>'type' = ?)", o.data, "ChatMessage")
+      )
+    else
+      query
+    end
+  end
+
   defp exclude_id(query, %{"exclude_id" => id}) when is_binary(id) do
     from(activity in query, where: activity.id != ^id)
   end
@@ -1312,6 +1324,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_instance(opts)
     |> Activity.restrict_deactivated_users()
     |> exclude_poll_votes(opts)
+    |> exclude_chat_messages(opts)
     |> exclude_visibility(opts)
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
new file mode 100644
index 000000000..038ebb29d
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.ChatOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Create a chat",
+      responses: %{
+        200 =>
+          Operation.response("Chat", "application/json", %Schema{
+            type: :object,
+            description: "A created chat is returned",
+            properties: %{
+              id: %Schema{type: :integer}
+            }
+          })
+      }
+    }
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Get a list of chats that you participated in",
+      responses: %{
+        200 =>
+          Operation.response("Chats", "application/json", %Schema{
+            type: :array,
+            description: "A list of chats",
+            items: %Schema{
+              type: :object,
+              description: "A chat"
+            }
+          })
+      }
+    }
+  end
+
+  def messages_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Get the most recent messages of the chat",
+      responses: %{
+        200 =>
+          Operation.response("Messages", "application/json", %Schema{
+            type: :array,
+            description: "A list of chat messages",
+            items: %Schema{
+              type: :object,
+              description: "A chat message"
+            }
+          })
+      }
+    }
+  end
+
+  def post_chat_message_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Post a message to the chat",
+      responses: %{
+        200 =>
+          Operation.response("Message", "application/json", %Schema{
+            type: :object,
+            description: "A chat message"
+          })
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 2f13daf0c..c306c1e96 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.CommonAPI do
   require Pleroma.Constants
   require Logger
 
-  def post_chat_message(user, recipient, content) do
+  def post_chat_message(%User{} = user, %User{} = recipient, content) do
     transaction =
       Repo.transaction(fn ->
         with {_, {:ok, chat_message_data, _meta}} <-
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 972330f4e..5ec546847 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -9,6 +9,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.PleromaAPI.ChatView
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
 
   import Ecto.Query
 
@@ -17,6 +19,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   # - Views / Representers
   # - Error handling
 
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
+
   def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
         "id" => id,
         "content" => content
@@ -25,14 +29,9 @@ def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
          {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, content),
          message <- Object.normalize(activity) do
-      represented_message = %{
-        actor: message.data["actor"],
-        id: message.id,
-        content: message.data["content"]
-      }
-
       conn
-      |> json(represented_message)
+      |> put_view(ChatMessageView)
+      |> render("show.json", for: user, object: message, chat: chat)
     end
   end
 
@@ -60,18 +59,9 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) d
         )
         |> Repo.all()
 
-      represented_messages =
-        messages
-        |> Enum.map(fn message ->
-          %{
-            actor: message.data["actor"],
-            id: message.id,
-            content: message.data["content"]
-          }
-        end)
-
       conn
-      |> json(represented_messages)
+      |> put_view(ChatMessageView)
+      |> render("index.json", for: user, objects: messages, chat: chat)
     end
   end
 
@@ -83,31 +73,18 @@ def index(%{assigns: %{user: %{id: user_id}}} = conn, _params) do
       )
       |> Repo.all()
 
-    represented_chats =
-      Enum.map(chats, fn chat ->
-        %{
-          id: chat.id,
-          recipient: chat.recipient,
-          unread: chat.unread
-        }
-      end)
-
     conn
-    |> json(represented_chats)
+    |> put_view(ChatView)
+    |> render("index.json", chats: chats)
   end
 
   def create(%{assigns: %{user: user}} = conn, params) do
     recipient = params["ap_id"] |> URI.decode_www_form()
 
     with {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
-      represented_chat = %{
-        id: chat.id,
-        recipient: chat.recipient,
-        unread: chat.unread
-      }
-
       conn
-      |> json(represented_chat)
+      |> put_view(ChatView)
+      |> render("show.json", chat: chat)
     end
   end
 end
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
new file mode 100644
index 000000000..2df591358
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Chat
+
+  def render(
+        "show.json",
+        %{
+          object: %{id: id, data: %{"type" => "ChatMessage"} = chat_message},
+          chat: %Chat{id: chat_id}
+        }
+      ) do
+    %{
+      id: id,
+      content: chat_message["content"],
+      chat_id: chat_id,
+      actor: chat_message["actor"]
+    }
+  end
+
+  def render("index.json", opts) do
+    render_many(opts[:objects], __MODULE__, "show.json", Map.put(opts, :as, :object))
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
new file mode 100644
index 000000000..ee48385bf
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.ChatView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Chat
+
+  def render("show.json", %{chat: %Chat{} = chat}) do
+    %{
+      id: chat.id,
+      recipient: chat.recipient,
+      unread: chat.unread
+    }
+  end
+
+  def render("index.json", %{chats: chats}) do
+    render_many(chats, __MODULE__, "show.json")
+  end
+end
diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs
index 06efdc901..a5c227991 100644
--- a/test/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -51,6 +51,9 @@ test "the home timeline", %{user: user, conn: conn} do
       {:ok, activity} = CommonAPI.post(third_user, %{"status" => "repeated post"})
       {:ok, _, _} = CommonAPI.repeat(activity.id, following)
 
+      # This one should not show up in the TL
+      {:ok, _activity} = CommonAPI.post_chat_message(third_user, user, ":gun:")
+
       ret_conn = get(conn, uri)
 
       assert Enum.empty?(json_response(ret_conn, :ok))
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index b4230e5ad..dad37a889 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -23,14 +23,13 @@ test "it posts a message to the chat", %{conn: conn} do
         |> json_response(200)
 
       assert result["content"] == "Hallo!!"
+      assert result["chat_id"] == chat.id
     end
   end
 
   describe "GET /api/v1/pleroma/chats/:id/messages" do
     # TODO
-    # - Test that statuses don't show
     # - Test the case where it's not the user's chat
-    # - Test the returned data
     test "it returns the messages for a given chat", %{conn: conn} do
       user = insert(:user)
       other_user = insert(:user)
@@ -49,6 +48,11 @@ test "it returns the messages for a given chat", %{conn: conn} do
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
         |> json_response(200)
 
+      result
+      |> Enum.each(fn message ->
+        assert message["chat_id"] == chat.id
+      end)
+
       assert length(result) == 3
     end
   end
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
new file mode 100644
index 000000000..e690da022
--- /dev/null
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Chat
+  alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
+
+  import Pleroma.Factory
+
+  test "it displays a chat message" do
+    user = insert(:user)
+    recipient = insert(:user)
+    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis")
+
+    chat = Chat.get(user.id, recipient.ap_id)
+
+    object = Object.normalize(activity)
+
+    chat_message = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+
+    assert chat_message[:id] == object.id
+    assert chat_message[:content] == "kippis"
+    assert chat_message[:actor] == user.ap_id
+    assert chat_message[:chat_id]
+
+    {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk")
+
+    object = Object.normalize(activity)
+
+    chat_message_two = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+
+    assert chat_message_two[:id] == object.id
+    assert chat_message_two[:content] == "gkgkgk"
+    assert chat_message_two[:actor] == recipient.ap_id
+    assert chat_message_two[:chat_id] == chat_message[:chat_id]
+  end
+end

From 3d4eca5dd4be297f03c244497d78db03e82a9d81 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 16 Apr 2020 12:56:29 +0200
Subject: [PATCH 020/283] CommonAPI: Escape HTML for chat messages.

---
 lib/pleroma/web/common_api/common_api.ex |  8 +++++++-
 test/web/common_api/common_api_test.exs  | 11 +++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index c306c1e96..2c25850db 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -17,6 +17,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Formatter
 
   import Pleroma.Web.Gettext
   import Pleroma.Web.CommonAPI.Utils
@@ -28,7 +29,12 @@ def post_chat_message(%User{} = user, %User{} = recipient, content) do
     transaction =
       Repo.transaction(fn ->
         with {_, {:ok, chat_message_data, _meta}} <-
-               {:build_object, Builder.chat_message(user, recipient.ap_id, content)},
+               {:build_object,
+                Builder.chat_message(
+                  user,
+                  recipient.ap_id,
+                  content |> Formatter.html_escape("text/plain")
+                )},
              {_, {:ok, chat_message_object}} <-
                {:create_object, Object.create(chat_message_data)},
              {_, {:ok, create_activity_data, _meta}} <-
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 168721c81..abe3e6f8d 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -27,7 +27,12 @@ test "it posts a chat message" do
       author = insert(:user)
       recipient = insert(:user)
 
-      {:ok, activity} = CommonAPI.post_chat_message(author, recipient, "a test message")
+      {:ok, activity} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          "a test message <script>alert('uuu')</script>"
+        )
 
       assert activity.data["type"] == "Create"
       assert activity.local
@@ -35,7 +40,9 @@ test "it posts a chat message" do
 
       assert object.data["type"] == "ChatMessage"
       assert object.data["to"] == [recipient.ap_id]
-      assert object.data["content"] == "a test message"
+
+      assert object.data["content"] ==
+               "a test message &lt;script&gt;alert(&#39;uuu&#39;)&lt;/script&gt;"
 
       assert Chat.get(author.id, recipient.ap_id)
       assert Chat.get(recipient.id, author.ap_id)

From e2ced0491770d6260fe51d5144b81200fd97f268 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 16 Apr 2020 15:21:47 +0200
Subject: [PATCH 021/283] ChatMessages: Better validation.

---
 .../web/activity_pub/object_validator.ex      |  6 ++-
 .../chat_message_validator.ex                 | 26 ++++++++++
 .../object_validators/common_validations.ex   |  6 ++-
 .../create_chat_message_validator.ex          |  5 ++
 .../transmogrifier/chat_message_handling.ex   |  3 ++
 test/fixtures/create-chat-message.json        |  2 +-
 .../activity_pub/object_validator_test.exs    | 52 +++++++++++++++++++
 .../transmogrifier/chat_message_test.exs      | 34 +++++++++++-
 8 files changed, 128 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 49cc72561..259bbeb64 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -31,7 +31,8 @@ def validate(%{"type" => "Like"} = object, meta) do
   def validate(%{"type" => "ChatMessage"} = object, meta) do
     with {:ok, object} <-
            object
-           |> ChatMessageValidator.cast_and_apply() do
+           |> ChatMessageValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
     end
@@ -40,7 +41,8 @@ def validate(%{"type" => "ChatMessage"} = object, meta) do
   def validate(%{"type" => "Create"} = object, meta) do
     with {:ok, object} <-
            object
-           |> CreateChatMessageValidator.cast_and_apply() do
+           |> CreateChatMessageValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
     end
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index ab5be3596..a4e4460cd 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   use Ecto.Schema
 
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.User
 
   import Ecto.Changeset
 
@@ -54,5 +55,30 @@ def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["ChatMessage"])
     |> validate_required([:id, :actor, :to, :type, :content])
+    |> validate_length(:to, is: 1)
+    |> validate_local_concern()
+  end
+
+  @doc "Validates if at least one of the users in this ChatMessage is a local user, otherwise we don't want the message in our system. It also validates the presence of both users in our system."
+  def validate_local_concern(cng) do
+    with actor_ap <- get_field(cng, :actor),
+         {_, %User{} = actor} <- {:find_actor, User.get_cached_by_ap_id(actor_ap)},
+         {_, %User{} = recipient} <-
+           {:find_recipient, User.get_cached_by_ap_id(get_field(cng, :to) |> hd())},
+         {_, true} <- {:local?, Enum.any?([actor, recipient], & &1.local)} do
+      cng
+    else
+      {:local?, false} ->
+        cng
+        |> add_error(:actor, "actor and recipient are both remote")
+
+      {:find_actor, _} ->
+        cng
+        |> add_error(:actor, "can't find user")
+
+      {:find_recipient, _} ->
+        cng
+        |> add_error(:to, "can't find user")
+    end
   end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
index b479c3918..02f3a6438 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -8,7 +8,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   alias Pleroma.Object
   alias Pleroma.User
 
-  def validate_actor_presence(cng, field_name \\ :actor) do
+  def validate_actor_presence(cng) do
+    validate_user_presence(cng, :actor)
+  end
+
+  def validate_user_presence(cng, field_name) do
     cng
     |> validate_change(field_name, fn field_name, actor ->
       if User.get_cached_by_ap_id(actor) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index 659311480..ce52d5623 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -32,4 +32,9 @@ def cast_and_apply(data) do
   def cast_data(data) do
     cast(%__MODULE__{}, data, __schema__(:fields))
   end
+
+  # No validation yet
+  def cast_and_validate(data) do
+    cast_data(data)
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index b5843736f..815b866c9 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -25,6 +25,9 @@ def handle_incoming(
          {_, {:ok, activity, _meta}} <-
            {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do
       {:ok, activity}
+    else
+      e ->
+        {:error, e}
     end
   end
 end
diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json
index 4aa17f4a5..2e4608f43 100644
--- a/test/fixtures/create-chat-message.json
+++ b/test/fixtures/create-chat-message.json
@@ -3,7 +3,7 @@
     "id": "http://2hu.gensokyo/objects/1",
     "object": {
         "attributedTo": "http://2hu.gensokyo/users/raymoo",
-        "content": "You expected a cute girl? Too bad.",
+        "content": "You expected a cute girl? Too bad. <script>alert('XSS')</script>",
         "id": "http://2hu.gensokyo/objects/2",
         "published": "2020-02-12T14:08:20Z",
         "to": [
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 3c5c3696e..bf0bfdfaf 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -5,9 +5,61 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.ActivityPub.Builder
 
   import Pleroma.Factory
 
+  describe "chat messages" do
+    setup do
+      user = insert(:user)
+      recipient = insert(:user, local: false)
+
+      {:ok, valid_chat_message, _} = Builder.chat_message(user, recipient.ap_id, "hey")
+
+      %{user: user, recipient: recipient, valid_chat_message: valid_chat_message}
+    end
+
+    test "validates for a basic object we build", %{valid_chat_message: valid_chat_message} do
+      assert {:ok, _object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+    end
+
+    test "does not validate if the actor or the recipient is not in our system", %{
+      valid_chat_message: valid_chat_message
+    } do
+      chat_message =
+        valid_chat_message
+        |> Map.put("actor", "https://raymoo.com/raymoo")
+
+      {:error, _} = ObjectValidator.validate(chat_message, [])
+
+      chat_message =
+        valid_chat_message
+        |> Map.put("to", ["https://raymoo.com/raymoo"])
+
+      {:error, _} = ObjectValidator.validate(chat_message, [])
+    end
+
+    test "does not validate for a message with multiple recipients", %{
+      valid_chat_message: valid_chat_message,
+      user: user,
+      recipient: recipient
+    } do
+      chat_message =
+        valid_chat_message
+        |> Map.put("to", [user.ap_id, recipient.ap_id])
+
+      assert {:error, _} = ObjectValidator.validate(chat_message, [])
+    end
+
+    test "does not validate if it doesn't concern local users" do
+      user = insert(:user, local: false)
+      recipient = insert(:user, local: false)
+
+      {:ok, valid_chat_message, _} = Builder.chat_message(user, recipient.ap_id, "hey")
+      assert {:error, _} = ObjectValidator.validate(valid_chat_message, [])
+    end
+  end
+
   describe "likes" do
     setup do
       user = insert(:user)
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index aed62c520..5b238f9c4 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -12,13 +12,43 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   describe "handle_incoming" do
-    test "it insert it" do
+    test "it rejects messages that don't contain content" do
+      data =
+        File.read!("test/fixtures/create-chat-message.json")
+        |> Poison.decode!()
+
+      object =
+        data["object"]
+        |> Map.delete("content")
+
+      data =
+        data
+        |> Map.put("object", object)
+
+      _author = insert(:user, ap_id: data["actor"], local: false)
+      _recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
+
+      {:error, _} = Transmogrifier.handle_incoming(data)
+    end
+
+    test "it rejects messages that don't concern local users" do
+      data =
+        File.read!("test/fixtures/create-chat-message.json")
+        |> Poison.decode!()
+
+      _author = insert(:user, ap_id: data["actor"], local: false)
+      _recipient = insert(:user, ap_id: List.first(data["to"]), local: false)
+
+      {:error, _} = Transmogrifier.handle_incoming(data)
+    end
+
+    test "it inserts it and creates a chat" do
       data =
         File.read!("test/fixtures/create-chat-message.json")
         |> Poison.decode!()
 
       author = insert(:user, ap_id: data["actor"], local: false)
-      recipient = insert(:user, ap_id: List.first(data["to"]), local: false)
+      recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
 
       {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data)
 

From ca598e9c27a7a66b014523845e62046d19364f2f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 16 Apr 2020 15:27:35 +0200
Subject: [PATCH 022/283] AccountView: Return user ap_id.

---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 1 +
 test/web/mastodon_api/views/account_view_test.exs  | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 8fb96a22a..f20453744 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -234,6 +234,7 @@ defp do_render("show.json", %{user: user} = opts) do
 
       # Pleroma extension
       pleroma: %{
+        ap_id: user.ap_id,
         confirmation_pending: user.confirmation_pending,
         tags: user.tags,
         hide_followers_count: user.hide_followers_count,
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index 4435f69ff..2be0d8d0f 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -82,6 +82,7 @@ test "Represent a user account" do
         fields: []
       },
       pleroma: %{
+        ap_id: user.ap_id,
         background_image: "https://example.com/images/asuka_hospital.png",
         confirmation_pending: false,
         tags: [],
@@ -152,6 +153,7 @@ test "Represent a Service(bot) account" do
         fields: []
       },
       pleroma: %{
+        ap_id: user.ap_id,
         background_image: nil,
         confirmation_pending: false,
         tags: [],
@@ -351,6 +353,7 @@ test "represent an embedded relationship" do
         fields: []
       },
       pleroma: %{
+        ap_id: user.ap_id,
         background_image: nil,
         confirmation_pending: false,
         tags: [],

From e983f708846a5784e23b7e18734a61ed7f6e3636 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 16 Apr 2020 17:50:24 +0200
Subject: [PATCH 023/283] ChatMessagesHandling: Strip HTML of incoming
 messages.

---
 .../web/activity_pub/transmogrifier/chat_message_handling.ex   | 3 +++
 test/web/activity_pub/transmogrifier/chat_message_test.exs     | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index 815b866c9..11bd10456 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -19,6 +19,9 @@ def handle_incoming(
          {_, {:ok, object_cast_data_sym}} <-
            {:casting_object_data, object_data |> ChatMessageValidator.cast_and_apply()},
          object_cast_data = ObjectValidator.stringify_keys(object_cast_data_sym),
+         # For now, just strip HTML
+         stripped_content = Pleroma.HTML.strip_tags(object_cast_data["content"]),
+         object_cast_data = object_cast_data |> Map.put("content", stripped_content),
          {_, {:ok, validated_object, _meta}} <-
            {:validate_object, ObjectValidator.validate(object_cast_data, %{})},
          {_, {:ok, _created_object}} <- {:persist_object, Object.create(validated_object)},
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index 5b238f9c4..7e7f9ebec 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -56,7 +56,9 @@ test "it inserts it and creates a chat" do
       assert activity.recipients == [recipient.ap_id, author.ap_id]
 
       %Object{} = object = Object.get_by_ap_id(activity.data["object"])
+
       assert object
+      assert object.data["content"] == "You expected a cute girl? Too bad. alert(&#39;XSS&#39;)"
     end
   end
 end

From f8c3ae7a627817789776f11497041445bb273c19 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 16 Apr 2020 18:43:31 +0200
Subject: [PATCH 024/283] ChatController: Handle pagination.

---
 .../controllers/chat_controller.ex            | 12 ++--
 .../pleroma_api/views/chat_message_view.ex    |  4 +-
 .../web/pleroma_api/views/chat_view.ex        |  2 +-
 .../controllers/chat_controller_test.exs      | 62 ++++++++++++++++++-
 4 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 5ec546847..8cf8d82e4 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Pagination
 
   import Ecto.Query
 
@@ -35,7 +36,7 @@ def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
     end
   end
 
-  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) do
+  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       messages =
         from(o in Object,
@@ -54,10 +55,9 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) d
               ^chat.recipient,
               o.data,
               ^[user.ap_id]
-            ),
-          order_by: [desc: o.id]
+            )
         )
-        |> Repo.all()
+        |> Pagination.fetch_paginated(params)
 
       conn
       |> put_view(ChatMessageView)
@@ -65,13 +65,13 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id}) d
     end
   end
 
-  def index(%{assigns: %{user: %{id: user_id}}} = conn, _params) do
+  def index(%{assigns: %{user: %{id: user_id}}} = conn, params) do
     chats =
       from(c in Chat,
         where: c.user_id == ^user_id,
         order_by: [desc: c.updated_at]
       )
-      |> Repo.all()
+      |> Pagination.fetch_paginated(params)
 
     conn
     |> put_view(ChatView)
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index 2df591358..fdbb9ff1b 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -15,9 +15,9 @@ def render(
         }
       ) do
     %{
-      id: id,
+      id: id |> to_string(),
       content: chat_message["content"],
-      chat_id: chat_id,
+      chat_id: chat_id |> to_string(),
       actor: chat_message["actor"]
     }
   end
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index ee48385bf..7b8c6450a 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
 
   def render("show.json", %{chat: %Chat{} = chat}) do
     %{
-      id: chat.id,
+      id: chat.id |> to_string(),
       recipient: chat.recipient,
       unread: chat.unread
     }
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index dad37a889..f30fd6615 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -23,11 +23,38 @@ test "it posts a message to the chat", %{conn: conn} do
         |> json_response(200)
 
       assert result["content"] == "Hallo!!"
-      assert result["chat_id"] == chat.id
+      assert result["chat_id"] == chat.id |> to_string()
     end
   end
 
   describe "GET /api/v1/pleroma/chats/:id/messages" do
+    test "it paginates", %{conn: conn} do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      Enum.each(1..30, fn _ ->
+        {:ok, _} = CommonAPI.post_chat_message(user, recipient, "hey")
+      end)
+
+      chat = Chat.get(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
+        |> json_response(200)
+
+      assert length(result) == 20
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats/#{chat.id}/messages", %{"max_id" => List.last(result)["id"]})
+        |> json_response(200)
+
+      assert length(result) == 10
+    end
+
     # TODO
     # - Test the case where it's not the user's chat
     test "it returns the messages for a given chat", %{conn: conn} do
@@ -50,7 +77,7 @@ test "it returns the messages for a given chat", %{conn: conn} do
 
       result
       |> Enum.each(fn message ->
-        assert message["chat_id"] == chat.id
+        assert message["chat_id"] == chat.id |> to_string()
       end)
 
       assert length(result) == 3
@@ -73,6 +100,31 @@ test "it creates or returns a chat", %{conn: conn} do
   end
 
   describe "GET /api/v1/pleroma/chats" do
+    test "it paginates", %{conn: conn} do
+      user = insert(:user)
+
+      Enum.each(1..30, fn _ ->
+        recipient = insert(:user)
+        {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
+      end)
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats")
+        |> json_response(200)
+
+      assert length(result) == 20
+
+      result =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/pleroma/chats", %{max_id: List.last(result)["id"]})
+        |> json_response(200)
+
+      assert length(result) == 10
+    end
+
     test "it return a list of chats the current user is participating in, in descending order of updates",
          %{conn: conn} do
       user = insert(:user)
@@ -98,7 +150,11 @@ test "it return a list of chats the current user is participating in, in descend
 
       ids = Enum.map(result, & &1["id"])
 
-      assert ids == [chat_2.id, chat_3.id, chat_1.id]
+      assert ids == [
+               chat_2.id |> to_string(),
+               chat_3.id |> to_string(),
+               chat_1.id |> to_string()
+             ]
     end
   end
 end

From d45ae6485811189e98f774ecdb46f0ccdfa8b2b3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 Apr 2020 13:04:46 +0200
Subject: [PATCH 025/283] ChatController: Use OAuth scopes.

---
 .../controllers/chat_controller.ex            | 18 +++++++-
 .../controllers/chat_controller_test.exs      | 41 +++++++++----------
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 8cf8d82e4..31c723426 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
@@ -16,10 +17,18 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   import Ecto.Query
 
   # TODO
-  # - Oauth stuff
-  # - Views / Representers
   # - Error handling
 
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:statuses"]} when action in [:post_chat_message, :create]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:statuses"]} when action in [:messages, :index]
+  )
+
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
   def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
@@ -62,6 +71,11 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id} =
       conn
       |> put_view(ChatMessageView)
       |> render("index.json", for: user, objects: messages, chat: chat)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(%{error: "not found"})
     end
   end
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index f30fd6615..0750c7273 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -10,15 +10,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   import Pleroma.Factory
 
   describe "POST /api/v1/pleroma/chats/:id/messages" do
-    test "it posts a message to the chat", %{conn: conn} do
-      user = insert(:user)
+    setup do: oauth_access(["write:statuses"])
+
+    test "it posts a message to the chat", %{conn: conn, user: user} do
       other_user = insert(:user)
 
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
 
       result =
         conn
-        |> assign(:user, user)
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
         |> json_response(200)
 
@@ -28,8 +28,9 @@ test "it posts a message to the chat", %{conn: conn} do
   end
 
   describe "GET /api/v1/pleroma/chats/:id/messages" do
-    test "it paginates", %{conn: conn} do
-      user = insert(:user)
+    setup do: oauth_access(["read:statuses"])
+
+    test "it paginates", %{conn: conn, user: user} do
       recipient = insert(:user)
 
       Enum.each(1..30, fn _ ->
@@ -40,7 +41,6 @@ test "it paginates", %{conn: conn} do
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
         |> json_response(200)
 
@@ -48,17 +48,13 @@ test "it paginates", %{conn: conn} do
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages", %{"max_id" => List.last(result)["id"]})
         |> json_response(200)
 
       assert length(result) == 10
     end
 
-    # TODO
-    # - Test the case where it's not the user's chat
-    test "it returns the messages for a given chat", %{conn: conn} do
-      user = insert(:user)
+    test "it returns the messages for a given chat", %{conn: conn, user: user} do
       other_user = insert(:user)
       third_user = insert(:user)
 
@@ -71,7 +67,6 @@ test "it returns the messages for a given chat", %{conn: conn} do
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
         |> json_response(200)
 
@@ -81,17 +76,25 @@ test "it returns the messages for a given chat", %{conn: conn} do
       end)
 
       assert length(result) == 3
+
+      # Trying to get the chat of a different user
+      result =
+        conn
+        |> assign(:user, other_user)
+        |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
+
+      assert result |> json_response(404)
     end
   end
 
   describe "POST /api/v1/pleroma/chats/by-ap-id/:id" do
+    setup do: oauth_access(["write:statuses"])
+
     test "it creates or returns a chat", %{conn: conn} do
-      user = insert(:user)
       other_user = insert(:user)
 
       result =
         conn
-        |> assign(:user, user)
         |> post("/api/v1/pleroma/chats/by-ap-id/#{URI.encode_www_form(other_user.ap_id)}")
         |> json_response(200)
 
@@ -100,9 +103,9 @@ test "it creates or returns a chat", %{conn: conn} do
   end
 
   describe "GET /api/v1/pleroma/chats" do
-    test "it paginates", %{conn: conn} do
-      user = insert(:user)
+    setup do: oauth_access(["read:statuses"])
 
+    test "it paginates", %{conn: conn, user: user} do
       Enum.each(1..30, fn _ ->
         recipient = insert(:user)
         {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
@@ -110,7 +113,6 @@ test "it paginates", %{conn: conn} do
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats")
         |> json_response(200)
 
@@ -118,7 +120,6 @@ test "it paginates", %{conn: conn} do
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats", %{max_id: List.last(result)["id"]})
         |> json_response(200)
 
@@ -126,8 +127,7 @@ test "it paginates", %{conn: conn} do
     end
 
     test "it return a list of chats the current user is participating in, in descending order of updates",
-         %{conn: conn} do
-      user = insert(:user)
+         %{conn: conn, user: user} do
       har = insert(:user)
       jafnhar = insert(:user)
       tridi = insert(:user)
@@ -144,7 +144,6 @@ test "it return a list of chats the current user is participating in, in descend
 
       result =
         conn
-        |> assign(:user, user)
         |> get("/api/v1/pleroma/chats")
         |> json_response(200)
 

From 372614cfd3119b589c9c47619445714e8ae6c07e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 Apr 2020 14:23:59 +0200
Subject: [PATCH 026/283] ChatView: Add a mastodon api representation of the
 recipient.

---
 .../transmogrifier/chat_message_handling.ex   |  1 +
 .../web/pleroma_api/views/chat_view.ex        |  7 ++++-
 .../transmogrifier/chat_message_test.exs      | 19 ++++++++++++
 test/web/pleroma_api/views/chat_view_test.exs | 29 +++++++++++++++++++
 4 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 test/web/pleroma_api/views/chat_view_test.exs

diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index 11bd10456..cfe3b767b 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -22,6 +22,7 @@ def handle_incoming(
          # For now, just strip HTML
          stripped_content = Pleroma.HTML.strip_tags(object_cast_data["content"]),
          object_cast_data = object_cast_data |> Map.put("content", stripped_content),
+         {_, true} <- {:to_fields_match, cast_data["to"] == object_cast_data["to"]},
          {_, {:ok, validated_object, _meta}} <-
            {:validate_object, ObjectValidator.validate(object_cast_data, %{})},
          {_, {:ok, _created_object}} <- {:persist_object, Object.create(validated_object)},
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 7b8c6450a..1e9ef4356 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -6,11 +6,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   use Pleroma.Web, :view
 
   alias Pleroma.Chat
+  alias Pleroma.User
+  alias Pleroma.Web.MastodonAPI.AccountView
+
+  def render("show.json", %{chat: %Chat{} = chat} = opts) do
+    recipient = User.get_cached_by_ap_id(chat.recipient)
 
-  def render("show.json", %{chat: %Chat{} = chat}) do
     %{
       id: chat.id |> to_string(),
       recipient: chat.recipient,
+      recipient_account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
       unread: chat.unread
     }
   end
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index 7e7f9ebec..4d6f24609 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
   import Pleroma.Factory
 
   alias Pleroma.Activity
+  alias Pleroma.Chat
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
@@ -42,6 +43,21 @@ test "it rejects messages that don't concern local users" do
       {:error, _} = Transmogrifier.handle_incoming(data)
     end
 
+    test "it rejects messages where the `to` field of activity and object don't match" do
+      data =
+        File.read!("test/fixtures/create-chat-message.json")
+        |> Poison.decode!()
+
+      author = insert(:user, ap_id: data["actor"])
+      _recipient = insert(:user, ap_id: List.first(data["to"]))
+
+      data =
+        data
+        |> Map.put("to", author.ap_id)
+
+      {:error, _} = Transmogrifier.handle_incoming(data)
+    end
+
     test "it inserts it and creates a chat" do
       data =
         File.read!("test/fixtures/create-chat-message.json")
@@ -59,6 +75,9 @@ test "it inserts it and creates a chat" do
 
       assert object
       assert object.data["content"] == "You expected a cute girl? Too bad. alert(&#39;XSS&#39;)"
+
+      refute Chat.get(author.id, recipient.ap_id)
+      assert Chat.get(recipient.id, author.ap_id)
     end
   end
 end
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
new file mode 100644
index 000000000..1eb0c6241
--- /dev/null
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -0,0 +1,29 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Chat
+  alias Pleroma.Web.PleromaAPI.ChatView
+  alias Pleroma.Web.MastodonAPI.AccountView
+
+  import Pleroma.Factory
+
+  test "it represents a chat" do
+    user = insert(:user)
+    recipient = insert(:user)
+
+    {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
+
+    represented_chat = ChatView.render("show.json", chat: chat)
+
+    assert represented_chat == %{
+             id: "#{chat.id}",
+             recipient: recipient.ap_id,
+             recipient_account: AccountView.render("show.json", user: recipient),
+             unread: 0
+           }
+  end
+end

From c8458209110ef65101f965e460329308e5843559 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 17 Apr 2020 16:55:01 +0200
Subject: [PATCH 027/283] Notifications: Create a chat notification.

---
 lib/pleroma/web/activity_pub/side_effects.ex  |  2 ++
 .../mastodon_api/views/notification_view.ex   | 30 ++++++++++++++++++-
 test/web/activity_pub/side_effects_test.exs   | 19 ++++++++++++
 .../views/notification_view_test.exs          | 26 ++++++++++++++++
 4 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 594f32700..f32a99ec4 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -28,6 +28,8 @@ def handle(%{data: %{"type" => "Create", "object" => object_id}} = activity, met
 
     {:ok, _object} = handle_object_creation(object)
 
+    Notification.create_notifications(activity)
+
     {:ok, activity, meta}
   end
 
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 734ffbf39..5d231f0c4 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -8,11 +8,13 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Activity
   alias Pleroma.Notification
   alias Pleroma.User
+  alias Pleroma.Object
   alias Pleroma.UserRelationship
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
 
   def render("index.json", %{notifications: notifications, for: reading_user} = opts) do
     activities = Enum.map(notifications, & &1.activity)
@@ -81,7 +83,20 @@ def render(
       end
     end
 
-    mastodon_type = Activity.mastodon_notification_type(activity)
+    # This returns the notification type by activity, but both chats and statuses are in "Create" activities.
+    mastodon_type =
+      case Activity.mastodon_notification_type(activity) do
+        "mention" ->
+          object = Object.normalize(activity)
+
+          case object do
+            %{data: %{"type" => "ChatMessage"}} -> "pleroma:chat_mention"
+            _ -> "mention"
+          end
+
+        type ->
+          type
+      end
 
     render_opts = %{
       relationships: opts[:relationships],
@@ -125,6 +140,9 @@ def render(
           |> put_status(parent_activity_fn.(), reading_user, render_opts)
           |> put_emoji(activity)
 
+        "pleroma:chat_mention" ->
+          put_chat_message(response, activity, reading_user, render_opts)
+
         _ ->
           nil
       end
@@ -137,6 +155,16 @@ defp put_emoji(response, activity) do
     Map.put(response, :emoji, activity.data["content"])
   end
 
+  defp put_chat_message(response, activity, reading_user, opts) do
+    object = Object.normalize(activity)
+    author = User.get_cached_by_ap_id(object.data["actor"])
+    chat = Pleroma.Chat.get(reading_user.id, author.ap_id)
+    render_opts = Map.merge(opts, %{object: object, for: reading_user, chat: chat})
+    chat_message_render = ChatMessageView.render("show.json", render_opts)
+
+    Map.put(response, :chat_message, chat_message_render)
+  end
+
   defp put_status(response, activity, reading_user, opts) do
     status_render_opts = Map.merge(opts, %{activity: activity, for: reading_user})
     status_render = StatusView.render("show.json", status_render_opts)
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index b629d0d5d..d3ad4866c 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -6,7 +6,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
+  alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.SideEffects
@@ -34,6 +36,23 @@ test "add the like to the original object", %{like: like, user: user} do
   end
 
   describe "creation of ChatMessages" do
+    test "notifies the recipient" do
+      author = insert(:user, local: false)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+      {:ok, chat_message_object} = Object.create(chat_message_data)
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+
+      assert Repo.get_by(Notification, user_id: recipient.id, activity_id: create_activity.id)
+    end
+
     test "it creates a Chat for the local users and bumps the unread count" do
       author = insert(:user, local: false)
       recipient = insert(:user, local: true)
diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs
index c3ec9dfec..a48c298f2 100644
--- a/test/web/mastodon_api/views/notification_view_test.exs
+++ b/test/web/mastodon_api/views/notification_view_test.exs
@@ -6,7 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
+  alias Pleroma.Chat
   alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
@@ -14,6 +16,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
   import Pleroma.Factory
 
   defp test_notifications_rendering(notifications, user, expected_result) do
@@ -31,6 +34,29 @@ defp test_notifications_rendering(notifications, user, expected_result) do
     assert expected_result == result
   end
 
+  test "ChatMessage notification" do
+    user = insert(:user)
+    recipient = insert(:user)
+    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "what's up my dude")
+
+    {:ok, [notification]} = Notification.create_notifications(activity)
+
+    object = Object.normalize(activity)
+    chat = Chat.get(recipient.id, user.ap_id)
+
+    expected = %{
+      id: to_string(notification.id),
+      pleroma: %{is_seen: false},
+      type: "pleroma:chat_mention",
+      account: AccountView.render("show.json", %{user: user, for: recipient}),
+      chat_message:
+        ChatMessageView.render("show.json", %{object: object, for: recipient, chat: chat}),
+      created_at: Utils.to_masto_date(notification.inserted_at)
+    }
+
+    test_notifications_rendering([notification], recipient, [expected])
+  end
+
   test "Mention notification" do
     user = insert(:user)
     mentioned_user = insert(:user)

From ce23673ca1539350802326c62d6e72bd040950f6 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 11:45:11 +0200
Subject: [PATCH 028/283] ChatMessageValidator: Don't validate messages that
 are too long.

---
 .../object_validators/chat_message_validator.ex           | 1 +
 test/web/activity_pub/object_validator_test.exs           | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index a4e4460cd..caf2138a7 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -56,6 +56,7 @@ def validate_data(data_cng) do
     |> validate_inclusion(:type, ["ChatMessage"])
     |> validate_required([:id, :actor, :to, :type, :content])
     |> validate_length(:to, is: 1)
+    |> validate_length(:content, max: Pleroma.Config.get([:instance, :remote_limit]))
     |> validate_local_concern()
   end
 
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index bf0bfdfaf..e416e0808 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
 
   describe "chat messages" do
     setup do
+      clear_config([:instance, :remote_limit])
       user = insert(:user)
       recipient = insert(:user, local: false)
 
@@ -23,6 +24,13 @@ test "validates for a basic object we build", %{valid_chat_message: valid_chat_m
       assert {:ok, _object, _meta} = ObjectValidator.validate(valid_chat_message, [])
     end
 
+    test "does not validate if the message is longer than the remote_limit", %{
+      valid_chat_message: valid_chat_message
+    } do
+      Pleroma.Config.put([:instance, :remote_limit], 2)
+      refute match?({:ok, _object, _meta}, ObjectValidator.validate(valid_chat_message, []))
+    end
+
     test "does not validate if the actor or the recipient is not in our system", %{
       valid_chat_message: valid_chat_message
     } do

From 5b6818b3e5dc39e328f6f8d4b8f4587e5e1cef94 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 12:08:47 +0200
Subject: [PATCH 029/283] CommonAPI: Obey local limit for chat messages.

---
 lib/pleroma/web/common_api/common_api.ex       |  8 +++++++-
 test/web/common_api/common_api_test.exs        | 18 ++++++++++++++++++
 .../views/chat_message_view_test.exs           |  4 ++--
 test/web/pleroma_api/views/chat_view_test.exs  |  2 +-
 4 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 2b8add2fa..fcb0af4e8 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -28,7 +28,10 @@ defmodule Pleroma.Web.CommonAPI do
   def post_chat_message(%User{} = user, %User{} = recipient, content) do
     transaction =
       Repo.transaction(fn ->
-        with {_, {:ok, chat_message_data, _meta}} <-
+        with {_, true} <-
+               {:content_length,
+                String.length(content) <= Pleroma.Config.get([:instance, :chat_limit])},
+             {_, {:ok, chat_message_data, _meta}} <-
                {:build_object,
                 Builder.chat_message(
                   user,
@@ -43,6 +46,9 @@ def post_chat_message(%User{} = user, %User{} = recipient, content) do
              {_, {:ok, %Activity{} = activity, _meta}} <-
                {:common_pipeline, Pipeline.common_pipeline(create_activity_data, local: true)} do
           {:ok, activity}
+        else
+          {:content_length, false} -> {:error, :content_too_long}
+          e -> e
         end
       end)
 
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 1984aac8d..c17e30210 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -23,6 +23,8 @@ defmodule Pleroma.Web.CommonAPITest do
   setup do: clear_config([:instance, :max_pinned_statuses])
 
   describe "posting chat messages" do
+    setup do: clear_config([:instance, :chat_limit])
+
     test "it posts a chat message" do
       author = insert(:user)
       recipient = insert(:user)
@@ -47,6 +49,22 @@ test "it posts a chat message" do
       assert Chat.get(author.id, recipient.ap_id)
       assert Chat.get(recipient.id, author.ap_id)
     end
+
+    test "it reject messages over the local limit" do
+      Pleroma.Config.put([:instance, :chat_limit], 2)
+
+      author = insert(:user)
+      recipient = insert(:user)
+
+      {:error, message} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          "123"
+        )
+
+      assert message == :content_too_long
+    end
   end
 
   test "when replying to a conversation / participation, it will set the correct context id even if no explicit reply_to is given" do
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index e690da022..ad8febee6 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -23,7 +23,7 @@ test "it displays a chat message" do
 
     chat_message = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
 
-    assert chat_message[:id] == object.id
+    assert chat_message[:id] == object.id |> to_string()
     assert chat_message[:content] == "kippis"
     assert chat_message[:actor] == user.ap_id
     assert chat_message[:chat_id]
@@ -34,7 +34,7 @@ test "it displays a chat message" do
 
     chat_message_two = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
 
-    assert chat_message_two[:id] == object.id
+    assert chat_message_two[:id] == object.id |> to_string()
     assert chat_message_two[:content] == "gkgkgk"
     assert chat_message_two[:actor] == recipient.ap_id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 1eb0c6241..3dca555e8 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
+defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat

From 970b74383b43aa9a54c3cf59012944355e3eafbc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 12:29:19 +0200
Subject: [PATCH 030/283] Credo fixes.

---
 lib/pleroma/chat.ex                                         | 2 +-
 lib/pleroma/web/activity_pub/object_validator.ex            | 2 +-
 .../object_validators/chat_message_validator.ex             | 2 +-
 lib/pleroma/web/common_api/common_api.ex                    | 2 +-
 lib/pleroma/web/mastodon_api/views/notification_view.ex     | 5 +++--
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex  | 6 +++---
 test/web/activity_pub/object_validator_test.exs             | 2 +-
 test/web/pleroma_api/views/chat_view_test.exs               | 2 +-
 8 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index c2044881f..b8545063a 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
   import Ecto.Changeset
 
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   @moduledoc """
   Chat keeps a reference to ChatMessage conversations between a user and an recipient. The recipient can be a user (for now) or a group (not implemented yet).
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 259bbeb64..03db681ec 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -11,9 +11,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   alias Pleroma.Object
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
 
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index caf2138a7..6e3477cd1 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   use Ecto.Schema
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index fcb0af4e8..9e25f4c2f 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation.Participation
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.ThreadMute
@@ -17,7 +18,6 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Formatter
 
   import Pleroma.Web.Gettext
   import Pleroma.Web.CommonAPI.Utils
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 5d231f0c4..0b05d178b 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -7,8 +7,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
 
   alias Pleroma.Activity
   alias Pleroma.Notification
-  alias Pleroma.User
   alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
@@ -83,7 +83,8 @@ def render(
       end
     end
 
-    # This returns the notification type by activity, but both chats and statuses are in "Create" activities.
+    # This returns the notification type by activity, but both chats and statuses
+    # are in "Create" activities.
     mastodon_type =
       case Activity.mastodon_notification_type(activity) do
         "mention" ->
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 31c723426..9d8b9b3cf 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -6,13 +6,13 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   alias Pleroma.Chat
   alias Pleroma.Object
+  alias Pleroma.Pagination
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
-  alias Pleroma.Pagination
+  alias Pleroma.Web.PleromaAPI.ChatView
 
   import Ecto.Query
 
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index e416e0808..3ac5ecaf4 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -1,11 +1,11 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.ActivityPub.Builder
 
   import Pleroma.Factory
 
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 3dca555e8..725da5ff8 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.PleromaAPI.ChatView
 
   import Pleroma.Factory
 

From b836d3d104f75841d71f9cf7c5c8cb5c07ba7294 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 13:14:59 +0200
Subject: [PATCH 031/283] ChatMessageValidator: Require `published` field

---
 lib/pleroma/web/activity_pub/builder.ex                     | 6 ++++--
 .../object_validators/chat_message_validator.ex             | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index f0a6c1e1b..b67166a30 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -17,7 +17,8 @@ def create(actor, object_id, recipients) do
        "actor" => actor.ap_id,
        "to" => recipients,
        "object" => object_id,
-       "type" => "Create"
+       "type" => "Create",
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601()
      }, []}
   end
 
@@ -28,7 +29,8 @@ def chat_message(actor, recipient, content) do
        "actor" => actor.ap_id,
        "type" => "ChatMessage",
        "to" => [recipient],
-       "content" => content
+       "content" => content,
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601()
      }, []}
   end
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 6e3477cd1..9b8262553 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -54,7 +54,7 @@ def changeset(struct, data) do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["ChatMessage"])
-    |> validate_required([:id, :actor, :to, :type, :content])
+    |> validate_required([:id, :actor, :to, :type, :content, :published])
     |> validate_length(:to, is: 1)
     |> validate_length(:content, max: Pleroma.Config.get([:instance, :remote_limit]))
     |> validate_local_concern()

From 7e53da250e3b41e01073148efea0fc4f49dea9d5 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 14:08:54 +0200
Subject: [PATCH 032/283] ChatMessage: Support emoji.

---
 lib/pleroma/web/activity_pub/builder.ex       |  4 ++-
 .../chat_message_validator.ex                 |  1 +
 test/fixtures/create-chat-message.json        | 30 +++++++++----------
 .../activity_pub/object_validator_test.exs    |  6 ++--
 test/web/common_api/common_api_test.exs       |  8 +++--
 5 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index b67166a30..7576ed278 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   This module encodes our addressing policies and general shape of our objects.
   """
 
+  alias Pleroma.Emoji
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Utils
@@ -30,7 +31,8 @@ def chat_message(actor, recipient, content) do
        "type" => "ChatMessage",
        "to" => [recipient],
        "content" => content,
-       "published" => DateTime.utc_now() |> DateTime.to_iso8601()
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
+       "emoji" => Emoji.Formatter.get_emoji_map(content)
      }, []}
   end
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 9b8262553..2feb65f29 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -20,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
     field(:content, :string)
     field(:actor, Types.ObjectID)
     field(:published, Types.DateTime)
+    field(:emoji, :map, default: %{})
   end
 
   def cast_and_apply(data) do
diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json
index 2e4608f43..6db5b9f5c 100644
--- a/test/fixtures/create-chat-message.json
+++ b/test/fixtures/create-chat-message.json
@@ -1,19 +1,19 @@
 {
-    "actor": "http://2hu.gensokyo/users/raymoo",
-    "id": "http://2hu.gensokyo/objects/1",
-    "object": {
-        "attributedTo": "http://2hu.gensokyo/users/raymoo",
-        "content": "You expected a cute girl? Too bad. <script>alert('XSS')</script>",
-        "id": "http://2hu.gensokyo/objects/2",
-        "published": "2020-02-12T14:08:20Z",
-        "to": [
-	    "http://2hu.gensokyo/users/marisa"
-        ],
-        "type": "ChatMessage"
-    },
-    "published": "2018-02-12T14:08:20Z",
+  "actor": "http://2hu.gensokyo/users/raymoo",
+  "id": "http://2hu.gensokyo/objects/1",
+  "object": {
+    "attributedTo": "http://2hu.gensokyo/users/raymoo",
+    "content": "You expected a cute girl? Too bad. <script>alert('XSS')</script>",
+    "id": "http://2hu.gensokyo/objects/2",
+    "published": "2020-02-12T14:08:20Z",
     "to": [
-        "http://2hu.gensokyo/users/marisa"
+      "http://2hu.gensokyo/users/marisa"
     ],
-    "type": "Create"
+    "type": "ChatMessage"
+  },
+  "published": "2018-02-12T14:08:20Z",
+  "to": [
+    "http://2hu.gensokyo/users/marisa"
+  ],
+  "type": "Create"
 }
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 3ac5ecaf4..8230ae0d9 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -15,13 +15,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
       user = insert(:user)
       recipient = insert(:user, local: false)
 
-      {:ok, valid_chat_message, _} = Builder.chat_message(user, recipient.ap_id, "hey")
+      {:ok, valid_chat_message, _} = Builder.chat_message(user, recipient.ap_id, "hey :firefox:")
 
       %{user: user, recipient: recipient, valid_chat_message: valid_chat_message}
     end
 
     test "validates for a basic object we build", %{valid_chat_message: valid_chat_message} do
-      assert {:ok, _object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+      assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+
+      assert object == valid_chat_message
     end
 
     test "does not validate if the message is longer than the remote_limit", %{
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index c17e30210..86b3648ac 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -33,7 +33,7 @@ test "it posts a chat message" do
         CommonAPI.post_chat_message(
           author,
           recipient,
-          "a test message <script>alert('uuu')</script>"
+          "a test message <script>alert('uuu')</script> :firefox:"
         )
 
       assert activity.data["type"] == "Create"
@@ -44,7 +44,11 @@ test "it posts a chat message" do
       assert object.data["to"] == [recipient.ap_id]
 
       assert object.data["content"] ==
-               "a test message &lt;script&gt;alert(&#39;uuu&#39;)&lt;/script&gt;"
+               "a test message &lt;script&gt;alert(&#39;uuu&#39;)&lt;/script&gt; :firefox:"
+
+      assert object.data["emoji"] == %{
+               "firefox" => "http://localhost:4001/emoji/Firefox.gif"
+             }
 
       assert Chat.get(author.id, recipient.ap_id)
       assert Chat.get(recipient.id, author.ap_id)

From b5df4a98e4044cf1360f03f7dc3a0b59932ec8f5 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 14:38:53 +0200
Subject: [PATCH 033/283] ChatMessageView: Support emoji.

---
 lib/pleroma/web/pleroma_api/views/chat_message_view.ex | 6 +++++-
 test/web/pleroma_api/views/chat_message_view_test.exs  | 6 ++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index fdbb9ff1b..b40ab92a0 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -6,6 +6,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
   use Pleroma.Web, :view
 
   alias Pleroma.Chat
+  alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MastodonAPI.StatusView
 
   def render(
         "show.json",
@@ -18,7 +20,9 @@ def render(
       id: id |> to_string(),
       content: chat_message["content"],
       chat_id: chat_id |> to_string(),
-      actor: chat_message["actor"]
+      actor: chat_message["actor"],
+      created_at: Utils.to_masto_date(chat_message["published"]),
+      emojis: StatusView.build_emojis(chat_message["emoji"])
     }
   end
 
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index ad8febee6..115335f10 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
   test "it displays a chat message" do
     user = insert(:user)
     recipient = insert(:user)
-    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis")
+    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:")
 
     chat = Chat.get(user.id, recipient.ap_id)
 
@@ -24,9 +24,11 @@ test "it displays a chat message" do
     chat_message = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
 
     assert chat_message[:id] == object.id |> to_string()
-    assert chat_message[:content] == "kippis"
+    assert chat_message[:content] == "kippis :firefox:"
     assert chat_message[:actor] == user.ap_id
     assert chat_message[:chat_id]
+    assert chat_message[:created_at]
+    assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
 
     {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk")
 

From 97ad0c45977261df3068ca4f0c3febce3173c058 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 Apr 2020 17:51:06 +0200
Subject: [PATCH 034/283] Chats: Add API specs.

---
 .../web/api_spec/operations/chat_operation.ex | 96 ++++++++++++-------
 .../schemas/chat_message_create_request.ex    | 20 ++++
 .../api_spec/schemas/chat_message_response.ex | 38 ++++++++
 .../schemas/chat_messages_response.ex         | 41 ++++++++
 .../web/api_spec/schemas/chat_response.ex     | 73 ++++++++++++++
 .../web/api_spec/schemas/chats_response.ex    | 69 +++++++++++++
 .../controllers/chat_controller_test.exs      | 42 ++++++++
 7 files changed, 346 insertions(+), 33 deletions(-)
 create mode 100644 lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
 create mode 100644 lib/pleroma/web/api_spec/schemas/chat_message_response.ex
 create mode 100644 lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
 create mode 100644 lib/pleroma/web/api_spec/schemas/chat_response.ex
 create mode 100644 lib/pleroma/web/api_spec/schemas/chats_response.ex

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 038ebb29d..5bd41ec4f 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -4,7 +4,12 @@
 
 defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
-  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
 
   @spec open_api_operation(atom) :: Operation.t()
   def open_api_operation(action) do
@@ -16,16 +21,25 @@ def create_operation do
     %Operation{
       tags: ["chat"],
       summary: "Create a chat",
+      parameters: [
+        Operation.parameter(
+          :ap_id,
+          :path,
+          :string,
+          "The ActivityPub id of the recipient of this chat.",
+          required: true,
+          example: "https://lain.com/users/lain"
+        )
+      ],
       responses: %{
         200 =>
-          Operation.response("Chat", "application/json", %Schema{
-            type: :object,
-            description: "A created chat is returned",
-            properties: %{
-              id: %Schema{type: :integer}
-            }
-          })
-      }
+          Operation.response("The created or existing chat", "application/json", ChatResponse)
+      },
+      security: [
+        %{
+          "oAuth" => ["write"]
+        }
+      ]
     }
   end
 
@@ -33,17 +47,19 @@ def index_operation do
     %Operation{
       tags: ["chat"],
       summary: "Get a list of chats that you participated in",
+      parameters: [
+        Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
+        Operation.parameter(:min_id, :query, :string, "Return only chats after this id"),
+        Operation.parameter(:max_id, :query, :string, "Return only chats before this id")
+      ],
       responses: %{
-        200 =>
-          Operation.response("Chats", "application/json", %Schema{
-            type: :array,
-            description: "A list of chats",
-            items: %Schema{
-              type: :object,
-              description: "A chat"
-            }
-          })
-      }
+        200 => Operation.response("The chats of the user", "application/json", ChatsResponse)
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
     }
   end
 
@@ -51,17 +67,21 @@ def messages_operation do
     %Operation{
       tags: ["chat"],
       summary: "Get the most recent messages of the chat",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
+        Operation.parameter(:min_id, :query, :string, "Return only messages after this id"),
+        Operation.parameter(:max_id, :query, :string, "Return only messages before this id")
+      ],
       responses: %{
         200 =>
-          Operation.response("Messages", "application/json", %Schema{
-            type: :array,
-            description: "A list of chat messages",
-            items: %Schema{
-              type: :object,
-              description: "A chat message"
-            }
-          })
-      }
+          Operation.response("The messages in the chat", "application/json", ChatMessagesResponse)
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
     }
   end
 
@@ -69,13 +89,23 @@ def post_chat_message_operation do
     %Operation{
       tags: ["chat"],
       summary: "Post a message to the chat",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat")
+      ],
+      requestBody: Helpers.request_body("Parameters", ChatMessageCreateRequest, required: true),
       responses: %{
         200 =>
-          Operation.response("Message", "application/json", %Schema{
-            type: :object,
-            description: "A chat message"
-          })
-      }
+          Operation.response(
+            "The newly created ChatMessage",
+            "application/json",
+            ChatMessageResponse
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write"]
+        }
+      ]
     }
   end
 end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex b/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
new file mode 100644
index 000000000..4dafcda43
--- /dev/null
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest do
+  alias OpenApiSpex.Schema
+  require OpenApiSpex
+
+  OpenApiSpex.schema(%{
+    title: "ChatMessageCreateRequest",
+    description: "POST body for creating an chat message",
+    type: :object,
+    properties: %{
+      content: %Schema{type: :string, description: "The content of your message"}
+    },
+    example: %{
+      "content" => "Hey wanna buy feet pics?"
+    }
+  })
+end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
new file mode 100644
index 000000000..e94c00369
--- /dev/null
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
+  alias OpenApiSpex.Schema
+
+  require OpenApiSpex
+
+  OpenApiSpex.schema(%{
+    title: "ChatMessageResponse",
+    description: "Response schema for a ChatMessage",
+    type: :object,
+    properties: %{
+      id: %Schema{type: :string},
+      actor: %Schema{type: :string, description: "The ActivityPub id of the actor"},
+      chat_id: %Schema{type: :string},
+      content: %Schema{type: :string},
+      created_at: %Schema{type: :string, format: :datetime},
+      emojis: %Schema{type: :array}
+    },
+    example: %{
+      "actor" => "https://dontbulling.me/users/lain",
+      "chat_id" => "1",
+      "content" => "hey you again",
+      "created_at" => "2020-04-21T15:06:45.000Z",
+      "emojis" => [
+        %{
+          "static_url" => "https://dontbulling.me/emoji/Firefox.gif",
+          "visible_in_picker" => false,
+          "shortcode" => "firefox",
+          "url" => "https://dontbulling.me/emoji/Firefox.gif"
+        }
+      ],
+      "id" => "14"
+    }
+  })
+end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex b/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
new file mode 100644
index 000000000..302bdec95
--- /dev/null
+++ b/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse do
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
+
+  require OpenApiSpex
+
+  OpenApiSpex.schema(%{
+    title: "ChatMessagesResponse",
+    description: "Response schema for multiple ChatMessages",
+    type: :array,
+    items: ChatMessageResponse,
+    example: [
+      %{
+        "emojis" => [
+          %{
+            "static_url" => "https://dontbulling.me/emoji/Firefox.gif",
+            "visible_in_picker" => false,
+            "shortcode" => "firefox",
+            "url" => "https://dontbulling.me/emoji/Firefox.gif"
+          }
+        ],
+        "created_at" => "2020-04-21T15:11:46.000Z",
+        "content" => "Check this out :firefox:",
+        "id" => "13",
+        "chat_id" => "1",
+        "actor" => "https://dontbulling.me/users/lain"
+      },
+      %{
+        "actor" => "https://dontbulling.me/users/lain",
+        "content" => "Whats' up?",
+        "id" => "12",
+        "chat_id" => "1",
+        "emojis" => [],
+        "created_at" => "2020-04-21T15:06:45.000Z"
+      }
+    ]
+  })
+end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_response.ex b/lib/pleroma/web/api_spec/schemas/chat_response.ex
new file mode 100644
index 000000000..a80f4d173
--- /dev/null
+++ b/lib/pleroma/web/api_spec/schemas/chat_response.ex
@@ -0,0 +1,73 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatResponse do
+  alias OpenApiSpex.Schema
+
+  require OpenApiSpex
+
+  OpenApiSpex.schema(%{
+    title: "ChatResponse",
+    description: "Response schema for a Chat",
+    type: :object,
+    properties: %{
+      id: %Schema{type: :string},
+      recipient: %Schema{type: :string},
+      # TODO: Make this reference the account structure.
+      recipient_account: %Schema{type: :object},
+      unread: %Schema{type: :integer}
+    },
+    example: %{
+      "recipient" => "https://dontbulling.me/users/lain",
+      "recipient_account" => %{
+        "pleroma" => %{
+          "is_admin" => false,
+          "confirmation_pending" => false,
+          "hide_followers_count" => false,
+          "is_moderator" => false,
+          "hide_favorites" => true,
+          "ap_id" => "https://dontbulling.me/users/lain",
+          "hide_follows_count" => false,
+          "hide_follows" => false,
+          "background_image" => nil,
+          "skip_thread_containment" => false,
+          "hide_followers" => false,
+          "relationship" => %{},
+          "tags" => []
+        },
+        "avatar" =>
+          "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+        "following_count" => 0,
+        "header_static" => "https://originalpatchou.li/images/banner.png",
+        "source" => %{
+          "sensitive" => false,
+          "note" => "lain",
+          "pleroma" => %{
+            "discoverable" => false,
+            "actor_type" => "Person"
+          },
+          "fields" => []
+        },
+        "statuses_count" => 1,
+        "locked" => false,
+        "created_at" => "2020-04-16T13:40:15.000Z",
+        "display_name" => "lain",
+        "fields" => [],
+        "acct" => "lain@dontbulling.me",
+        "id" => "9u6Qw6TAZANpqokMkK",
+        "emojis" => [],
+        "avatar_static" =>
+          "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+        "username" => "lain",
+        "followers_count" => 0,
+        "header" => "https://originalpatchou.li/images/banner.png",
+        "bot" => false,
+        "note" => "lain",
+        "url" => "https://dontbulling.me/users/lain"
+      },
+      "id" => "1",
+      "unread" => 2
+    }
+  })
+end
diff --git a/lib/pleroma/web/api_spec/schemas/chats_response.ex b/lib/pleroma/web/api_spec/schemas/chats_response.ex
new file mode 100644
index 000000000..3349e0691
--- /dev/null
+++ b/lib/pleroma/web/api_spec/schemas/chats_response.ex
@@ -0,0 +1,69 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatsResponse do
+  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
+
+  require OpenApiSpex
+
+  OpenApiSpex.schema(%{
+    title: "ChatsResponse",
+    description: "Response schema for multiple Chats",
+    type: :array,
+    items: ChatResponse,
+    example: [
+      %{
+        "recipient" => "https://dontbulling.me/users/lain",
+        "recipient_account" => %{
+          "pleroma" => %{
+            "is_admin" => false,
+            "confirmation_pending" => false,
+            "hide_followers_count" => false,
+            "is_moderator" => false,
+            "hide_favorites" => true,
+            "ap_id" => "https://dontbulling.me/users/lain",
+            "hide_follows_count" => false,
+            "hide_follows" => false,
+            "background_image" => nil,
+            "skip_thread_containment" => false,
+            "hide_followers" => false,
+            "relationship" => %{},
+            "tags" => []
+          },
+          "avatar" =>
+            "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+          "following_count" => 0,
+          "header_static" => "https://originalpatchou.li/images/banner.png",
+          "source" => %{
+            "sensitive" => false,
+            "note" => "lain",
+            "pleroma" => %{
+              "discoverable" => false,
+              "actor_type" => "Person"
+            },
+            "fields" => []
+          },
+          "statuses_count" => 1,
+          "locked" => false,
+          "created_at" => "2020-04-16T13:40:15.000Z",
+          "display_name" => "lain",
+          "fields" => [],
+          "acct" => "lain@dontbulling.me",
+          "id" => "9u6Qw6TAZANpqokMkK",
+          "emojis" => [],
+          "avatar_static" =>
+            "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+          "username" => "lain",
+          "followers_count" => 0,
+          "header" => "https://originalpatchou.li/images/banner.png",
+          "bot" => false,
+          "note" => "lain",
+          "url" => "https://dontbulling.me/users/lain"
+        },
+        "id" => "1",
+        "unread" => 2
+      }
+    ]
+  })
+end
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 0750c7273..52a34d23f 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,8 +5,14 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
+  alias Pleroma.Web.ApiSpec
+  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse
   alias Pleroma.Web.CommonAPI
 
+  import OpenApiSpex.TestAssertions
   import Pleroma.Factory
 
   describe "POST /api/v1/pleroma/chats/:id/messages" do
@@ -24,6 +30,7 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
 
       assert result["content"] == "Hallo!!"
       assert result["chat_id"] == chat.id |> to_string()
+      assert_schema(result, "ChatMessageResponse", ApiSpec.spec())
     end
   end
 
@@ -45,6 +52,7 @@ test "it paginates", %{conn: conn, user: user} do
         |> json_response(200)
 
       assert length(result) == 20
+      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
 
       result =
         conn
@@ -52,6 +60,7 @@ test "it paginates", %{conn: conn, user: user} do
         |> json_response(200)
 
       assert length(result) == 10
+      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
     end
 
     test "it returns the messages for a given chat", %{conn: conn, user: user} do
@@ -76,6 +85,7 @@ test "it returns the messages for a given chat", %{conn: conn, user: user} do
       end)
 
       assert length(result) == 3
+      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
 
       # Trying to get the chat of a different user
       result =
@@ -99,6 +109,7 @@ test "it creates or returns a chat", %{conn: conn} do
         |> json_response(200)
 
       assert result["id"]
+      assert_schema(result, "ChatResponse", ApiSpec.spec())
     end
   end
 
@@ -117,6 +128,7 @@ test "it paginates", %{conn: conn, user: user} do
         |> json_response(200)
 
       assert length(result) == 20
+      assert_schema(result, "ChatsResponse", ApiSpec.spec())
 
       result =
         conn
@@ -124,6 +136,8 @@ test "it paginates", %{conn: conn, user: user} do
         |> json_response(200)
 
       assert length(result) == 10
+
+      assert_schema(result, "ChatsResponse", ApiSpec.spec())
     end
 
     test "it return a list of chats the current user is participating in, in descending order of updates",
@@ -154,6 +168,34 @@ test "it return a list of chats the current user is participating in, in descend
                chat_3.id |> to_string(),
                chat_1.id |> to_string()
              ]
+
+      assert_schema(result, "ChatsResponse", ApiSpec.spec())
+    end
+  end
+
+  describe "schemas" do
+    test "Chat example matches schema" do
+      api_spec = ApiSpec.spec()
+      schema = ChatResponse.schema()
+      assert_schema(schema.example, "ChatResponse", api_spec)
+    end
+
+    test "Chats example matches schema" do
+      api_spec = ApiSpec.spec()
+      schema = ChatsResponse.schema()
+      assert_schema(schema.example, "ChatsResponse", api_spec)
+    end
+
+    test "ChatMessage example matches schema" do
+      api_spec = ApiSpec.spec()
+      schema = ChatMessageResponse.schema()
+      assert_schema(schema.example, "ChatMessageResponse", api_spec)
+    end
+
+    test "ChatsMessage example matches schema" do
+      api_spec = ApiSpec.spec()
+      schema = ChatMessagesResponse.schema()
+      assert_schema(schema.example, "ChatMessagesResponse", api_spec)
     end
   end
 end

From 66c2eb670b273d808f0a9c1ae087df064718ca3d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 Apr 2020 18:23:00 +0200
Subject: [PATCH 035/283] ChatController: Validate parameters.

---
 .../web/api_spec/operations/chat_operation.ex |  4 ++++
 .../controllers/chat_controller.ex            | 22 ++++++++++++-------
 .../controllers/chat_controller_test.exs      |  5 +++--
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 5bd41ec4f..dc99bd773 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -21,6 +21,7 @@ def create_operation do
     %Operation{
       tags: ["chat"],
       summary: "Create a chat",
+      operationId: "ChatController.create",
       parameters: [
         Operation.parameter(
           :ap_id,
@@ -47,6 +48,7 @@ def index_operation do
     %Operation{
       tags: ["chat"],
       summary: "Get a list of chats that you participated in",
+      operationId: "ChatController.index",
       parameters: [
         Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
         Operation.parameter(:min_id, :query, :string, "Return only chats after this id"),
@@ -67,6 +69,7 @@ def messages_operation do
     %Operation{
       tags: ["chat"],
       summary: "Get the most recent messages of the chat",
+      operationId: "ChatController.messages",
       parameters: [
         Operation.parameter(:id, :path, :string, "The ID of the Chat"),
         Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
@@ -89,6 +92,7 @@ def post_chat_message_operation do
     %Operation{
       tags: ["chat"],
       summary: "Post a message to the chat",
+      operationId: "ChatController.post_chat_message",
       parameters: [
         Operation.parameter(:id, :path, :string, "The ID of the Chat")
       ],
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 9d8b9b3cf..771ad6217 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Web.PleromaAPI.ChatMessageView
   alias Pleroma.Web.PleromaAPI.ChatView
 
+  import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
+
   import Ecto.Query
 
   # TODO
@@ -29,12 +31,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
     %{scopes: ["read:statuses"]} when action in [:messages, :index]
   )
 
+  plug(OpenApiSpex.Plug.CastAndValidate)
+
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
-  def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
-        "id" => id,
-        "content" => content
-      }) do
+  def post_chat_message(
+        %{body_params: %{content: content}, assigns: %{user: %{id: user_id} = user}} = conn,
+        %{
+          id: id
+        }
+      ) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
          {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, content),
@@ -45,7 +51,7 @@ def post_chat_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
     end
   end
 
-  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id} = params) do
+  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       messages =
         from(o in Object,
@@ -66,7 +72,7 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => id} =
               ^[user.ap_id]
             )
         )
-        |> Pagination.fetch_paginated(params)
+        |> Pagination.fetch_paginated(params |> stringify_keys())
 
       conn
       |> put_view(ChatMessageView)
@@ -85,7 +91,7 @@ def index(%{assigns: %{user: %{id: user_id}}} = conn, params) do
         where: c.user_id == ^user_id,
         order_by: [desc: c.updated_at]
       )
-      |> Pagination.fetch_paginated(params)
+      |> Pagination.fetch_paginated(params |> stringify_keys)
 
     conn
     |> put_view(ChatView)
@@ -93,7 +99,7 @@ def index(%{assigns: %{user: %{id: user_id}}} = conn, params) do
   end
 
   def create(%{assigns: %{user: user}} = conn, params) do
-    recipient = params["ap_id"] |> URI.decode_www_form()
+    recipient = params[:ap_id]
 
     with {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
       conn
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 52a34d23f..84610e511 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -25,6 +25,7 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
 
       result =
         conn
+        |> put_req_header("content-type", "application/json")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
         |> json_response(200)
 
@@ -56,7 +57,7 @@ test "it paginates", %{conn: conn, user: user} do
 
       result =
         conn
-        |> get("/api/v1/pleroma/chats/#{chat.id}/messages", %{"max_id" => List.last(result)["id"]})
+        |> get("/api/v1/pleroma/chats/#{chat.id}/messages?max_id=#{List.last(result)["id"]}")
         |> json_response(200)
 
       assert length(result) == 10
@@ -132,7 +133,7 @@ test "it paginates", %{conn: conn, user: user} do
 
       result =
         conn
-        |> get("/api/v1/pleroma/chats", %{max_id: List.last(result)["id"]})
+        |> get("/api/v1/pleroma/chats?max_id=#{List.last(result)["id"]}")
         |> json_response(200)
 
       assert length(result) == 10

From 6c8390fa4d47a86c34bcc71681ba30f04d14eae9 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 Apr 2020 18:32:30 +0200
Subject: [PATCH 036/283] ChatControllerTest: Credo fixes.

---
 test/web/pleroma_api/controllers/chat_controller_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 84610e511..07b698013 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -6,10 +6,10 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   alias Pleroma.Chat
   alias Pleroma.Web.ApiSpec
-  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
   alias Pleroma.Web.CommonAPI
 
   import OpenApiSpex.TestAssertions

From 2e62a63749e040b108b8afe2c8839c470f89fa04 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 Apr 2020 12:48:52 +0200
Subject: [PATCH 037/283] ChatMessageValidator: Validation changes

Don't validate if the recipient is blocking the actor.
---
 .../object_validators/chat_message_validator.ex      | 12 +++++++++++-
 test/web/activity_pub/object_validator_test.exs      |  9 +++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 2feb65f29..8b5bb4fdc 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -61,15 +61,25 @@ def validate_data(data_cng) do
     |> validate_local_concern()
   end
 
-  @doc "Validates if at least one of the users in this ChatMessage is a local user, otherwise we don't want the message in our system. It also validates the presence of both users in our system."
+  @doc """
+  Validates the following
+  - If both users are in our system
+  - If at least one of the users in this ChatMessage is a local user
+  - If the recipient is not blocking the actor
+  """
   def validate_local_concern(cng) do
     with actor_ap <- get_field(cng, :actor),
          {_, %User{} = actor} <- {:find_actor, User.get_cached_by_ap_id(actor_ap)},
          {_, %User{} = recipient} <-
            {:find_recipient, User.get_cached_by_ap_id(get_field(cng, :to) |> hd())},
+         {_, false} <- {:blocking_actor?, User.blocks?(recipient, actor)},
          {_, true} <- {:local?, Enum.any?([actor, recipient], & &1.local)} do
       cng
     else
+      {:blocking_actor?, true} ->
+        cng
+        |> add_error(:actor, "actor is blocked by recipient")
+
       {:local?, false} ->
         cng
         |> add_error(:actor, "actor and recipient are both remote")
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 8230ae0d9..bc2317e55 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -33,6 +33,15 @@ test "does not validate if the message is longer than the remote_limit", %{
       refute match?({:ok, _object, _meta}, ObjectValidator.validate(valid_chat_message, []))
     end
 
+    test "does not validate if the recipient is blocking the actor", %{
+      valid_chat_message: valid_chat_message,
+      user: user,
+      recipient: recipient
+    } do
+      Pleroma.User.block(recipient, user)
+      refute match?({:ok, _object, _meta}, ObjectValidator.validate(valid_chat_message, []))
+    end
+
     test "does not validate if the actor or the recipient is not in our system", %{
       valid_chat_message: valid_chat_message
     } do

From 1d6338f2d38284e94e17be58c21c7f34b5621ab7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 22 Apr 2020 12:52:39 +0200
Subject: [PATCH 038/283] Litepub: Add ChatMessage.

---
 priv/static/schemas/litepub-0.1.jsonld | 1 +
 1 file changed, 1 insertion(+)

diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld
index 278ad2f96..7cc3fee40 100644
--- a/priv/static/schemas/litepub-0.1.jsonld
+++ b/priv/static/schemas/litepub-0.1.jsonld
@@ -30,6 +30,7 @@
                 "@type": "@id"
             },
             "EmojiReact": "litepub:EmojiReact",
+            "ChatMessage": "litepub:ChatMessage",
             "alsoKnownAs": {
                 "@id": "as:alsoKnownAs",
                 "@type": "@id"

From 1e28d34592a5fae0f3403763f1ff86cc393a52b0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 23 Apr 2020 16:19:49 +0200
Subject: [PATCH 039/283] ChatMessage: Correctly ingest emoji tags.

---
 .../object_validators/chat_message_validator.ex      |  2 ++
 test/fixtures/create-chat-message.json               | 12 ++++++++++++
 .../transmogrifier/chat_message_test.exs             |  1 +
 3 files changed, 15 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 8b5bb4fdc..f07045d9d 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1]
 
   @primary_key false
   @derive Jason.Encoder
@@ -42,6 +43,7 @@ def cast_data(data) do
 
   def fix(data) do
     data
+    |> fix_emoji()
     |> Map.put_new("actor", data["attributedTo"])
   end
 
diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json
index 6db5b9f5c..9c23a1c9b 100644
--- a/test/fixtures/create-chat-message.json
+++ b/test/fixtures/create-chat-message.json
@@ -9,6 +9,18 @@
     "to": [
       "http://2hu.gensokyo/users/marisa"
     ],
+    "tag": [
+      {
+        "icon": {
+          "type": "Image",
+          "url": "http://2hu.gensokyo/emoji/Firefox.gif"
+        },
+        "id": "http://2hu.gensokyo/emoji/Firefox.gif",
+        "name": ":firefox:",
+        "type": "Emoji",
+        "updated": "1970-01-01T00:00:00Z"
+      }
+    ],
     "type": "ChatMessage"
   },
   "published": "2018-02-12T14:08:20Z",
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index 4d6f24609..a63a31e6e 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -75,6 +75,7 @@ test "it inserts it and creates a chat" do
 
       assert object
       assert object.data["content"] == "You expected a cute girl? Too bad. alert(&#39;XSS&#39;)"
+      assert match?(%{"firefox" => _}, object.data["emoji"])
 
       refute Chat.get(author.id, recipient.ap_id)
       assert Chat.get(recipient.id, author.ap_id)

From a51cdafc0192b66ce75659b424a690f52c9b2a49 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 23 Apr 2020 16:55:00 +0200
Subject: [PATCH 040/283] Docs: Add documentation about chatmessages

---
 docs/ap_extensions.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 docs/ap_extensions.md

diff --git a/docs/ap_extensions.md b/docs/ap_extensions.md
new file mode 100644
index 000000000..c4550a1ac
--- /dev/null
+++ b/docs/ap_extensions.md
@@ -0,0 +1,35 @@
+# ChatMessages
+
+ChatMessages are the messages sent in 1-on-1 chats. They are similar to
+`Note`s, but the addresing is done by having a single AP actor in the `to`
+field. Addressing multiple actors is not allowed. These messages are always
+private, there is no public version of them. They are created with a `Create`
+activity.
+
+Example:
+
+```json
+{
+  "actor": "http://2hu.gensokyo/users/raymoo",
+  "id": "http://2hu.gensokyo/objects/1",
+  "object": {
+    "attributedTo": "http://2hu.gensokyo/users/raymoo",
+    "content": "You expected a cute girl? Too bad.",
+    "id": "http://2hu.gensokyo/objects/2",
+    "published": "2020-02-12T14:08:20Z",
+    "to": [
+      "http://2hu.gensokyo/users/marisa"
+    ],
+    "type": "ChatMessage"
+  },
+  "published": "2018-02-12T14:08:20Z",
+  "to": [
+    "http://2hu.gensokyo/users/marisa"
+  ],
+  "type": "Create"
+}
+```
+
+This setup does not prevent multi-user chats, but these will have to go through
+a `Group`, which will be the recipient of the messages and then `Announce` them
+to the users in the `Group`.

From b429a49504b1df6fa085cccbb3e461cd378b15c4 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 23 Apr 2020 23:44:03 +0200
Subject: [PATCH 041/283] mix.exs: Fix for MacOS

---
 mix.exs | 43 +++++++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/mix.exs b/mix.exs
index b76aef180..021217400 100644
--- a/mix.exs
+++ b/mix.exs
@@ -220,32 +220,39 @@ defp aliases do
   defp version(version) do
     identifier_filter = ~r/[^0-9a-z\-]+/i
 
-    # Pre-release version, denoted from patch version with a hyphen
-    {tag, tag_err} =
-      System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
-
-    {describe, describe_err} = System.cmd("git", ["describe", "--tags", "--abbrev=8"])
-    {commit_hash, commit_hash_err} = System.cmd("git", ["rev-parse", "--short", "HEAD"])
+    {_gitpath, git_present} = System.cmd("sh", ["-c", "command -v git"])
 
     git_pre_release =
-      cond do
-        tag_err == 0 and describe_err == 0 ->
-          describe
-          |> String.trim()
-          |> String.replace(String.trim(tag), "")
-          |> String.trim_leading("-")
-          |> String.trim()
+      if git_present do
+        {tag, tag_err} =
+          System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
 
-        commit_hash_err == 0 ->
-          "0-g" <> String.trim(commit_hash)
+        {describe, describe_err} = System.cmd("git", ["describe", "--tags", "--abbrev=8"])
+        {commit_hash, commit_hash_err} = System.cmd("git", ["rev-parse", "--short", "HEAD"])
 
-        true ->
-          ""
+        # Pre-release version, denoted from patch version with a hyphen
+        cond do
+          tag_err == 0 and describe_err == 0 ->
+            describe
+            |> String.trim()
+            |> String.replace(String.trim(tag), "")
+            |> String.trim_leading("-")
+            |> String.trim()
+
+          commit_hash_err == 0 ->
+            "0-g" <> String.trim(commit_hash)
+
+          true ->
+            ""
+        end
+      else
+        ""
       end
 
     # Branch name as pre-release version component, denoted with a dot
     branch_name =
-      with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
+      with true <- git_present,
+           {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
            branch_name <- String.trim(branch_name),
            branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name,
            true <-

From c63d6ba0b2686db847f70cf251f92bfed57c4e5f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 23 Apr 2020 23:44:30 +0200
Subject: [PATCH 042/283] mix.exs: branch_name fallbacks to ""

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 021217400..15a65c0fb 100644
--- a/mix.exs
+++ b/mix.exs
@@ -266,7 +266,7 @@ defp version(version) do
 
         branch_name
       else
-        _ -> "stable"
+        _ -> ""
       end
 
     build_name =

From 053c46153076f351c5273c2d6b1fb0843e7b6a6d Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 24 Apr 2020 00:26:24 +0200
Subject: [PATCH 043/283] mix.exs: proper check on 0, remove else in
 git_pre_release

---
 mix.exs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/mix.exs b/mix.exs
index 15a65c0fb..ebb8bdb08 100644
--- a/mix.exs
+++ b/mix.exs
@@ -220,10 +220,10 @@ defp aliases do
   defp version(version) do
     identifier_filter = ~r/[^0-9a-z\-]+/i
 
-    {_gitpath, git_present} = System.cmd("sh", ["-c", "command -v git"])
+    {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"])
 
     git_pre_release =
-      if git_present do
+      if cmdgit_err == 0 do
         {tag, tag_err} =
           System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
 
@@ -243,15 +243,13 @@ defp version(version) do
             "0-g" <> String.trim(commit_hash)
 
           true ->
-            ""
+            nil
         end
-      else
-        ""
       end
 
     # Branch name as pre-release version component, denoted with a dot
     branch_name =
-      with true <- git_present,
+      with 0 <- cmdgit_err,
            {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
            branch_name <- String.trim(branch_name),
            branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name,

From d2bbea1a8076401645600ceb953dd66ec023b3ad Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 12:19:27 +0200
Subject: [PATCH 044/283] ChatControllerTest: Use new schema testing functions.

---
 .../controllers/chat_controller_test.exs      | 58 +++----------------
 1 file changed, 8 insertions(+), 50 deletions(-)

diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 07b698013..84d7b543e 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,14 +5,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
-  alias Pleroma.Web.ApiSpec
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
   alias Pleroma.Web.CommonAPI
 
-  import OpenApiSpex.TestAssertions
   import Pleroma.Factory
 
   describe "POST /api/v1/pleroma/chats/:id/messages" do
@@ -27,11 +21,10 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert result["content"] == "Hallo!!"
       assert result["chat_id"] == chat.id |> to_string()
-      assert_schema(result, "ChatMessageResponse", ApiSpec.spec())
     end
   end
 
@@ -50,18 +43,16 @@ test "it paginates", %{conn: conn, user: user} do
       result =
         conn
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert length(result) == 20
-      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
 
       result =
         conn
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages?max_id=#{List.last(result)["id"]}")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert length(result) == 10
-      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
     end
 
     test "it returns the messages for a given chat", %{conn: conn, user: user} do
@@ -78,7 +69,7 @@ test "it returns the messages for a given chat", %{conn: conn, user: user} do
       result =
         conn
         |> get("/api/v1/pleroma/chats/#{chat.id}/messages")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       result
       |> Enum.each(fn message ->
@@ -86,7 +77,6 @@ test "it returns the messages for a given chat", %{conn: conn, user: user} do
       end)
 
       assert length(result) == 3
-      assert_schema(result, "ChatMessagesResponse", ApiSpec.spec())
 
       # Trying to get the chat of a different user
       result =
@@ -107,10 +97,9 @@ test "it creates or returns a chat", %{conn: conn} do
       result =
         conn
         |> post("/api/v1/pleroma/chats/by-ap-id/#{URI.encode_www_form(other_user.ap_id)}")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert result["id"]
-      assert_schema(result, "ChatResponse", ApiSpec.spec())
     end
   end
 
@@ -126,19 +115,16 @@ test "it paginates", %{conn: conn, user: user} do
       result =
         conn
         |> get("/api/v1/pleroma/chats")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert length(result) == 20
-      assert_schema(result, "ChatsResponse", ApiSpec.spec())
 
       result =
         conn
         |> get("/api/v1/pleroma/chats?max_id=#{List.last(result)["id"]}")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       assert length(result) == 10
-
-      assert_schema(result, "ChatsResponse", ApiSpec.spec())
     end
 
     test "it return a list of chats the current user is participating in, in descending order of updates",
@@ -160,7 +146,7 @@ test "it return a list of chats the current user is participating in, in descend
       result =
         conn
         |> get("/api/v1/pleroma/chats")
-        |> json_response(200)
+        |> json_response_and_validate_schema(200)
 
       ids = Enum.map(result, & &1["id"])
 
@@ -169,34 +155,6 @@ test "it return a list of chats the current user is participating in, in descend
                chat_3.id |> to_string(),
                chat_1.id |> to_string()
              ]
-
-      assert_schema(result, "ChatsResponse", ApiSpec.spec())
-    end
-  end
-
-  describe "schemas" do
-    test "Chat example matches schema" do
-      api_spec = ApiSpec.spec()
-      schema = ChatResponse.schema()
-      assert_schema(schema.example, "ChatResponse", api_spec)
-    end
-
-    test "Chats example matches schema" do
-      api_spec = ApiSpec.spec()
-      schema = ChatsResponse.schema()
-      assert_schema(schema.example, "ChatsResponse", api_spec)
-    end
-
-    test "ChatMessage example matches schema" do
-      api_spec = ApiSpec.spec()
-      schema = ChatMessageResponse.schema()
-      assert_schema(schema.example, "ChatMessageResponse", api_spec)
-    end
-
-    test "ChatsMessage example matches schema" do
-      api_spec = ApiSpec.spec()
-      schema = ChatMessagesResponse.schema()
-      assert_schema(schema.example, "ChatMessagesResponse", api_spec)
     end
   end
 end

From 15ba3700af76c44e63bf8881021f3ee2a5a7dafd Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 12:45:59 +0200
Subject: [PATCH 045/283] Chat Schemas: Inline unimportant Schemas.

---
 .../web/api_spec/operations/chat_operation.ex | 113 +++++++++++++++++-
 .../schemas/chat_messages_response.ex         |  41 -------
 .../web/api_spec/schemas/chats_response.ex    |  69 -----------
 3 files changed, 108 insertions(+), 115 deletions(-)
 delete mode 100644 lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
 delete mode 100644 lib/pleroma/web/api_spec/schemas/chats_response.ex

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index dc99bd773..6f55cbd59 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -7,9 +7,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse
   alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatsResponse
+  alias OpenApiSpex.Schema
 
   @spec open_api_operation(atom) :: Operation.t()
   def open_api_operation(action) do
@@ -34,7 +33,11 @@ def create_operation do
       ],
       responses: %{
         200 =>
-          Operation.response("The created or existing chat", "application/json", ChatResponse)
+          Operation.response(
+            "The created or existing chat",
+            "application/json",
+            ChatResponse
+          )
       },
       security: [
         %{
@@ -55,7 +58,7 @@ def index_operation do
         Operation.parameter(:max_id, :query, :string, "Return only chats before this id")
       ],
       responses: %{
-        200 => Operation.response("The chats of the user", "application/json", ChatsResponse)
+        200 => Operation.response("The chats of the user", "application/json", chats_response())
       },
       security: [
         %{
@@ -78,7 +81,11 @@ def messages_operation do
       ],
       responses: %{
         200 =>
-          Operation.response("The messages in the chat", "application/json", ChatMessagesResponse)
+          Operation.response(
+            "The messages in the chat",
+            "application/json",
+            chat_messages_response()
+          )
       },
       security: [
         %{
@@ -112,4 +119,100 @@ def post_chat_message_operation do
       ]
     }
   end
+
+  def chats_response() do
+    %Schema{
+      title: "ChatsResponse",
+      description: "Response schema for multiple Chats",
+      type: :array,
+      items: ChatResponse,
+      example: [
+        %{
+          "recipient" => "https://dontbulling.me/users/lain",
+          "recipient_account" => %{
+            "pleroma" => %{
+              "is_admin" => false,
+              "confirmation_pending" => false,
+              "hide_followers_count" => false,
+              "is_moderator" => false,
+              "hide_favorites" => true,
+              "ap_id" => "https://dontbulling.me/users/lain",
+              "hide_follows_count" => false,
+              "hide_follows" => false,
+              "background_image" => nil,
+              "skip_thread_containment" => false,
+              "hide_followers" => false,
+              "relationship" => %{},
+              "tags" => []
+            },
+            "avatar" =>
+              "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+            "following_count" => 0,
+            "header_static" => "https://originalpatchou.li/images/banner.png",
+            "source" => %{
+              "sensitive" => false,
+              "note" => "lain",
+              "pleroma" => %{
+                "discoverable" => false,
+                "actor_type" => "Person"
+              },
+              "fields" => []
+            },
+            "statuses_count" => 1,
+            "locked" => false,
+            "created_at" => "2020-04-16T13:40:15.000Z",
+            "display_name" => "lain",
+            "fields" => [],
+            "acct" => "lain@dontbulling.me",
+            "id" => "9u6Qw6TAZANpqokMkK",
+            "emojis" => [],
+            "avatar_static" =>
+              "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
+            "username" => "lain",
+            "followers_count" => 0,
+            "header" => "https://originalpatchou.li/images/banner.png",
+            "bot" => false,
+            "note" => "lain",
+            "url" => "https://dontbulling.me/users/lain"
+          },
+          "id" => "1",
+          "unread" => 2
+        }
+      ]
+    }
+  end
+
+  def chat_messages_response() do
+    %Schema{
+      title: "ChatMessagesResponse",
+      description: "Response schema for multiple ChatMessages",
+      type: :array,
+      items: ChatMessageResponse,
+      example: [
+        %{
+          "emojis" => [
+            %{
+              "static_url" => "https://dontbulling.me/emoji/Firefox.gif",
+              "visible_in_picker" => false,
+              "shortcode" => "firefox",
+              "url" => "https://dontbulling.me/emoji/Firefox.gif"
+            }
+          ],
+          "created_at" => "2020-04-21T15:11:46.000Z",
+          "content" => "Check this out :firefox:",
+          "id" => "13",
+          "chat_id" => "1",
+          "actor" => "https://dontbulling.me/users/lain"
+        },
+        %{
+          "actor" => "https://dontbulling.me/users/lain",
+          "content" => "Whats' up?",
+          "id" => "12",
+          "chat_id" => "1",
+          "emojis" => [],
+          "created_at" => "2020-04-21T15:06:45.000Z"
+        }
+      ]
+    }
+  end
 end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex b/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
deleted file mode 100644
index 302bdec95..000000000
--- a/lib/pleroma/web/api_spec/schemas/chat_messages_response.ex
+++ /dev/null
@@ -1,41 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessagesResponse do
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
-
-  require OpenApiSpex
-
-  OpenApiSpex.schema(%{
-    title: "ChatMessagesResponse",
-    description: "Response schema for multiple ChatMessages",
-    type: :array,
-    items: ChatMessageResponse,
-    example: [
-      %{
-        "emojis" => [
-          %{
-            "static_url" => "https://dontbulling.me/emoji/Firefox.gif",
-            "visible_in_picker" => false,
-            "shortcode" => "firefox",
-            "url" => "https://dontbulling.me/emoji/Firefox.gif"
-          }
-        ],
-        "created_at" => "2020-04-21T15:11:46.000Z",
-        "content" => "Check this out :firefox:",
-        "id" => "13",
-        "chat_id" => "1",
-        "actor" => "https://dontbulling.me/users/lain"
-      },
-      %{
-        "actor" => "https://dontbulling.me/users/lain",
-        "content" => "Whats' up?",
-        "id" => "12",
-        "chat_id" => "1",
-        "emojis" => [],
-        "created_at" => "2020-04-21T15:06:45.000Z"
-      }
-    ]
-  })
-end
diff --git a/lib/pleroma/web/api_spec/schemas/chats_response.ex b/lib/pleroma/web/api_spec/schemas/chats_response.ex
deleted file mode 100644
index 3349e0691..000000000
--- a/lib/pleroma/web/api_spec/schemas/chats_response.ex
+++ /dev/null
@@ -1,69 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ApiSpec.Schemas.ChatsResponse do
-  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
-
-  require OpenApiSpex
-
-  OpenApiSpex.schema(%{
-    title: "ChatsResponse",
-    description: "Response schema for multiple Chats",
-    type: :array,
-    items: ChatResponse,
-    example: [
-      %{
-        "recipient" => "https://dontbulling.me/users/lain",
-        "recipient_account" => %{
-          "pleroma" => %{
-            "is_admin" => false,
-            "confirmation_pending" => false,
-            "hide_followers_count" => false,
-            "is_moderator" => false,
-            "hide_favorites" => true,
-            "ap_id" => "https://dontbulling.me/users/lain",
-            "hide_follows_count" => false,
-            "hide_follows" => false,
-            "background_image" => nil,
-            "skip_thread_containment" => false,
-            "hide_followers" => false,
-            "relationship" => %{},
-            "tags" => []
-          },
-          "avatar" =>
-            "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
-          "following_count" => 0,
-          "header_static" => "https://originalpatchou.li/images/banner.png",
-          "source" => %{
-            "sensitive" => false,
-            "note" => "lain",
-            "pleroma" => %{
-              "discoverable" => false,
-              "actor_type" => "Person"
-            },
-            "fields" => []
-          },
-          "statuses_count" => 1,
-          "locked" => false,
-          "created_at" => "2020-04-16T13:40:15.000Z",
-          "display_name" => "lain",
-          "fields" => [],
-          "acct" => "lain@dontbulling.me",
-          "id" => "9u6Qw6TAZANpqokMkK",
-          "emojis" => [],
-          "avatar_static" =>
-            "https://dontbulling.me/media/065a4dd3c6740dab13ff9c71ec7d240bb9f8be9205c9e7467fb2202117da1e32.jpg",
-          "username" => "lain",
-          "followers_count" => 0,
-          "header" => "https://originalpatchou.li/images/banner.png",
-          "bot" => false,
-          "note" => "lain",
-          "url" => "https://dontbulling.me/users/lain"
-        },
-        "id" => "1",
-        "unread" => 2
-      }
-    ]
-  })
-end

From e62f8542a1933ba71dfd236741ad3afc76b89f22 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 13:48:09 +0200
Subject: [PATCH 046/283] Docs: Add chat motivation and api description.

---
 docs/API/chats.md | 197 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)
 create mode 100644 docs/API/chats.md

diff --git a/docs/API/chats.md b/docs/API/chats.md
new file mode 100644
index 000000000..39f493b47
--- /dev/null
+++ b/docs/API/chats.md
@@ -0,0 +1,197 @@
+# Chats
+
+Chats are a way to represent an IM-style conversation between two actors. They are not the same as direct messages and they are not `Status`es, even though they have a lot in common.
+
+## Why Chats?
+
+There are no 'visibility levels' in ActivityPub, their definition is purely a Mastodon convention. Direct Messaging between users on the fediverse has mostly been modeled by using ActivityPub addressing following Mastodon conventions on normal `Note` objects. In this case, a 'direct message' would be a message that has no followers addressed and also does not address the special public actor, but just the recipients in the `to` field. It would still be a `Note` and is presented with other `Note`s as a `Status` in the API.
+
+This is an awkward setup for a few reasons:
+
+- As DMs generally still follow the usual `Status` conventions, it is easy to accidentally pull somebody into a DM thread by mentioning them. (e.g. "I hate @badguy so much")
+- It is possible to go from a publicly addressed `Status` to a DM reply, back to public, then to a 'followers only' reply, and so on. This can be become very confusing, as it is unclear which user can see which part of the conversation.
+- The standard `Status` format of implicit addressing also leads to rather ugly results if you try to display the messages as a chat, because all the recipients are always mentioned by name in the message.
+- As direct messages are posted with the same api call (and usually same frontend component) as public messages, accidentally making a public message private or vice versa can happen easily. Client bugs can also lead to this, accidentally making private messages public.
+
+As a measure to improve this situation, the `Conversation` concept and related Pleroma extensions were introduced. While it made it possible to work around a few of the issues, many of the problems remained and it didn't see much adoption because it was too complicated to use correctly. 
+
+## Chats explained
+For this reasons, Chats are a new and different entity, both in the API as well as in ActivityPub. A quick overview:
+
+- Chats are meant to represent an instant message conversation between two actors. For now these are only 1-on-1 conversations, but the other actor can be a group in the future.
+- Chat messages have the ActivityPub type `ChatMessage`. They are not `Note`s. Servers that don't understand them will just drop them.
+- The only addressing allowed in `ChatMessage`s is one single ActivityPub actor in the `to` field.
+- There's always only one Chat between two actors. If you start chatting with someone and later start a 'new' Chat, the old Chat will be continued.
+- `ChatMessage`s are posted with a different api, making it very hard to accidentally send a message to the wrong person.
+- `ChatMessage`s don't show up in the existing timelines.
+- Chats can never go from private to public. They are always private between the two actors.
+
+## Caveats
+
+- Chats are NOT E2E encrypted (yet). Security is still the same as email.
+
+## API
+
+In general, the way to send a `ChatMessage` is to first create a `Chat`, then post a message to that `Chat`. The actors in the API are generally given by their ActivityPub id to make it easier to support later `Group` scenarios. 
+
+This is the overview of using the API. The API is also documented via OpenAPI, so you can view it and play with it by pointing SwaggerUI or a similar OpenAPI tool to `https://yourinstance.tld/api/openapi`.
+
+### Creating or getting a chat.
+
+To create or get an existing Chat for a certain recipient (identified by AP ID)
+you can call:
+
+`POST /api/v1/pleroma/chats/by-ap-id/{ap_id}`
+
+The ap_id of the recipients needs to be www-form encoded, so 
+
+```
+https://originalpatchou.li/user/lambda
+```
+
+would become
+
+```
+https%3A%2F%2Foriginalpatchou.li%2Fuser%2Flambda
+```
+
+The full call would then be
+
+```
+POST /api/v1/pleroma/chats/by-ap-id/https%3A%2F%2Foriginalpatchou.li%2Fuser%2Flambda
+```
+
+There will only ever be ONE Chat for you and a given recipient, so this call
+will return the same Chat if you already have one with that user.
+
+Returned data:
+
+```json
+{
+  "recipient" : "https://dontbulling.me/users/lain",
+  "recipient_account": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "id" : "1",
+  "unread" : 2
+}
+```
+
+### Getting a list of Chats
+
+`GET /api/v1/pleroma/chats`
+
+This will return a list of chats that you have been involved in, sorted by their
+last update (so new chats will be at the top).
+
+Returned data:
+
+```json
+[
+   {
+      "recipient" : "https://dontbulling.me/users/lain",
+      "recipient_account": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "id" : "1",
+      "unread" : 2
+   }
+]
+```
+
+The recipient of messages that are sent to this chat is given by their AP ID.
+The usual pagination options are implemented.
+
+### Getting the messages for a Chat
+
+For a given Chat id, you can get the associated messages with
+
+`GET /api/v1/pleroma/chats/{id}/messages`
+
+This will return all messages, sorted by most recent to least recent. The usual
+pagination options are implemented
+
+Returned data:
+
+```json
+[
+  {
+    "actor": "https://dontbulling.me/users/lain",
+    "chat_id": "1",
+    "content": "Check this out :firefox:",
+    "created_at": "2020-04-21T15:11:46.000Z",
+    "emojis": [
+      {
+        "shortcode": "firefox",
+        "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+        "url": "https://dontbulling.me/emoji/Firefox.gif",
+        "visible_in_picker": false
+      }
+    ],
+    "id": "13"
+  },
+  {
+    "actor": "https://dontbulling.me/users/lain",
+    "chat_id": "1",
+    "content": "Whats' up?",
+    "created_at": "2020-04-21T15:06:45.000Z",
+    "emojis": [],
+    "id": "12"
+  }
+]
+```
+
+### Posting a chat message
+
+Posting a chat message for given Chat id works like this:
+
+`POST /api/v1/pleroma/chats/{id}/messages`
+
+Parameters:
+- content: The text content of the message
+
+Currently, no formatting beyond basic escaping and emoji is implemented, as well as no
+attachments. This will most probably change.
+
+Returned data:
+
+```json
+{
+  "actor": "https://dontbulling.me/users/lain",
+  "chat_id": "1",
+  "content": "Check this out :firefox:",
+  "created_at": "2020-04-21T15:11:46.000Z",
+  "emojis": [
+    {
+      "shortcode": "firefox",
+      "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+      "url": "https://dontbulling.me/emoji/Firefox.gif",
+      "visible_in_picker": false
+    }
+  ],
+  "id": "13"
+}
+```
+
+### Notifications
+
+There's a new `pleroma:chat_mention` notification, which has this form:
+
+```json
+{
+  "id": "someid",
+  "type": "pleroma:chat_mention",
+  "account": { ... } // User account of the sender,
+  "chat_message": {
+    "chat_id": "1",
+    "id": "10",
+    "content": "Hello",
+    "actor": "https://dontbulling.me/users/lain"
+  },
+  "created_at": "somedate"
+}
+```

From 00e956528b392689326d5f5527543a422a874bcc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 14:02:11 +0200
Subject: [PATCH 047/283] Credo fixes.

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 6f55cbd59..546bc4d9b 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -4,11 +4,11 @@
 
 defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
   alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
-  alias OpenApiSpex.Schema
 
   @spec open_api_operation(atom) :: Operation.t()
   def open_api_operation(action) do
@@ -120,7 +120,7 @@ def post_chat_message_operation do
     }
   end
 
-  def chats_response() do
+  def chats_response do
     %Schema{
       title: "ChatsResponse",
       description: "Response schema for multiple Chats",
@@ -182,7 +182,7 @@ def chats_response() do
     }
   end
 
-  def chat_messages_response() do
+  def chat_messages_response do
     %Schema{
       title: "ChatMessagesResponse",
       description: "Response schema for multiple ChatMessages",

From 49e673dfea0a0cc94bba9691ce171b60f8a2fd75 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 16:08:03 +0200
Subject: [PATCH 048/283] ChatView: Add actor_account_id

---
 lib/pleroma/web/api_spec/schemas/chat_message_response.ex | 2 ++
 lib/pleroma/web/pleroma_api/views/chat_message_view.ex    | 2 ++
 test/web/pleroma_api/views/chat_message_view_test.exs     | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
index e94c00369..9459d210b 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
     properties: %{
       id: %Schema{type: :string},
       actor: %Schema{type: :string, description: "The ActivityPub id of the actor"},
+      actor_account_id: %Schema{type: :string, description: "The Mastodon API id of the actor"},
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string},
       created_at: %Schema{type: :string, format: :datetime},
@@ -21,6 +22,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
     },
     example: %{
       "actor" => "https://dontbulling.me/users/lain",
+      "actor_account_id" => "someflakeid",
       "chat_id" => "1",
       "content" => "hey you again",
       "created_at" => "2020-04-21T15:06:45.000Z",
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index b40ab92a0..5b740cc44 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
   alias Pleroma.Chat
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.User
 
   def render(
         "show.json",
@@ -21,6 +22,7 @@ def render(
       content: chat_message["content"],
       chat_id: chat_id |> to_string(),
       actor: chat_message["actor"],
+      actor_account_id: User.get_cached_by_ap_id(chat_message["actor"]).id,
       created_at: Utils.to_masto_date(chat_message["published"]),
       emojis: StatusView.build_emojis(chat_message["emoji"])
     }
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index 115335f10..7e3aeefab 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -26,6 +26,7 @@ test "it displays a chat message" do
     assert chat_message[:id] == object.id |> to_string()
     assert chat_message[:content] == "kippis :firefox:"
     assert chat_message[:actor] == user.ap_id
+    assert chat_message[:actor_account_id] == user.id
     assert chat_message[:chat_id]
     assert chat_message[:created_at]
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
@@ -39,6 +40,7 @@ test "it displays a chat message" do
     assert chat_message_two[:id] == object.id |> to_string()
     assert chat_message_two[:content] == "gkgkgk"
     assert chat_message_two[:actor] == recipient.ap_id
+    assert chat_message_two[:actor_account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
   end
 end

From ad82a216ff0676507a118e610209bd4259456b3c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 17:48:34 +0200
Subject: [PATCH 049/283] Chat API: Align more to Pleroma/Mastodon API.

---
 .../web/api_spec/operations/chat_operation.ex       | 13 ++++++-------
 .../web/api_spec/schemas/chat_message_response.ex   |  6 ++----
 lib/pleroma/web/api_spec/schemas/chat_response.ex   | 11 ++++-------
 .../web/pleroma_api/controllers/chat_controller.ex  |  5 ++---
 .../web/pleroma_api/views/chat_message_view.ex      |  3 +--
 lib/pleroma/web/pleroma_api/views/chat_view.ex      |  3 +--
 lib/pleroma/web/router.ex                           |  2 +-
 .../controllers/chat_controller_test.exs            |  4 ++--
 .../pleroma_api/views/chat_message_view_test.exs    |  6 ++----
 test/web/pleroma_api/views/chat_view_test.exs       |  3 +--
 10 files changed, 22 insertions(+), 34 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 546bc4d9b..59539e890 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -23,12 +23,12 @@ def create_operation do
       operationId: "ChatController.create",
       parameters: [
         Operation.parameter(
-          :ap_id,
+          :id,
           :path,
           :string,
-          "The ActivityPub id of the recipient of this chat.",
+          "The account id of the recipient of this chat",
           required: true,
-          example: "https://lain.com/users/lain"
+          example: "someflakeid"
         )
       ],
       responses: %{
@@ -128,8 +128,7 @@ def chats_response do
       items: ChatResponse,
       example: [
         %{
-          "recipient" => "https://dontbulling.me/users/lain",
-          "recipient_account" => %{
+          "account" => %{
             "pleroma" => %{
               "is_admin" => false,
               "confirmation_pending" => false,
@@ -202,10 +201,10 @@ def chat_messages_response do
           "content" => "Check this out :firefox:",
           "id" => "13",
           "chat_id" => "1",
-          "actor" => "https://dontbulling.me/users/lain"
+          "actor_id" => "someflakeid"
         },
         %{
-          "actor" => "https://dontbulling.me/users/lain",
+          "actor_id" => "someflakeid",
           "content" => "Whats' up?",
           "id" => "12",
           "chat_id" => "1",
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
index 9459d210b..b7a662cbb 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
@@ -13,16 +13,14 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
     type: :object,
     properties: %{
       id: %Schema{type: :string},
-      actor: %Schema{type: :string, description: "The ActivityPub id of the actor"},
-      actor_account_id: %Schema{type: :string, description: "The Mastodon API id of the actor"},
+      account_id: %Schema{type: :string, description: "The Mastodon API id of the actor"},
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string},
       created_at: %Schema{type: :string, format: :datetime},
       emojis: %Schema{type: :array}
     },
     example: %{
-      "actor" => "https://dontbulling.me/users/lain",
-      "actor_account_id" => "someflakeid",
+      "account_id" => "someflakeid",
       "chat_id" => "1",
       "content" => "hey you again",
       "created_at" => "2020-04-21T15:06:45.000Z",
diff --git a/lib/pleroma/web/api_spec/schemas/chat_response.ex b/lib/pleroma/web/api_spec/schemas/chat_response.ex
index a80f4d173..aa435165d 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_response.ex
@@ -12,15 +12,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatResponse do
     description: "Response schema for a Chat",
     type: :object,
     properties: %{
-      id: %Schema{type: :string},
-      recipient: %Schema{type: :string},
-      # TODO: Make this reference the account structure.
-      recipient_account: %Schema{type: :object},
-      unread: %Schema{type: :integer}
+      id: %Schema{type: :string, nullable: false},
+      account: %Schema{type: :object, nullable: false},
+      unread: %Schema{type: :integer, nullable: false}
     },
     example: %{
-      "recipient" => "https://dontbulling.me/users/lain",
-      "recipient_account" => %{
+      "account" => %{
         "pleroma" => %{
           "is_admin" => false,
           "confirmation_pending" => false,
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 771ad6217..8654f4295 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -99,9 +99,8 @@ def index(%{assigns: %{user: %{id: user_id}}} = conn, params) do
   end
 
   def create(%{assigns: %{user: user}} = conn, params) do
-    recipient = params[:ap_id]
-
-    with {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
+    with %User{ap_id: recipient} <- User.get_by_id(params[:id]),
+         {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
       conn
       |> put_view(ChatView)
       |> render("show.json", chat: chat)
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index 5b740cc44..28f12d9b0 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -21,8 +21,7 @@ def render(
       id: id |> to_string(),
       content: chat_message["content"],
       chat_id: chat_id |> to_string(),
-      actor: chat_message["actor"],
-      actor_account_id: User.get_cached_by_ap_id(chat_message["actor"]).id,
+      account_id: User.get_cached_by_ap_id(chat_message["actor"]).id,
       created_at: Utils.to_masto_date(chat_message["published"]),
       emojis: StatusView.build_emojis(chat_message["emoji"])
     }
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 1e9ef4356..bc3af5ef5 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -14,8 +14,7 @@ def render("show.json", %{chat: %Chat{} = chat} = opts) do
 
     %{
       id: chat.id |> to_string(),
-      recipient: chat.recipient,
-      recipient_account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
+      account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
       unread: chat.unread
     }
   end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 0c56318ee..aad2e3b98 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -275,7 +275,7 @@ defmodule Pleroma.Web.Router do
     scope [] do
       pipe_through(:authenticated_api)
 
-      post("/chats/by-ap-id/:ap_id", ChatController, :create)
+      post("/chats/by-account-id/:id", ChatController, :create)
       get("/chats", ChatController, :index)
       get("/chats/:id/messages", ChatController, :messages)
       post("/chats/:id/messages", ChatController, :post_chat_message)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 84d7b543e..b1044574b 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -88,7 +88,7 @@ test "it returns the messages for a given chat", %{conn: conn, user: user} do
     end
   end
 
-  describe "POST /api/v1/pleroma/chats/by-ap-id/:id" do
+  describe "POST /api/v1/pleroma/chats/by-account-id/:id" do
     setup do: oauth_access(["write:statuses"])
 
     test "it creates or returns a chat", %{conn: conn} do
@@ -96,7 +96,7 @@ test "it creates or returns a chat", %{conn: conn} do
 
       result =
         conn
-        |> post("/api/v1/pleroma/chats/by-ap-id/#{URI.encode_www_form(other_user.ap_id)}")
+        |> post("/api/v1/pleroma/chats/by-account-id/#{other_user.id}")
         |> json_response_and_validate_schema(200)
 
       assert result["id"]
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index 7e3aeefab..5c4c8b0d5 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -25,8 +25,7 @@ test "it displays a chat message" do
 
     assert chat_message[:id] == object.id |> to_string()
     assert chat_message[:content] == "kippis :firefox:"
-    assert chat_message[:actor] == user.ap_id
-    assert chat_message[:actor_account_id] == user.id
+    assert chat_message[:account_id] == user.id
     assert chat_message[:chat_id]
     assert chat_message[:created_at]
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
@@ -39,8 +38,7 @@ test "it displays a chat message" do
 
     assert chat_message_two[:id] == object.id |> to_string()
     assert chat_message_two[:content] == "gkgkgk"
-    assert chat_message_two[:actor] == recipient.ap_id
-    assert chat_message_two[:actor_account_id] == recipient.id
+    assert chat_message_two[:account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
   end
 end
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 725da5ff8..1ac3483d1 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -21,8 +21,7 @@ test "it represents a chat" do
 
     assert represented_chat == %{
              id: "#{chat.id}",
-             recipient: recipient.ap_id,
-             recipient_account: AccountView.render("show.json", user: recipient),
+             account: AccountView.render("show.json", user: recipient),
              unread: 0
            }
   end

From b550ef56119b9f735cf3fe279a5457e36ab92951 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 17:52:16 +0200
Subject: [PATCH 050/283] Docs: Align chat api changes with docs.

---
 docs/API/chats.md | 36 +++++++++++-------------------------
 1 file changed, 11 insertions(+), 25 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 39f493b47..24c4b4d06 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -32,33 +32,21 @@ For this reasons, Chats are a new and different entity, both in the API as well
 
 ## API
 
-In general, the way to send a `ChatMessage` is to first create a `Chat`, then post a message to that `Chat`. The actors in the API are generally given by their ActivityPub id to make it easier to support later `Group` scenarios. 
+In general, the way to send a `ChatMessage` is to first create a `Chat`, then post a message to that `Chat`. `Group`s will later be supported by making them a sub-type of `Account`.
 
 This is the overview of using the API. The API is also documented via OpenAPI, so you can view it and play with it by pointing SwaggerUI or a similar OpenAPI tool to `https://yourinstance.tld/api/openapi`.
 
 ### Creating or getting a chat.
 
-To create or get an existing Chat for a certain recipient (identified by AP ID)
+To create or get an existing Chat for a certain recipient (identified by Account ID)
 you can call:
 
-`POST /api/v1/pleroma/chats/by-ap-id/{ap_id}`
+`POST /api/v1/pleroma/chats/by-account-id/{account_id}`
 
-The ap_id of the recipients needs to be www-form encoded, so 
+The account id is the normal FlakeId of the usre
 
 ```
-https://originalpatchou.li/user/lambda
-```
-
-would become
-
-```
-https%3A%2F%2Foriginalpatchou.li%2Fuser%2Flambda
-```
-
-The full call would then be
-
-```
-POST /api/v1/pleroma/chats/by-ap-id/https%3A%2F%2Foriginalpatchou.li%2Fuser%2Flambda
+POST /api/v1/pleroma/chats/by-account-id/someflakeid
 ```
 
 There will only ever be ONE Chat for you and a given recipient, so this call
@@ -68,8 +56,7 @@ Returned data:
 
 ```json
 {
-  "recipient" : "https://dontbulling.me/users/lain",
-  "recipient_account": {
+  "account": {
     "id": "someflakeid",
     "username": "somenick",
     ...
@@ -91,8 +78,7 @@ Returned data:
 ```json
 [
    {
-      "recipient" : "https://dontbulling.me/users/lain",
-      "recipient_account": {
+      "account": {
         "id": "someflakeid",
         "username": "somenick",
         ...
@@ -120,7 +106,7 @@ Returned data:
 ```json
 [
   {
-    "actor": "https://dontbulling.me/users/lain",
+    "account_id": "someflakeid",
     "chat_id": "1",
     "content": "Check this out :firefox:",
     "created_at": "2020-04-21T15:11:46.000Z",
@@ -135,7 +121,7 @@ Returned data:
     "id": "13"
   },
   {
-    "actor": "https://dontbulling.me/users/lain",
+    "account_id": "someflakeid",
     "chat_id": "1",
     "content": "Whats' up?",
     "created_at": "2020-04-21T15:06:45.000Z",
@@ -161,7 +147,7 @@ Returned data:
 
 ```json
 {
-  "actor": "https://dontbulling.me/users/lain",
+  "account_id": "someflakeid",
   "chat_id": "1",
   "content": "Check this out :firefox:",
   "created_at": "2020-04-21T15:11:46.000Z",
@@ -190,7 +176,7 @@ There's a new `pleroma:chat_mention` notification, which has this form:
     "chat_id": "1",
     "id": "10",
     "content": "Hello",
-    "actor": "https://dontbulling.me/users/lain"
+    "account_id": "someflakeid"
   },
   "created_at": "somedate"
 }

From 3d040b1a87da66ed53a763f781477bd4f5a146d3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 27 Apr 2020 17:55:29 +0200
Subject: [PATCH 051/283] Credo fixes.

---
 lib/pleroma/web/pleroma_api/views/chat_message_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index 28f12d9b0..a821479ab 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -6,9 +6,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
   use Pleroma.Web, :view
 
   alias Pleroma.Chat
+  alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
-  alias Pleroma.User
 
   def render(
         "show.json",

From 906cf53ab94742327d073f56255f695c91339295 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 28 Apr 2020 13:38:02 +0200
Subject: [PATCH 052/283] Recipient Type: Cast all elements as ObjectIDs.

---
 .../object_validators/types/recipients.ex         | 15 +++++++++++++--
 .../object_validators/types/recipients_test.exs   | 12 ++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
index 5a3040842..48fe61e1a 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
@@ -1,13 +1,24 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do
   use Ecto.Type
 
-  def type, do: {:array, :string}
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
+
+  def type, do: {:array, ObjectID}
 
   def cast(object) when is_binary(object) do
     cast([object])
   end
 
-  def cast([_ | _] = data), do: {:ok, data}
+  def cast(data) when is_list(data) do
+    data
+    |> Enum.reduce({:ok, []}, fn element, acc ->
+      case {acc, ObjectID.cast(element)} do
+        {:error, _} -> :error
+        {_, :error} -> :error
+        {{:ok, list}, {:ok, id}} -> {:ok, [id | list]}
+      end
+    end)
+  end
 
   def cast(_) do
     :error
diff --git a/test/web/activity_pub/object_validators/types/recipients_test.exs b/test/web/activity_pub/object_validators/types/recipients_test.exs
index 2f9218774..f278f039b 100644
--- a/test/web/activity_pub/object_validators/types/recipients_test.exs
+++ b/test/web/activity_pub/object_validators/types/recipients_test.exs
@@ -2,11 +2,23 @@ defmodule Pleroma.Web.ObjectValidators.Types.RecipientsTest do
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients
   use Pleroma.DataCase
 
+  test "it asserts that all elements of the list are object ids" do
+    list = ["https://lain.com/users/lain", "invalid"]
+
+    assert :error == Recipients.cast(list)
+  end
+
   test "it works with a list" do
     list = ["https://lain.com/users/lain"]
     assert {:ok, list} == Recipients.cast(list)
   end
 
+  test "it works with a list with whole objects" do
+    list = ["https://lain.com/users/lain", %{"id" => "https://gensokyo.2hu/users/raymoo"}]
+    resulting_list = ["https://gensokyo.2hu/users/raymoo", "https://lain.com/users/lain"]
+    assert {:ok, resulting_list} == Recipients.cast(list)
+  end
+
   test "it turns a single string into a list" do
     recipient = "https://lain.com/users/lain"
 

From f8e56d4271f8c495316d304dd0de7f0a63eb0645 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 28 Apr 2020 13:43:58 +0200
Subject: [PATCH 053/283] SideEffects: Use Object.normalize to get the object.

---
 lib/pleroma/web/activity_pub/side_effects.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index ebe3071b0..a2b4da8d6 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -30,8 +30,8 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
     result
   end
 
-  def handle(%{data: %{"type" => "Create", "object" => object_id}} = activity, meta) do
-    object = Object.get_by_ap_id(object_id)
+  def handle(%{data: %{"type" => "Create"}} = activity, meta) do
+    object = Object.normalize(activity, false)
 
     {:ok, _object} = handle_object_creation(object)
 

From 6aa116eca7d6ef6567dcef03b8c776bd2134bf3f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 28 Apr 2020 16:26:19 +0200
Subject: [PATCH 054/283] Create activity handling: Flip it and reverse it

Both objects and create activities will now go through the common
pipeline and will be validated. Objects are now created as a side
effect of the Create activity, rolling back a transaction if it's
not possible to insert the object.
---
 lib/pleroma/notification.ex                   |  2 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  |  7 ++++
 .../web/activity_pub/object_validator.ex      |  4 +-
 .../chat_message_validator.ex                 |  2 +-
 .../create_chat_message_validator.ex          | 27 +++++++++++-
 .../object_validators/types/safe_text.ex      | 25 +++++++++++
 lib/pleroma/web/activity_pub/pipeline.ex      | 12 ++++--
 lib/pleroma/web/activity_pub/side_effects.ex  | 41 +++++++++++--------
 .../transmogrifier/chat_message_handling.ex   | 28 ++++++++-----
 lib/pleroma/web/common_api/common_api.ex      | 10 +++--
 lib/pleroma/web/common_api/utils.ex           |  2 +-
 .../types/safe_text_test.exs                  | 23 +++++++++++
 test/web/activity_pub/side_effects_test.exs   | 21 ++++++----
 .../transmogrifier/chat_message_test.exs      |  3 +-
 14 files changed, 155 insertions(+), 52 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
 create mode 100644 test/web/activity_pub/object_validators/types/safe_text_test.exs

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 73e19bf97..d96c12440 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -275,7 +275,7 @@ def dismiss(%{id: user_id} = _user, id) do
   end
 
   def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, false)
 
     if object && object.data["type"] == "Answer" do
       {:ok, []}
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 69ac06f6b..ecb13d76a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -126,7 +126,14 @@ def increase_poll_votes_if_vote(%{
 
   def increase_poll_votes_if_vote(_create_data), do: :noop
 
+  @object_types ["ChatMessage"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+  def persist(%{"type" => type} = object, meta) when type in @object_types do
+    with {:ok, object} <- Object.create(object) do
+      {:ok, object, meta}
+    end
+  end
+
   def persist(object, meta) do
     with local <- Keyword.fetch!(meta, :local),
          {recipients, _, _} <- get_recipients(object),
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 03db681ec..a4da9242a 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -23,7 +23,7 @@ def validate(%{"type" => "Like"} = object, meta) do
            object
            |> LikeValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
@@ -41,7 +41,7 @@ def validate(%{"type" => "ChatMessage"} = object, meta) do
   def validate(%{"type" => "Create"} = object, meta) do
     with {:ok, object} <-
            object
-           |> CreateChatMessageValidator.cast_and_validate()
+           |> CreateChatMessageValidator.cast_and_validate(meta)
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index f07045d9d..e87c1ac2e 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
     field(:id, Types.ObjectID, primary_key: true)
     field(:to, Types.Recipients, default: [])
     field(:type, :string)
-    field(:content, :string)
+    field(:content, Types.SafeText)
     field(:actor, Types.ObjectID)
     field(:published, Types.DateTime)
     field(:emoji, :map, default: %{})
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index ce52d5623..21c7a5ba4 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -33,8 +33,31 @@ def cast_data(data) do
     cast(%__MODULE__{}, data, __schema__(:fields))
   end
 
-  # No validation yet
-  def cast_and_validate(data) do
+  def cast_and_validate(data, meta \\ []) do
     cast_data(data)
+    |> validate_data(meta)
+  end
+
+  def validate_data(cng, meta \\ []) do
+    cng
+    |> validate_required([:id, :actor, :to, :type, :object])
+    |> validate_inclusion(:type, ["Create"])
+    |> validate_recipients_match(meta)
+  end
+
+  def validate_recipients_match(cng, meta) do
+    object_recipients = meta[:object_data]["to"] || []
+
+    cng
+    |> validate_change(:to, fn :to, recipients ->
+      activity_set = MapSet.new(recipients)
+      object_set = MapSet.new(object_recipients)
+
+      if MapSet.equal?(activity_set, object_set) do
+        []
+      else
+        [{:to, "Recipients don't match with object recipients"}]
+      end
+    end)
   end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
new file mode 100644
index 000000000..822e8d2c1
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do
+  use Ecto.Type
+
+  alias Pleroma.HTML
+
+  def type, do: :string
+
+  def cast(str) when is_binary(str) do
+    {:ok, HTML.strip_tags(str)}
+  end
+
+  def cast(_), do: :error
+
+  def dump(data) do
+    {:ok, data}
+  end
+
+  def load(data) do
+    {:ok, data}
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 7ccee54c9..4213ba751 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -4,20 +4,22 @@
 
 defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.SideEffects
   alias Pleroma.Web.Federator
 
-  @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t(), keyword()} | {:error, any()}
+  @spec common_pipeline(map(), keyword()) ::
+          {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}
   def common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
          {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
-         {_, {:ok, %Activity{} = activity, meta}} <-
+         {_, {:ok, activity, meta}} <-
            {:persist_object, ActivityPub.persist(mrfd_object, meta)},
-         {_, {:ok, %Activity{} = activity, meta}} <-
+         {_, {:ok, activity, meta}} <-
            {:execute_side_effects, SideEffects.handle(activity, meta)},
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
@@ -27,7 +29,9 @@ def common_pipeline(object, meta) do
     end
   end
 
-  defp maybe_federate(activity, meta) do
+  defp maybe_federate(%Object{}, _), do: {:ok, :not_federated}
+
+  defp maybe_federate(%Activity{} = activity, meta) do
     with {:ok, local} <- Keyword.fetch(meta, :local) do
       if local do
         Federator.publish(activity)
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a2b4da8d6..794a46267 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -8,7 +8,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Chat
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
 
   def handle(object, meta \\ [])
@@ -30,14 +32,17 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
     result
   end
 
+  # Tasks this handles
+  # - Actually create object
+  # - Rollback if we couldn't create it
+  # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
-    object = Object.normalize(activity, false)
-
-    {:ok, _object} = handle_object_creation(object)
-
-    Notification.create_notifications(activity)
-
-    {:ok, activity, meta}
+    with {:ok, _object, _meta} <- handle_object_creation(meta[:object_data], meta) do
+      Notification.create_notifications(activity)
+      {:ok, activity, meta}
+    else
+      e -> Repo.rollback(e)
+    end
   end
 
   # Nothing to do
@@ -45,18 +50,20 @@ def handle(object, meta) do
     {:ok, object, meta}
   end
 
-  def handle_object_creation(%{data: %{"type" => "ChatMessage"}} = object) do
-    actor = User.get_cached_by_ap_id(object.data["actor"])
-    recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
+  def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
+      actor = User.get_cached_by_ap_id(object.data["actor"])
+      recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
 
-    [[actor, recipient], [recipient, actor]]
-    |> Enum.each(fn [user, other_user] ->
-      if user.local do
-        Chat.bump_or_create(user.id, other_user.ap_id)
-      end
-    end)
+      [[actor, recipient], [recipient, actor]]
+      |> Enum.each(fn [user, other_user] ->
+        if user.local do
+          Chat.bump_or_create(user.id, other_user.ap_id)
+        end
+      end)
 
-    {:ok, object}
+      {:ok, object, meta}
+    end
   end
 
   # Nothing to do
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index cfe3b767b..043d847d1 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -3,31 +3,39 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling do
-  alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.Pipeline
 
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object_data} = data,
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
         _options
       ) do
+    # Create has to be run inside a transaction because the object is created as a side effect.
+    # If this does not work, we need to roll back creating the activity.
+    case Repo.transaction(fn -> do_handle_incoming(data) end) do
+      {:ok, value} ->
+        value
+
+      {:error, e} ->
+        {:error, e}
+    end
+  end
+
+  def do_handle_incoming(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object_data} = data
+      ) do
     with {_, {:ok, cast_data_sym}} <-
            {:casting_data, data |> CreateChatMessageValidator.cast_and_apply()},
          cast_data = ObjectValidator.stringify_keys(cast_data_sym),
          {_, {:ok, object_cast_data_sym}} <-
            {:casting_object_data, object_data |> ChatMessageValidator.cast_and_apply()},
          object_cast_data = ObjectValidator.stringify_keys(object_cast_data_sym),
-         # For now, just strip HTML
-         stripped_content = Pleroma.HTML.strip_tags(object_cast_data["content"]),
-         object_cast_data = object_cast_data |> Map.put("content", stripped_content),
-         {_, true} <- {:to_fields_match, cast_data["to"] == object_cast_data["to"]},
-         {_, {:ok, validated_object, _meta}} <-
-           {:validate_object, ObjectValidator.validate(object_cast_data, %{})},
-         {_, {:ok, _created_object}} <- {:persist_object, Object.create(validated_object)},
          {_, {:ok, activity, _meta}} <-
-           {:common_pipeline, Pipeline.common_pipeline(cast_data, local: false)} do
+           {:common_pipeline,
+            Pipeline.common_pipeline(cast_data, local: false, object_data: object_cast_data)} do
       {:ok, activity}
     else
       e ->
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 5eb221668..c39d1cee6 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -38,13 +38,15 @@ def post_chat_message(%User{} = user, %User{} = recipient, content) do
                   recipient.ap_id,
                   content |> Formatter.html_escape("text/plain")
                 )},
-             {_, {:ok, chat_message_object}} <-
-               {:create_object, Object.create(chat_message_data)},
              {_, {:ok, create_activity_data, _meta}} <-
                {:build_create_activity,
-                Builder.create(user, chat_message_object.data["id"], [recipient.ap_id])},
+                Builder.create(user, chat_message_data["id"], [recipient.ap_id])},
              {_, {:ok, %Activity{} = activity, _meta}} <-
-               {:common_pipeline, Pipeline.common_pipeline(create_activity_data, local: true)} do
+               {:common_pipeline,
+                Pipeline.common_pipeline(create_activity_data,
+                  local: true,
+                  object_data: chat_message_data
+                )} do
           {:ok, activity}
         else
           {:content_length, false} -> {:error, :content_too_long}
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 945e63e22..4afdf80af 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -425,7 +425,7 @@ def maybe_notify_mentioned_recipients(
         %Activity{data: %{"to" => _to, "type" => type} = data} = activity
       )
       when type == "Create" do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, false)
 
     object_data =
       cond do
diff --git a/test/web/activity_pub/object_validators/types/safe_text_test.exs b/test/web/activity_pub/object_validators/types/safe_text_test.exs
new file mode 100644
index 000000000..59ed0a1fe
--- /dev/null
+++ b/test/web/activity_pub/object_validators/types/safe_text_test.exs
@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeTextTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText
+
+  test "it lets normal text go through" do
+    text = "hey how are you"
+    assert {:ok, text} == SafeText.cast(text)
+  end
+
+  test "it removes html tags from text" do
+    text = "hey look xss <script>alert('foo')</script>"
+    assert {:ok, "hey look xss alert(&#39;foo&#39;)"} == SafeText.cast(text)
+  end
+
+  test "errors for non-text" do
+    assert :error == SafeText.cast(1)
+  end
+end
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 2889a577c..19abac6a6 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -47,14 +47,14 @@ test "notifies the recipient" do
       recipient = insert(:user, local: true)
 
       {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
-      {:ok, chat_message_object} = Object.create(chat_message_data)
 
       {:ok, create_activity_data, _meta} =
-        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+      {:ok, _create_activity, _meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
       assert Repo.get_by(Notification, user_id: recipient.id, activity_id: create_activity.id)
     end
@@ -64,14 +64,17 @@ test "it creates a Chat for the local users and bumps the unread count" do
       recipient = insert(:user, local: true)
 
       {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
-      {:ok, chat_message_object} = Object.create(chat_message_data)
 
       {:ok, create_activity_data, _meta} =
-        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+      {:ok, _create_activity, _meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
+
+      # An object is created
+      assert Object.get_by_ap_id(chat_message_data["id"])
 
       # The remote user won't get a chat
       chat = Chat.get(author.id, recipient.ap_id)
@@ -85,14 +88,14 @@ test "it creates a Chat for the local users and bumps the unread count" do
       recipient = insert(:user, local: true)
 
       {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
-      {:ok, chat_message_object} = Object.create(chat_message_data)
 
       {:ok, create_activity_data, _meta} =
-        Builder.create(author, chat_message_object.data["id"], [recipient.ap_id])
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      {:ok, _create_activity, _meta} = SideEffects.handle(create_activity)
+      {:ok, _create_activity, _meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
       # Both users are local and get the chat
       chat = Chat.get(author.id, recipient.ap_id)
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index a63a31e6e..ceaee614c 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -55,7 +55,8 @@ test "it rejects messages where the `to` field of activity and object don't matc
         data
         |> Map.put("to", author.ap_id)
 
-      {:error, _} = Transmogrifier.handle_incoming(data)
+      assert match?({:error, _}, Transmogrifier.handle_incoming(data))
+      refute Object.get_by_ap_id(data["object"]["id"])
     end
 
     test "it inserts it and creates a chat" do

From abd09282292f7e902c77b158ae3d86e9bfd5b986 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 28 Apr 2020 16:45:28 +0200
Subject: [PATCH 055/283] CreateChatMessageValidator: Validate object existence

---
 .../create_chat_message_validator.ex             | 14 +++++++++++++-
 test/web/activity_pub/object_validator_test.exs  | 16 ++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index 21c7a5ba4..dfc91bf71 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -5,10 +5,10 @@
 # NOTES
 # - Can probably be a generic create validator
 # - doesn't embed, will only get the object id
-# - object has to be validated first, maybe with some meta info from the surrounding create
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
   use Ecto.Schema
 
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
@@ -43,6 +43,18 @@ def validate_data(cng, meta \\ []) do
     |> validate_required([:id, :actor, :to, :type, :object])
     |> validate_inclusion(:type, ["Create"])
     |> validate_recipients_match(meta)
+    |> validate_object_nonexistence()
+  end
+
+  def validate_object_nonexistence(cng) do
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      if Object.get_cached_by_ap_id(object_id) do
+        [{:object, "The object to create already exists"}]
+      else
+        []
+      end
+    end)
   end
 
   def validate_recipients_match(cng, meta) do
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index bc2317e55..baa4b2585 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
@@ -9,6 +10,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
 
   import Pleroma.Factory
 
+  describe "chat message create activities" do
+    test "it is invalid if the object already exists" do
+      user = insert(:user)
+      recipient = insert(:user)
+      {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "hey")
+      object = Object.normalize(activity, false)
+
+      {:ok, create_data, _} = Builder.create(user, object.data["id"], [recipient.ap_id])
+
+      {:error, cng} = ObjectValidator.validate(create_data, [])
+
+      assert {:object, {"The object to create already exists", []}} in cng.errors
+    end
+  end
+
   describe "chat messages" do
     setup do
       clear_config([:instance, :remote_limit])

From dedffd100c231aa69d7a7f7cd7126b90a84fc1ec Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 28 Apr 2020 17:29:54 +0200
Subject: [PATCH 056/283] Pipeline: Unify, refactor, DRY.

---
 lib/pleroma/web/activity_pub/builder.ex       |  4 +--
 .../web/activity_pub/object_validator.ex      | 18 ++++++++---
 .../transmogrifier/chat_message_handling.ex   | 31 ++++---------------
 lib/pleroma/web/common_api/common_api.ex      |  5 ++-
 .../activity_pub/object_validator_test.exs    |  2 +-
 5 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index 7576ed278..7f9c071b3 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -11,13 +11,13 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
 
-  def create(actor, object_id, recipients) do
+  def create(actor, object, recipients) do
     {:ok,
      %{
        "id" => Utils.generate_activity_id(),
        "actor" => actor.ap_id,
        "to" => recipients,
-       "object" => object_id,
+       "object" => object,
        "type" => "Create",
        "published" => DateTime.utc_now() |> DateTime.to_iso8601()
      }, []}
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index a4da9242a..bada3509d 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -38,16 +38,24 @@ def validate(%{"type" => "ChatMessage"} = object, meta) do
     end
   end
 
-  def validate(%{"type" => "Create"} = object, meta) do
-    with {:ok, object} <-
-           object
+  def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do
+    with {:ok, object_data} <- cast_and_apply(object),
+         meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
+         {:ok, create_activity} <-
+           create_activity
            |> CreateChatMessageValidator.cast_and_validate(meta)
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
+      create_activity = stringify_keys(create_activity)
+      {:ok, create_activity, meta}
     end
   end
 
+  def cast_and_apply(%{"type" => "ChatMessage"} = object) do
+    ChatMessageValidator.cast_and_apply(object)
+  end
+
+  def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
+
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index 043d847d1..d9c36e313 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -4,9 +4,6 @@
 
 defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling do
   alias Pleroma.Repo
-  alias Pleroma.Web.ActivityPub.ObjectValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.Pipeline
 
   def handle_incoming(
@@ -15,31 +12,15 @@ def handle_incoming(
       ) do
     # Create has to be run inside a transaction because the object is created as a side effect.
     # If this does not work, we need to roll back creating the activity.
-    case Repo.transaction(fn -> do_handle_incoming(data) end) do
-      {:ok, value} ->
-        value
+    case Repo.transaction(fn -> Pipeline.common_pipeline(data, local: false) end) do
+      {:ok, {:ok, activity, _}} ->
+        {:ok, activity}
+
+      {:ok, e} ->
+        e
 
       {:error, e} ->
         {:error, e}
     end
   end
-
-  def do_handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object_data} = data
-      ) do
-    with {_, {:ok, cast_data_sym}} <-
-           {:casting_data, data |> CreateChatMessageValidator.cast_and_apply()},
-         cast_data = ObjectValidator.stringify_keys(cast_data_sym),
-         {_, {:ok, object_cast_data_sym}} <-
-           {:casting_object_data, object_data |> ChatMessageValidator.cast_and_apply()},
-         object_cast_data = ObjectValidator.stringify_keys(object_cast_data_sym),
-         {_, {:ok, activity, _meta}} <-
-           {:common_pipeline,
-            Pipeline.common_pipeline(cast_data, local: false, object_data: object_cast_data)} do
-      {:ok, activity}
-    else
-      e ->
-        {:error, e}
-    end
-  end
 end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index c39d1cee6..ef86ec1e4 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -40,12 +40,11 @@ def post_chat_message(%User{} = user, %User{} = recipient, content) do
                 )},
              {_, {:ok, create_activity_data, _meta}} <-
                {:build_create_activity,
-                Builder.create(user, chat_message_data["id"], [recipient.ap_id])},
+                Builder.create(user, chat_message_data, [recipient.ap_id])},
              {_, {:ok, %Activity{} = activity, _meta}} <-
                {:common_pipeline,
                 Pipeline.common_pipeline(create_activity_data,
-                  local: true,
-                  object_data: chat_message_data
+                  local: true
                 )} do
           {:ok, activity}
         else
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index baa4b2585..41f67964a 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -17,7 +17,7 @@ test "it is invalid if the object already exists" do
       {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "hey")
       object = Object.normalize(activity, false)
 
-      {:ok, create_data, _} = Builder.create(user, object.data["id"], [recipient.ap_id])
+      {:ok, create_data, _} = Builder.create(user, object.data, [recipient.ap_id])
 
       {:error, cng} = ObjectValidator.validate(create_data, [])
 

From 67659afe487def6bd4e0ccfbf8d015fda2a8ac61 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 13:34:43 +0200
Subject: [PATCH 057/283] ChatOperation: Refactor.

---
 .../web/api_spec/operations/chat_operation.ex | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 59539e890..88b9db048 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -5,11 +5,12 @@
 defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
-  alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
   alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
 
+  import Pleroma.Web.ApiSpec.Helpers
+
   @spec open_api_operation(atom) :: Operation.t()
   def open_api_operation(action) do
     operation = String.to_existing_atom("#{action}_operation")
@@ -52,11 +53,7 @@ def index_operation do
       tags: ["chat"],
       summary: "Get a list of chats that you participated in",
       operationId: "ChatController.index",
-      parameters: [
-        Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
-        Operation.parameter(:min_id, :query, :string, "Return only chats after this id"),
-        Operation.parameter(:max_id, :query, :string, "Return only chats before this id")
-      ],
+      parameters: pagination_params(),
       responses: %{
         200 => Operation.response("The chats of the user", "application/json", chats_response())
       },
@@ -73,12 +70,9 @@ def messages_operation do
       tags: ["chat"],
       summary: "Get the most recent messages of the chat",
       operationId: "ChatController.messages",
-      parameters: [
-        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
-        Operation.parameter(:limit, :query, :integer, "How many results to return", example: 20),
-        Operation.parameter(:min_id, :query, :string, "Return only messages after this id"),
-        Operation.parameter(:max_id, :query, :string, "Return only messages before this id")
-      ],
+      parameters:
+        [Operation.parameter(:id, :path, :string, "The ID of the Chat")] ++
+          pagination_params(),
       responses: %{
         200 =>
           Operation.response(
@@ -103,7 +97,7 @@ def post_chat_message_operation do
       parameters: [
         Operation.parameter(:id, :path, :string, "The ID of the Chat")
       ],
-      requestBody: Helpers.request_body("Parameters", ChatMessageCreateRequest, required: true),
+      requestBody: request_body("Parameters", ChatMessageCreateRequest, required: true),
       responses: %{
         200 =>
           Operation.response(

From e055b8d2036e18a95d84f6f1db08fc465fe9975d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 13:45:50 +0200
Subject: [PATCH 058/283] Pipeline: Always run common_pipeline in a transaction
 for now.

---
 lib/pleroma/web/activity_pub/pipeline.ex      | 11 ++++
 .../transmogrifier/chat_message_handling.ex   | 12 ++---
 lib/pleroma/web/common_api/common_api.ex      | 52 ++++++++-----------
 .../transmogrifier/chat_message_test.exs      |  1 +
 4 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 4213ba751..d5abb7567 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
@@ -14,6 +15,16 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
   @spec common_pipeline(map(), keyword()) ::
           {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}
   def common_pipeline(object, meta) do
+    case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do
+      {:ok, value} ->
+        value
+
+      {:error, e} ->
+        {:error, e}
+    end
+  end
+
+  def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
          {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
index d9c36e313..b1cc93481 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
@@ -3,24 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling do
-  alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.Pipeline
 
   def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
         _options
       ) do
-    # Create has to be run inside a transaction because the object is created as a side effect.
-    # If this does not work, we need to roll back creating the activity.
-    case Repo.transaction(fn -> Pipeline.common_pipeline(data, local: false) end) do
-      {:ok, {:ok, activity, _}} ->
+    case Pipeline.common_pipeline(data, local: false) do
+      {:ok, activity, _} ->
         {:ok, activity}
 
-      {:ok, e} ->
+      e ->
         e
-
-      {:error, e} ->
-        {:error, e}
     end
   end
 end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index ef86ec1e4..359045f48 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.FollowingRelationship
   alias Pleroma.Formatter
   alias Pleroma.Object
-  alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -26,36 +25,27 @@ defmodule Pleroma.Web.CommonAPI do
   require Logger
 
   def post_chat_message(%User{} = user, %User{} = recipient, content) do
-    transaction =
-      Repo.transaction(fn ->
-        with {_, true} <-
-               {:content_length,
-                String.length(content) <= Pleroma.Config.get([:instance, :chat_limit])},
-             {_, {:ok, chat_message_data, _meta}} <-
-               {:build_object,
-                Builder.chat_message(
-                  user,
-                  recipient.ap_id,
-                  content |> Formatter.html_escape("text/plain")
-                )},
-             {_, {:ok, create_activity_data, _meta}} <-
-               {:build_create_activity,
-                Builder.create(user, chat_message_data, [recipient.ap_id])},
-             {_, {:ok, %Activity{} = activity, _meta}} <-
-               {:common_pipeline,
-                Pipeline.common_pipeline(create_activity_data,
-                  local: true
-                )} do
-          {:ok, activity}
-        else
-          {:content_length, false} -> {:error, :content_too_long}
-          e -> e
-        end
-      end)
-
-    case transaction do
-      {:ok, value} -> value
-      error -> error
+    with {_, true} <-
+           {:content_length,
+            String.length(content) <= Pleroma.Config.get([:instance, :chat_limit])},
+         {_, {:ok, chat_message_data, _meta}} <-
+           {:build_object,
+            Builder.chat_message(
+              user,
+              recipient.ap_id,
+              content |> Formatter.html_escape("text/plain")
+            )},
+         {_, {:ok, create_activity_data, _meta}} <-
+           {:build_create_activity, Builder.create(user, chat_message_data, [recipient.ap_id])},
+         {_, {:ok, %Activity{} = activity, _meta}} <-
+           {:common_pipeline,
+            Pipeline.common_pipeline(create_activity_data,
+              local: true
+            )} do
+      {:ok, activity}
+    else
+      {:content_length, false} -> {:error, :content_too_long}
+      e -> e
     end
   end
 
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index ceaee614c..c5600e84e 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -68,6 +68,7 @@ test "it inserts it and creates a chat" do
       recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
 
       {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data)
+      assert activity.local == false
 
       assert activity.actor == author.ap_id
       assert activity.recipients == [recipient.ap_id, author.ap_id]

From 53e3063bd041409da83483e8f5c47030bf346123 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 13:52:23 +0200
Subject: [PATCH 059/283] Transmogrifier: Remove ChatMessageHandling module.

---
 lib/pleroma/web/activity_pub/side_effects.ex  | 13 ++++--------
 .../web/activity_pub/transmogrifier.ex        | 14 +++++++++----
 .../transmogrifier/chat_message_handling.ex   | 20 -------------------
 3 files changed, 14 insertions(+), 33 deletions(-)
 delete mode 100644 lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 794a46267..e394c75d7 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -19,17 +19,12 @@ def handle(object, meta \\ [])
   # - Add like to object
   # - Set up notification
   def handle(%{data: %{"type" => "Like"}} = object, meta) do
-    {:ok, result} =
-      Pleroma.Repo.transaction(fn ->
-        liked_object = Object.get_by_ap_id(object.data["object"])
-        Utils.add_like_to_object(object, liked_object)
+    liked_object = Object.get_by_ap_id(object.data["object"])
+    Utils.add_like_to_object(object, liked_object)
 
-        Notification.create_notifications(object)
+    Notification.create_notifications(object)
 
-        {:ok, object, meta}
-      end)
-
-    result
+    {:ok, object, meta}
   end
 
   # Tasks this handles
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 66975cf7d..3c2fe73a3 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -16,7 +16,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.Pipeline
-  alias Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
@@ -646,9 +645,16 @@ def handle_incoming(
 
   def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
-        options
-      ),
-      do: ChatMessageHandling.handle_incoming(data, options)
+        _options
+      ) do
+    case Pipeline.common_pipeline(data, local: false) do
+      {:ok, activity, _} ->
+        {:ok, activity}
+
+      e ->
+        e
+    end
+  end
 
   def handle_incoming(%{"type" => "Like"} = data, _options) do
     with {_, {:ok, cast_data_sym}} <-
diff --git a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex b/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
deleted file mode 100644
index b1cc93481..000000000
--- a/lib/pleroma/web/activity_pub/transmogrifier/chat_message_handling.ex
+++ /dev/null
@@ -1,20 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageHandling do
-  alias Pleroma.Web.ActivityPub.Pipeline
-
-  def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
-        _options
-      ) do
-    case Pipeline.common_pipeline(data, local: false) do
-      {:ok, activity, _} ->
-        {:ok, activity}
-
-      e ->
-        e
-    end
-  end
-end

From a88734a0a22810bcc47c17fc9120ef7881d670d8 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 14:25:33 +0200
Subject: [PATCH 060/283] Transmogrifier: Fetch missing actors for
 chatmessages.

---
 .../web/activity_pub/object_validator.ex      |  9 ++++-
 .../create_chat_message_validator.ex          |  2 +
 .../web/activity_pub/transmogrifier.ex        |  8 ++--
 .../transmogrifier/chat_message_test.exs      | 40 ++++++++++++++++---
 4 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index bada3509d..50904ed59 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
@@ -67,8 +68,14 @@ def stringify_keys(object) do
     |> Map.new(fn {key, val} -> {to_string(key), val} end)
   end
 
+  def fetch_actor(object) do
+    with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do
+      User.get_or_fetch_by_ap_id(actor)
+    end
+  end
+
   def fetch_actor_and_object(object) do
-    User.get_or_fetch_by_ap_id(object["actor"])
+    fetch_actor(object)
     Object.normalize(object["object"])
     :ok
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index dfc91bf71..88e903182 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -12,6 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   @primary_key false
 
@@ -42,6 +43,7 @@ def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:id, :actor, :to, :type, :object])
     |> validate_inclusion(:type, ["Create"])
+    |> validate_actor_presence()
     |> validate_recipients_match(meta)
     |> validate_object_nonexistence()
   end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 3c2fe73a3..6dbd3f588 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -647,10 +647,10 @@ def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
         _options
       ) do
-    case Pipeline.common_pipeline(data, local: false) do
-      {:ok, activity, _} ->
-        {:ok, activity}
-
+    with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
+         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+      {:ok, activity}
+    else
       e ->
         e
     end
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index c5600e84e..85644d787 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -26,8 +26,15 @@ test "it rejects messages that don't contain content" do
         data
         |> Map.put("object", object)
 
-      _author = insert(:user, ap_id: data["actor"], local: false)
-      _recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
+      _author =
+        insert(:user, ap_id: data["actor"], local: false, last_refreshed_at: DateTime.utc_now())
+
+      _recipient =
+        insert(:user,
+          ap_id: List.first(data["to"]),
+          local: true,
+          last_refreshed_at: DateTime.utc_now()
+        )
 
       {:error, _} = Transmogrifier.handle_incoming(data)
     end
@@ -37,8 +44,15 @@ test "it rejects messages that don't concern local users" do
         File.read!("test/fixtures/create-chat-message.json")
         |> Poison.decode!()
 
-      _author = insert(:user, ap_id: data["actor"], local: false)
-      _recipient = insert(:user, ap_id: List.first(data["to"]), local: false)
+      _author =
+        insert(:user, ap_id: data["actor"], local: false, last_refreshed_at: DateTime.utc_now())
+
+      _recipient =
+        insert(:user,
+          ap_id: List.first(data["to"]),
+          local: false,
+          last_refreshed_at: DateTime.utc_now()
+        )
 
       {:error, _} = Transmogrifier.handle_incoming(data)
     end
@@ -59,12 +73,28 @@ test "it rejects messages where the `to` field of activity and object don't matc
       refute Object.get_by_ap_id(data["object"]["id"])
     end
 
+    test "it fetches the actor if they aren't in our system" do
+      Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+      data =
+        File.read!("test/fixtures/create-chat-message.json")
+        |> Poison.decode!()
+        |> Map.put("actor", "http://mastodon.example.org/users/admin")
+        |> put_in(["object", "actor"], "http://mastodon.example.org/users/admin")
+
+      _recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
+
+      {:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data)
+    end
+
     test "it inserts it and creates a chat" do
       data =
         File.read!("test/fixtures/create-chat-message.json")
         |> Poison.decode!()
 
-      author = insert(:user, ap_id: data["actor"], local: false)
+      author =
+        insert(:user, ap_id: data["actor"], local: false, last_refreshed_at: DateTime.utc_now())
+
       recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
 
       {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data)

From 20587aa931262a5479c98f13450311a135c5d356 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 14:53:53 +0200
Subject: [PATCH 061/283] Chat message creation: Check actor.

---
 .../create_chat_message_validator.ex               | 14 ++++++++++++++
 test/web/activity_pub/object_validator_test.exs    | 13 +++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index 88e903182..fc582400b 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -45,6 +45,7 @@ def validate_data(cng, meta \\ []) do
     |> validate_inclusion(:type, ["Create"])
     |> validate_actor_presence()
     |> validate_recipients_match(meta)
+    |> validate_actors_match(meta)
     |> validate_object_nonexistence()
   end
 
@@ -59,6 +60,19 @@ def validate_object_nonexistence(cng) do
     end)
   end
 
+  def validate_actors_match(cng, meta) do
+    object_actor = meta[:object_data]["actor"]
+
+    cng
+    |> validate_change(:actor, fn :actor, actor ->
+      if actor == object_actor do
+        []
+      else
+        [{:actor, "Actor doesn't match with object actor"}]
+      end
+    end)
+  end
+
   def validate_recipients_match(cng, meta) do
     object_recipients = meta[:object_data]["to"] || []
 
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 41f67964a..475b7bb21 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -23,6 +23,19 @@ test "it is invalid if the object already exists" do
 
       assert {:object, {"The object to create already exists", []}} in cng.errors
     end
+
+    test "it is invalid if the object data has a different `to` or `actor` field" do
+      user = insert(:user)
+      recipient = insert(:user)
+      {:ok, object_data, _} = Builder.chat_message(recipient, user.ap_id, "Hey")
+
+      {:ok, create_data, _} = Builder.create(user, object_data, [recipient.ap_id])
+
+      {:error, cng} = ObjectValidator.validate(create_data, [])
+
+      assert {:to, {"Recipients don't match with object recipients", []}} in cng.errors
+      assert {:actor, {"Actor doesn't match with object actor", []}} in cng.errors
+    end
   end
 
   describe "chat messages" do

From 528ea779a61d12f74ee9669bbd28783bf66aa5cc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 17:56:24 +0000
Subject: [PATCH 062/283] Apply suggestion to docs/API/chats.md

---
 docs/API/chats.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 24c4b4d06..26e83570e 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -99,7 +99,7 @@ For a given Chat id, you can get the associated messages with
 `GET /api/v1/pleroma/chats/{id}/messages`
 
 This will return all messages, sorted by most recent to least recent. The usual
-pagination options are implemented
+pagination options are implemented.
 
 Returned data:
 

From 89a6c340812a53daf00a203dacd8e12a25eb7ad2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 18:14:34 +0000
Subject: [PATCH 063/283] Apply suggestion to lib/pleroma/chat.ex

---
 lib/pleroma/chat.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index b8545063a..6b1f832ce 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -29,7 +29,7 @@ def creation_cng(struct, params) do
     |> validate_change(:recipient, fn
       :recipient, recipient ->
         case User.get_cached_by_ap_id(recipient) do
-          nil -> [recipient: "must a an existing user"]
+          nil -> [recipient: "must be an existing user"]
           _ -> []
         end
     end)

From 589ce1e96bcaba0bd2d864d3528992f10f4cf5f7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 19:47:16 +0000
Subject: [PATCH 064/283] Apply suggestion to
 lib/pleroma/web/activity_pub/transmogrifier.ex

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 6dbd3f588..d3a2e0362 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -650,9 +650,6 @@ def handle_incoming(
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
-    else
-      e ->
-        e
     end
   end
 

From 145d35ff70a59efcff881315d5f1f7a0248a34be Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 19:49:03 +0000
Subject: [PATCH 065/283] Apply suggestion to
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex

---
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 8654f4295..175257921 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -31,7 +31,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
     %{scopes: ["read:statuses"]} when action in [:messages, :index]
   )
 
-  plug(OpenApiSpex.Plug.CastAndValidate)
+  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 

From b68d56c8168f27f63e157d43558e22f7c221c0e2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Apr 2020 19:49:13 +0000
Subject: [PATCH 066/283] Apply suggestion to
 lib/pleroma/web/api_spec/schemas/chat_message_response.ex

---
 lib/pleroma/web/api_spec/schemas/chat_message_response.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
index b7a662cbb..707c9808b 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
       account_id: %Schema{type: :string, description: "The Mastodon API id of the actor"},
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string},
-      created_at: %Schema{type: :string, format: :datetime},
+      created_at: %Schema{type: :string, format: :"date-time"},
       emojis: %Schema{type: :array}
     },
     example: %{

From ad2182bbd231b475c5bfc70485f35ad1f8841912 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 30 Apr 2020 11:38:26 +0000
Subject: [PATCH 067/283] Apply suggestion to
 lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex

---
 lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex b/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
index 4dafcda43..8e1b7af14 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest do
     properties: %{
       content: %Schema{type: :string, description: "The content of your message"}
     },
+    required: [:content],
     example: %{
       "content" => "Hey wanna buy feet pics?"
     }

From a35b76431ce7c7bd7ed62374d781778922f0fe2f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 3 May 2020 14:58:24 +0200
Subject: [PATCH 068/283] Credo fixes.

---
 lib/pleroma/web/activity_pub/object_validator.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 50904ed59..20c7cceb6 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -11,10 +11,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   alias Pleroma.Object
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)

From 9249742f13445f47167d4b352751c49caf48aa8f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 3 May 2020 15:28:24 +0200
Subject: [PATCH 069/283] Types.Recipients: Simplify reducer.

---
 .../object_validators/types/recipients.ex            | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
index 48fe61e1a..408e0f6ee 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
@@ -11,11 +11,13 @@ def cast(object) when is_binary(object) do
 
   def cast(data) when is_list(data) do
     data
-    |> Enum.reduce({:ok, []}, fn element, acc ->
-      case {acc, ObjectID.cast(element)} do
-        {:error, _} -> :error
-        {_, :error} -> :error
-        {{:ok, list}, {:ok, id}} -> {:ok, [id | list]}
+    |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
+      case ObjectID.cast(element) do
+        {:ok, id} ->
+          {:cont, {:ok, [id | list]}}
+
+        _ ->
+          {:halt, :error}
       end
     end)
   end

From 651935f1379a1ed3c89e473803251310c13ea571 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 11:08:00 +0200
Subject: [PATCH 070/283] Schemas: Refactor to our naming scheme.

---
 .../web/api_spec/operations/chat_operation.ex        | 12 ++++++------
 .../api_spec/schemas/{chat_response.ex => chat.ex}   |  4 ++--
 .../{chat_message_response.ex => chat_message.ex}    |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)
 rename lib/pleroma/web/api_spec/schemas/{chat_response.ex => chat.ex} (96%)
 rename lib/pleroma/web/api_spec/schemas/{chat_message_response.ex => chat_message.ex} (91%)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 88b9db048..fc9d4608a 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse
-  alias Pleroma.Web.ApiSpec.Schemas.ChatResponse
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
+  alias Pleroma.Web.ApiSpec.Schemas.Chat
 
   import Pleroma.Web.ApiSpec.Helpers
 
@@ -37,7 +37,7 @@ def create_operation do
           Operation.response(
             "The created or existing chat",
             "application/json",
-            ChatResponse
+            Chat
           )
       },
       security: [
@@ -103,7 +103,7 @@ def post_chat_message_operation do
           Operation.response(
             "The newly created ChatMessage",
             "application/json",
-            ChatMessageResponse
+            ChatMessage
           )
       },
       security: [
@@ -119,7 +119,7 @@ def chats_response do
       title: "ChatsResponse",
       description: "Response schema for multiple Chats",
       type: :array,
-      items: ChatResponse,
+      items: Chat,
       example: [
         %{
           "account" => %{
@@ -180,7 +180,7 @@ def chat_messages_response do
       title: "ChatMessagesResponse",
       description: "Response schema for multiple ChatMessages",
       type: :array,
-      items: ChatMessageResponse,
+      items: ChatMessage,
       example: [
         %{
           "emojis" => [
diff --git a/lib/pleroma/web/api_spec/schemas/chat_response.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
similarity index 96%
rename from lib/pleroma/web/api_spec/schemas/chat_response.ex
rename to lib/pleroma/web/api_spec/schemas/chat.ex
index aa435165d..4d385d6ab 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -2,13 +2,13 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ApiSpec.Schemas.ChatResponse do
+defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
   alias OpenApiSpex.Schema
 
   require OpenApiSpex
 
   OpenApiSpex.schema(%{
-    title: "ChatResponse",
+    title: "Chat",
     description: "Response schema for a Chat",
     type: :object,
     properties: %{
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex
similarity index 91%
rename from lib/pleroma/web/api_spec/schemas/chat_message_response.ex
rename to lib/pleroma/web/api_spec/schemas/chat_message.ex
index 707c9808b..7c93b0c83 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message_response.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex
@@ -2,13 +2,13 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageResponse do
+defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
   alias OpenApiSpex.Schema
 
   require OpenApiSpex
 
   OpenApiSpex.schema(%{
-    title: "ChatMessageResponse",
+    title: "ChatMessage",
     description: "Response schema for a ChatMessage",
     type: :object,
     properties: %{

From dcf535fe770b638b68928f238f4d8d1cfd410524 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 11:32:11 +0200
Subject: [PATCH 071/283] Credo fixes.

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index fc9d4608a..ad05f5ac7 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
   alias Pleroma.Web.ApiSpec.Schemas.Chat
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
 
   import Pleroma.Web.ApiSpec.Helpers
 

From 57e6f2757afef8941fe3576dbe5e2014d2569c33 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 12:47:23 +0200
Subject: [PATCH 072/283] ChatOperation: Make simple schema into inline schema

---
 .../web/api_spec/operations/chat_operation.ex | 18 ++++++++++++++--
 .../schemas/chat_message_create_request.ex    | 21 -------------------
 2 files changed, 16 insertions(+), 23 deletions(-)
 delete mode 100644 lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index ad05f5ac7..e8b5eff1f 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.Chat
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
-  alias Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest
 
   import Pleroma.Web.ApiSpec.Helpers
 
@@ -97,7 +96,7 @@ def post_chat_message_operation do
       parameters: [
         Operation.parameter(:id, :path, :string, "The ID of the Chat")
       ],
-      requestBody: request_body("Parameters", ChatMessageCreateRequest, required: true),
+      requestBody: request_body("Parameters", chat_message_create(), required: true),
       responses: %{
         200 =>
           Operation.response(
@@ -208,4 +207,19 @@ def chat_messages_response do
       ]
     }
   end
+
+  def chat_message_create do
+    %Schema{
+      title: "ChatMessageCreateRequest",
+      description: "POST body for creating an chat message",
+      type: :object,
+      properties: %{
+        content: %Schema{type: :string, description: "The content of your message"}
+      },
+      required: [:content],
+      example: %{
+        "content" => "Hey wanna buy feet pics?"
+      }
+    }
+  end
 end
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex b/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
deleted file mode 100644
index 8e1b7af14..000000000
--- a/lib/pleroma/web/api_spec/schemas/chat_message_create_request.ex
+++ /dev/null
@@ -1,21 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessageCreateRequest do
-  alias OpenApiSpex.Schema
-  require OpenApiSpex
-
-  OpenApiSpex.schema(%{
-    title: "ChatMessageCreateRequest",
-    description: "POST body for creating an chat message",
-    type: :object,
-    properties: %{
-      content: %Schema{type: :string, description: "The content of your message"}
-    },
-    required: [:content],
-    example: %{
-      "content" => "Hey wanna buy feet pics?"
-    }
-  })
-end

From 30590cf46b88d0008c9a7163b8339aa9376f2378 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 12:53:40 +0200
Subject: [PATCH 073/283] CommonAPI: Refactor for readability

---
 lib/pleroma/web/common_api/common_api.ex | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 1eda0b2f2..e428cc17d 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -26,9 +26,7 @@ defmodule Pleroma.Web.CommonAPI do
   require Logger
 
   def post_chat_message(%User{} = user, %User{} = recipient, content) do
-    with {_, true} <-
-           {:content_length,
-            String.length(content) <= Pleroma.Config.get([:instance, :chat_limit])},
+    with :ok <- validate_chat_content_length(content),
          {_, {:ok, chat_message_data, _meta}} <-
            {:build_object,
             Builder.chat_message(
@@ -44,9 +42,14 @@ def post_chat_message(%User{} = user, %User{} = recipient, content) do
               local: true
             )} do
       {:ok, activity}
+    end
+  end
+
+  defp validate_chat_content_length(content) do
+    if String.length(content) <= Pleroma.Config.get([:instance, :chat_limit]) do
+      :ok
     else
-      {:content_length, false} -> {:error, :content_too_long}
-      e -> e
+      {:error, :content_too_long}
     end
   end
 

From b04328c3dec4812dbaf3cd89baa2b888d7bb7fbf Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 13:10:36 +0200
Subject: [PATCH 074/283] ChatController: Add mark_as_read

---
 lib/pleroma/chat.ex                           |  6 +++++
 .../web/api_spec/operations/chat_operation.ex | 22 ++++++++++++++++++
 .../controllers/chat_controller.ex            | 11 ++++++++-
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 23 +++++++++++++++++++
 5 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 6b1f832ce..6008196e4 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -60,4 +60,10 @@ def bump_or_create(user_id, recipient) do
       conflict_target: [:user_id, :recipient]
     )
   end
+
+  def mark_as_read(chat) do
+    chat
+    |> change(%{unread: 0})
+    |> Repo.update()
+  end
 end
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index e8b5eff1f..0fe0e07b2 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -16,6 +16,28 @@ def open_api_operation(action) do
     apply(__MODULE__, operation, [])
   end
 
+  def mark_as_read_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Mark all messages in the chat as read",
+      operationId: "ChatController.mark_as_read",
+      parameters: [Operation.parameter(:id, :path, :string, "The ID of the Chat")],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The updated chat",
+            "application/json",
+            Chat
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write"]
+        }
+      ]
+    }
+  end
+
   def create_operation do
     %Operation{
       tags: ["chat"],
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 175257921..bedae73bd 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["write:statuses"]} when action in [:post_chat_message, :create]
+    %{scopes: ["write:statuses"]} when action in [:post_chat_message, :create, :mark_as_read]
   )
 
   plug(
@@ -51,6 +51,15 @@ def post_chat_message(
     end
   end
 
+  def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
+    with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
+         {:ok, chat} <- Chat.mark_as_read(chat) do
+      conn
+      |> put_view(ChatView)
+      |> render("show.json", chat: chat)
+    end
+  end
+
   def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       messages =
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 3a5063d4a..d6803e8ac 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -293,6 +293,7 @@ defmodule Pleroma.Web.Router do
       get("/chats", ChatController, :index)
       get("/chats/:id/messages", ChatController, :messages)
       post("/chats/:id/messages", ChatController, :post_chat_message)
+      post("/chats/:id/read", ChatController, :mark_as_read)
     end
 
     scope [] do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index b1044574b..cdb2683c8 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -9,6 +9,29 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   import Pleroma.Factory
 
+  describe "POST /api/v1/pleroma/chats/:id/read" do
+    setup do: oauth_access(["write:statuses"])
+
+    test "it marks all messages in a chat as read", %{conn: conn, user: user} do
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+
+      assert chat.unread == 1
+
+      result =
+        conn
+        |> post("/api/v1/pleroma/chats/#{chat.id}/read")
+        |> json_response_and_validate_schema(200)
+
+      assert result["unread"] == 0
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      assert chat.unread == 0
+    end
+  end
+
   describe "POST /api/v1/pleroma/chats/:id/messages" do
     setup do: oauth_access(["write:statuses"])
 

From 7ff2a7dae2fa651cea579aeca40e2c030d19fcd5 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 4 May 2020 13:12:21 +0200
Subject: [PATCH 075/283] Docs: Add Chat mark_as_read docs

---
 docs/API/chats.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 26e83570e..8d925989c 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -66,6 +66,27 @@ Returned data:
 }
 ```
 
+### Marking a chat as read
+
+To set the `unread` count of a chat to 0, call
+
+`POST /api/v1/pleroma/chats/:id/read`
+
+Returned data:
+
+```json
+{
+  "account": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "id" : "1",
+  "unread" : 0
+}
+```
+
+
 ### Getting a list of Chats
 
 `GET /api/v1/pleroma/chats`

From 9637cded21cef1e6c531dd46d5f5245c4c3ed03c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 5 May 2020 20:07:47 +0200
Subject: [PATCH 076/283] Chat: Fix missing chat id on second 'get'

---
 lib/pleroma/chat.ex |  3 ++-
 test/chat_test.exs  | 13 ++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 6008196e4..1a092b992 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -46,7 +46,8 @@ def get_or_create(user_id, recipient) do
     %__MODULE__{}
     |> creation_cng(%{user_id: user_id, recipient: recipient})
     |> Repo.insert(
-      on_conflict: :nothing,
+      # Need to set something, otherwise we get nothing back at all
+      on_conflict: [set: [recipient: recipient]],
       returning: true,
       conflict_target: [:user_id, :recipient]
     )
diff --git a/test/chat_test.exs b/test/chat_test.exs
index 952598c87..943e48111 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -26,13 +26,24 @@ test "it creates a chat for a user and recipient" do
       assert chat.id
     end
 
-    test "it returns a chat for a user and recipient if it already exists" do
+    test "it returns and bumps a chat for a user and recipient if it already exists" do
       user = insert(:user)
       other_user = insert(:user)
 
       {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
       {:ok, chat_two} = Chat.bump_or_create(user.id, other_user.ap_id)
 
+      assert chat.id == chat_two.id
+      assert chat_two.unread == 2
+    end
+
+    test "it returns a chat for a user and recipient if it already exists" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      {:ok, chat_two} = Chat.get_or_create(user.id, other_user.ap_id)
+
       assert chat.id == chat_two.id
     end
 

From 20baa2eaf04425cf0a2eebc84760be6c12ee7f51 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 6 May 2020 16:12:36 +0200
Subject: [PATCH 077/283] ChatMessages: Add attachments.

---
 lib/pleroma/web/activity_pub/builder.ex       | 33 ++++++---
 .../web/activity_pub/object_validator.ex      | 11 ++-
 .../object_validators/attachment_validator.ex | 72 +++++++++++++++++++
 .../chat_message_validator.ex                 |  6 +-
 .../object_validators/url_object_validator.ex | 20 ++++++
 .../web/api_spec/operations/chat_operation.ex |  3 +-
 .../web/api_spec/schemas/chat_message.ex      |  6 +-
 lib/pleroma/web/common_api/common_api.ex      |  6 +-
 .../controllers/chat_controller.ex            |  6 +-
 .../pleroma_api/views/chat_message_view.ex    |  5 +-
 .../activity_pub/object_validator_test.exs    | 50 ++++++++++++-
 .../types/object_id_test.exs                  |  4 ++
 .../controllers/chat_controller_test.exs      | 27 +++++++
 .../views/chat_message_view_test.exs          | 12 +++-
 14 files changed, 237 insertions(+), 24 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index 7f9c071b3..67e65c7b9 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -23,17 +23,28 @@ def create(actor, object, recipients) do
      }, []}
   end
 
-  def chat_message(actor, recipient, content) do
-    {:ok,
-     %{
-       "id" => Utils.generate_object_id(),
-       "actor" => actor.ap_id,
-       "type" => "ChatMessage",
-       "to" => [recipient],
-       "content" => content,
-       "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
-       "emoji" => Emoji.Formatter.get_emoji_map(content)
-     }, []}
+  def chat_message(actor, recipient, content, opts \\ []) do
+    basic = %{
+      "id" => Utils.generate_object_id(),
+      "actor" => actor.ap_id,
+      "type" => "ChatMessage",
+      "to" => [recipient],
+      "content" => content,
+      "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
+      "emoji" => Emoji.Formatter.get_emoji_map(content)
+    }
+
+    case opts[:attachment] do
+      %Object{data: attachment_data} ->
+        {
+          :ok,
+          Map.put(basic, "attachment", attachment_data),
+          []
+        }
+
+      _ ->
+        {:ok, basic, []}
+    end
   end
 
   @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()}
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 20c7cceb6..d6c14f7b8 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -63,11 +63,18 @@ def stringify_keys(%{__struct__: _} = object) do
     |> stringify_keys
   end
 
-  def stringify_keys(object) do
+  def stringify_keys(object) when is_map(object) do
     object
-    |> Map.new(fn {key, val} -> {to_string(key), val} end)
+    |> Map.new(fn {key, val} -> {to_string(key), stringify_keys(val)} end)
   end
 
+  def stringify_keys(object) when is_list(object) do
+    object
+    |> Enum.map(&stringify_keys/1)
+  end
+
+  def stringify_keys(object), do: object
+
   def fetch_actor(object) do
     with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do
       User.get_or_fetch_by_ap_id(actor)
diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
new file mode 100644
index 000000000..16ed49051
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -0,0 +1,72 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
+
+  import Ecto.Changeset
+
+  @primary_key false
+  embedded_schema do
+    field(:type, :string)
+    field(:mediaType, :string)
+    field(:name, :string)
+
+    embeds_many(:url, UrlObjectValidator)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    data =
+      data
+      |> fix_media_type()
+      |> fix_url()
+
+    struct
+    |> cast(data, [:type, :mediaType, :name])
+    |> cast_embed(:url, required: true)
+  end
+
+  def fix_media_type(data) do
+    data
+    |> Map.put_new("mediaType", data["mimeType"])
+  end
+
+  def fix_url(data) do
+    case data["url"] do
+      url when is_binary(url) ->
+        data
+        |> Map.put(
+          "url",
+          [
+            %{
+              "href" => url,
+              "type" => "Link",
+              "mediaType" => data["mediaType"]
+            }
+          ]
+        )
+
+      _ ->
+        data
+    end
+  end
+
+  def validate_data(cng) do
+    cng
+    |> validate_required([:mediaType, :url, :type])
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index e87c1ac2e..99ffeba28 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
 
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1]
@@ -22,6 +23,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
     field(:actor, Types.ObjectID)
     field(:published, Types.DateTime)
     field(:emoji, :map, default: %{})
+
+    embeds_one(:attachment, AttachmentValidator)
   end
 
   def cast_and_apply(data) do
@@ -51,7 +54,8 @@ def changeset(struct, data) do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields))
+    |> cast(data, List.delete(__schema__(:fields), :attachment))
+    |> cast_embed(:attachment)
   end
 
   def validate_data(data_cng) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
new file mode 100644
index 000000000..47e231150
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
@@ -0,0 +1,20 @@
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+  @primary_key false
+
+  embedded_schema do
+    field(:type, :string)
+    field(:href, Types.Uri)
+    field(:mediaType, :string)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+    |> validate_required([:type, :href, :mediaType])
+  end
+end
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 0fe0e07b2..8b9dc2e44 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -236,7 +236,8 @@ def chat_message_create do
       description: "POST body for creating an chat message",
       type: :object,
       properties: %{
-        content: %Schema{type: :string, description: "The content of your message"}
+        content: %Schema{type: :string, description: "The content of your message"},
+        media_id: %Schema{type: :string, description: "The id of an upload"}
       },
       required: [:content],
       example: %{
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex
index 7c93b0c83..89e062ddd 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex
@@ -17,7 +17,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string},
       created_at: %Schema{type: :string, format: :"date-time"},
-      emojis: %Schema{type: :array}
+      emojis: %Schema{type: :array},
+      attachment: %Schema{type: :object, nullable: true}
     },
     example: %{
       "account_id" => "someflakeid",
@@ -32,7 +33,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
           "url" => "https://dontbulling.me/emoji/Firefox.gif"
         }
       ],
-      "id" => "14"
+      "id" => "14",
+      "attachment" => nil
     }
   })
 end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e428cc17d..38b5c6f7c 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -25,14 +25,16 @@ defmodule Pleroma.Web.CommonAPI do
   require Pleroma.Constants
   require Logger
 
-  def post_chat_message(%User{} = user, %User{} = recipient, content) do
+  def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do
     with :ok <- validate_chat_content_length(content),
+         maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]),
          {_, {:ok, chat_message_data, _meta}} <-
            {:build_object,
             Builder.chat_message(
               user,
               recipient.ap_id,
-              content |> Formatter.html_escape("text/plain")
+              content |> Formatter.html_escape("text/plain"),
+              attachment: maybe_attachment
             )},
          {_, {:ok, create_activity_data, _meta}} <-
            {:build_create_activity, Builder.create(user, chat_message_data, [recipient.ap_id])},
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index bedae73bd..450d85332 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -36,14 +36,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
   def post_chat_message(
-        %{body_params: %{content: content}, assigns: %{user: %{id: user_id} = user}} = conn,
+        %{body_params: %{content: content} = params, assigns: %{user: %{id: user_id} = user}} =
+          conn,
         %{
           id: id
         }
       ) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
-         {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, content),
+         {:ok, activity} <-
+           CommonAPI.post_chat_message(user, recipient, content, media_id: params[:media_id]),
          message <- Object.normalize(activity) do
       conn
       |> put_view(ChatMessageView)
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
index a821479ab..b088a8734 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
@@ -23,7 +23,10 @@ def render(
       chat_id: chat_id |> to_string(),
       account_id: User.get_cached_by_ap_id(chat_message["actor"]).id,
       created_at: Utils.to_masto_date(chat_message["published"]),
-      emojis: StatusView.build_emojis(chat_message["emoji"])
+      emojis: StatusView.build_emojis(chat_message["emoji"]),
+      attachment:
+        chat_message["attachment"] &&
+          StatusView.render("attachment.json", attachment: chat_message["attachment"])
     }
   end
 
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 60db7187f..951ed7800 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -2,14 +2,41 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   use Pleroma.DataCase
 
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
+  describe "attachments" do
+    test "it turns mastodon attachments into our attachments" do
+      attachment = %{
+        "url" =>
+          "http://mastodon.example.org/system/media_attachments/files/000/000/002/original/334ce029e7bfb920.jpg",
+        "type" => "Document",
+        "name" => nil,
+        "mediaType" => "image/jpeg"
+      }
+
+      {:ok, attachment} =
+        AttachmentValidator.cast_and_validate(attachment)
+        |> Ecto.Changeset.apply_action(:insert)
+
+      assert [
+               %{
+                 href:
+                   "http://mastodon.example.org/system/media_attachments/files/000/000/002/original/334ce029e7bfb920.jpg",
+                 type: "Link",
+                 mediaType: "image/jpeg"
+               }
+             ] = attachment.url
+    end
+  end
+
   describe "chat message create activities" do
     test "it is invalid if the object already exists" do
       user = insert(:user)
@@ -52,7 +79,28 @@ test "it is invalid if the object data has a different `to` or `actor` field" do
     test "validates for a basic object we build", %{valid_chat_message: valid_chat_message} do
       assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
 
-      assert object == valid_chat_message
+      assert Map.put(valid_chat_message, "attachment", nil) == object
+    end
+
+    test "validates for a basic object with an attachment", %{
+      valid_chat_message: valid_chat_message,
+      user: user
+    } do
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, attachment} = ActivityPub.upload(file, actor: user.ap_id)
+
+      valid_chat_message =
+        valid_chat_message
+        |> Map.put("attachment", attachment.data)
+
+      assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+
+      assert object["attachment"]
     end
 
     test "does not validate if the message is longer than the remote_limit", %{
diff --git a/test/web/activity_pub/object_validators/types/object_id_test.exs b/test/web/activity_pub/object_validators/types/object_id_test.exs
index 834213182..c8911948e 100644
--- a/test/web/activity_pub/object_validators/types/object_id_test.exs
+++ b/test/web/activity_pub/object_validators/types/object_id_test.exs
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ObjectValidators.Types.ObjectIDTest do
   alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
   use Pleroma.DataCase
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index cdb2683c8..72a9a91ff 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   alias Pleroma.Chat
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Pleroma.Factory
 
@@ -49,6 +50,32 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
       assert result["content"] == "Hallo!!"
       assert result["chat_id"] == chat.id |> to_string()
     end
+
+    test "it works with an attachment", %{conn: conn, user: user} do
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
+
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{
+          "content" => "Hallo!!",
+          "media_id" => to_string(upload.id)
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert result["content"] == "Hallo!!"
+      assert result["chat_id"] == chat.id |> to_string()
+    end
   end
 
   describe "GET /api/v1/pleroma/chats/:id/messages" do
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index 5c4c8b0d5..a13a41daa 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -9,12 +9,21 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Pleroma.Factory
 
   test "it displays a chat message" do
     user = insert(:user)
     recipient = insert(:user)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
     {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:")
 
     chat = Chat.get(user.id, recipient.ap_id)
@@ -30,7 +39,7 @@ test "it displays a chat message" do
     assert chat_message[:created_at]
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
 
-    {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk")
+    {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk", media_id: upload.id)
 
     object = Object.normalize(activity)
 
@@ -40,5 +49,6 @@ test "it displays a chat message" do
     assert chat_message_two[:content] == "gkgkgk"
     assert chat_message_two[:account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
+    assert chat_message_two[:attachment]
   end
 end

From fc9d0b6eec1b206a27f4ec19f7939b3318a209ef Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 6 May 2020 16:31:21 +0200
Subject: [PATCH 078/283] Credo fixes.

---
 .../activity_pub/object_validators/chat_message_validator.ex    | 2 +-
 test/web/activity_pub/object_validator_test.exs                 | 2 +-
 test/web/pleroma_api/controllers/chat_controller_test.exs       | 2 +-
 test/web/pleroma_api/views/chat_message_view_test.exs           | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 99ffeba28..e40c80ab4 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   use Ecto.Schema
 
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1]
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 951ed7800..fcc54c8a1 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 72a9a91ff..b4b73da90 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,8 +5,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_view_test.exs
index a13a41daa..d7a2d10a5 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_view_test.exs
@@ -7,9 +7,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
 
   alias Pleroma.Chat
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.ChatMessageView
-  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Pleroma.Factory
 

From d0bf8cfb8f852a16259af4b808565cdfd58f5e61 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 8 May 2020 14:11:58 +0200
Subject: [PATCH 079/283] Credo fixes.

---
 lib/pleroma/web/activity_pub/side_effects.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 28b519432..c8b675d54 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   liked object, a `Follow` activity will add the user to the follower
   collection, and so on.
   """
-  alias Pleroma.Chat
   alias Pleroma.Activity
+  alias Pleroma.Chat
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo

From 03529f6a0528ed01c7a956bb80628910584a9580 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 8 May 2020 18:26:35 +0200
Subject: [PATCH 080/283] Transmogrifier: Don't modify attachments for chats.

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 3 +++
 test/web/common_api/common_api_test.exs        | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 29f668cad..f04dec6be 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -1114,6 +1114,9 @@ def add_attributed_to(object) do
     Map.put(object, "attributedTo", attributed_to)
   end
 
+  # TODO: Revisit this
+  def prepare_attachments(%{"type" => "ChatMessage"} = object), do: object
+
   def prepare_attachments(object) do
     attachments =
       object
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 61affda5d..5501ba18b 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -54,6 +54,8 @@ test "it posts a chat message" do
 
       assert Chat.get(author.id, recipient.ap_id)
       assert Chat.get(recipient.id, author.ap_id)
+
+      assert :ok == Pleroma.Web.Federator.perform(:publish, activity)
     end
 
     test "it reject messages over the local limit" do

From 1054e897622d0a0727f30169d64c83a253a3d11e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 12:30:24 +0200
Subject: [PATCH 081/283] ChatOperation: Add media id to example

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 8b9dc2e44..16d3d5e22 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -241,7 +241,8 @@ def chat_message_create do
       },
       required: [:content],
       example: %{
-        "content" => "Hey wanna buy feet pics?"
+        "content" => "Hey wanna buy feet pics?",
+        "media_id" => "134234"
       }
     }
   end

From e297d8c649a03510023cff61dc6e0c7131bc29dc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 12:34:12 +0200
Subject: [PATCH 082/283] Documentation: Add attachment docs

---
 docs/API/chats.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 8d925989c..3ddc13541 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -160,6 +160,7 @@ Posting a chat message for given Chat id works like this:
 
 Parameters:
 - content: The text content of the message
+- media_id: The id of an upload that will be attached to the message.
 
 Currently, no formatting beyond basic escaping and emoji is implemented, as well as no
 attachments. This will most probably change.

From f335e1404a9cd19451b531e32e3591aa323761ff Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:00:01 +0200
Subject: [PATCH 083/283] ChatView: Add the last message to the view.

---
 lib/pleroma/chat.ex                           | 36 ++++++++++++++++++-
 .../controllers/chat_controller.ex            | 23 ++----------
 .../web/pleroma_api/views/chat_view.ex        |  7 +++-
 test/web/pleroma_api/views/chat_view_test.exs | 17 ++++++++-
 4 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 1a092b992..6a03ee3c1 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -4,8 +4,11 @@
 
 defmodule Pleroma.Chat do
   use Ecto.Schema
-  import Ecto.Changeset
 
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -23,6 +26,37 @@ defmodule Pleroma.Chat do
     timestamps()
   end
 
+  def last_message_for_chat(chat) do
+    messages_for_chat_query(chat)
+    |> order_by(desc: :id)
+    |> Repo.one()
+  end
+
+  def messages_for_chat_query(chat) do
+    chat =
+      chat
+      |> Repo.preload(:user)
+
+    from(o in Object,
+      where: fragment("?->>'type' = ?", o.data, "ChatMessage"),
+      where:
+        fragment(
+          """
+          (?->>'actor' = ? and ?->'to' = ?) 
+          OR (?->>'actor' = ? and ?->'to' = ?) 
+          """,
+          o.data,
+          ^chat.user.ap_id,
+          o.data,
+          ^[chat.recipient],
+          o.data,
+          ^chat.recipient,
+          o.data,
+          ^[chat.user.ap_id]
+        )
+    )
+  end
+
   def creation_cng(struct, params) do
     struct
     |> cast(params, [:user_id, :recipient, :unread])
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 450d85332..1ef3477c8 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -14,9 +14,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Web.PleromaAPI.ChatMessageView
   alias Pleroma.Web.PleromaAPI.ChatView
 
-  import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
-
   import Ecto.Query
+  import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
 
   # TODO
   # - Error handling
@@ -65,24 +64,8 @@ def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
   def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       messages =
-        from(o in Object,
-          where: fragment("?->>'type' = ?", o.data, "ChatMessage"),
-          where:
-            fragment(
-              """
-              (?->>'actor' = ? and ?->'to' = ?) 
-              OR (?->>'actor' = ? and ?->'to' = ?) 
-              """,
-              o.data,
-              ^user.ap_id,
-              o.data,
-              ^[chat.recipient],
-              o.data,
-              ^chat.recipient,
-              o.data,
-              ^[user.ap_id]
-            )
-        )
+        chat
+        |> Chat.messages_for_chat_query()
         |> Pagination.fetch_paginated(params |> stringify_keys())
 
       conn
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index bc3af5ef5..21f0612ff 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -8,14 +8,19 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   alias Pleroma.Chat
   alias Pleroma.User
   alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
 
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
 
+    last_message = Chat.last_message_for_chat(chat)
+
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
-      unread: chat.unread
+      unread: chat.unread,
+      last_message:
+        last_message && ChatMessageView.render("show.json", chat: chat, object: last_message)
     }
   end
 
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 1ac3483d1..8568d98c6 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -6,8 +6,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
+  alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.PleromaAPI.ChatView
+  alias Pleroma.Web.PleromaAPI.ChatMessageView
 
   import Pleroma.Factory
 
@@ -22,7 +25,19 @@ test "it represents a chat" do
     assert represented_chat == %{
              id: "#{chat.id}",
              account: AccountView.render("show.json", user: recipient),
-             unread: 0
+             unread: 0,
+             last_message: nil
            }
+
+    {:ok, chat_message_creation} = CommonAPI.post_chat_message(user, recipient, "hello")
+
+    chat_message = Object.normalize(chat_message_creation, false)
+
+    {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
+
+    represented_chat = ChatView.render("show.json", chat: chat)
+
+    assert represented_chat[:last_message] ==
+             ChatMessageView.render("show.json", chat: chat, object: chat_message)
   end
 end

From 17be3ff669865102848df034045eb2889eed3976 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:01:20 +0200
Subject: [PATCH 084/283] Documentation: Add last_message to chat docs.

---
 docs/API/chats.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 3ddc13541..1f6175f77 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -62,7 +62,8 @@ Returned data:
     ...
   },
   "id" : "1",
-  "unread" : 2
+  "unread" : 2,
+  "last_message" : {...} // The last message in that chat
 }
 ```
 
@@ -105,7 +106,8 @@ Returned data:
         ...
       },
       "id" : "1",
-      "unread" : 2
+      "unread" : 2,
+      "last_message" : {...} // The last message in that chat
    }
 ]
 ```

From 172d9b11936bb029093eac430c58c89a81592c08 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:08:01 +0200
Subject: [PATCH 085/283] Chat: Add last_message to schema.

---
 lib/pleroma/web/api_spec/schemas/chat.ex | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index 4d385d6ab..8aaa4a792 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
   alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
 
   require OpenApiSpex
 
@@ -12,9 +13,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
     description: "Response schema for a Chat",
     type: :object,
     properties: %{
-      id: %Schema{type: :string, nullable: false},
-      account: %Schema{type: :object, nullable: false},
-      unread: %Schema{type: :integer, nullable: false}
+      id: %Schema{type: :string},
+      account: %Schema{type: :object},
+      unread: %Schema{type: :integer},
+      last_message: %Schema{type: ChatMessage, nullable: true}
     },
     example: %{
       "account" => %{
@@ -64,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
         "url" => "https://dontbulling.me/users/lain"
       },
       "id" => "1",
-      "unread" => 2
+      "unread" => 2,
+      "last_message" => ChatMessage.schema().example()
     }
   })
 end

From 8d5597ff68de22ee7b126730467649ada248aaf7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:26:14 +0200
Subject: [PATCH 086/283] ChatController: Add GET /chats/:id

---
 .../web/api_spec/operations/chat_operation.ex | 31 +++++++++++++++++++
 lib/pleroma/web/api_spec/schemas/chat.ex      |  2 +-
 .../web/api_spec/schemas/chat_message.ex      |  1 +
 .../controllers/chat_controller.ex            | 10 +++++-
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 17 ++++++++++
 6 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 16d3d5e22..fe6c2f52f 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -38,6 +38,37 @@ def mark_as_read_operation do
     }
   end
 
+  def show_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Create a chat",
+      operationId: "ChatController.show",
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "The id of the chat",
+          required: true,
+          example: "1234"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The existing chat",
+            "application/json",
+            Chat
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
+    }
+  end
+
   def create_operation do
     %Operation{
       tags: ["chat"],
diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index 8aaa4a792..c6ec07c88 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
       id: %Schema{type: :string},
       account: %Schema{type: :object},
       unread: %Schema{type: :integer},
-      last_message: %Schema{type: ChatMessage, nullable: true}
+      last_message: ChatMessage
     },
     example: %{
       "account" => %{
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex
index 89e062ddd..6e8f1a10a 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
   OpenApiSpex.schema(%{
     title: "ChatMessage",
     description: "Response schema for a ChatMessage",
+    nullable: true,
     type: :object,
     properties: %{
       id: %Schema{type: :string},
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 1ef3477c8..04f136dcd 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["read:statuses"]} when action in [:messages, :index]
+    %{scopes: ["read:statuses"]} when action in [:messages, :index, :show]
   )
 
   plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
@@ -100,4 +100,12 @@ def create(%{assigns: %{user: user}} = conn, params) do
       |> render("show.json", chat: chat)
     end
   end
+
+  def show(%{assigns: %{user: user}} = conn, params) do
+    with %Chat{} = chat <- Repo.get_by(Chat, user_id: user.id, id: params[:id]) do
+      conn
+      |> put_view(ChatView)
+      |> render("show.json", chat: chat)
+    end
+  end
 end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 4b264c43e..3b1834d97 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -307,6 +307,7 @@ defmodule Pleroma.Web.Router do
 
       post("/chats/by-account-id/:id", ChatController, :create)
       get("/chats", ChatController, :index)
+      get("/chats/:id", ChatController, :show)
       get("/chats/:id/messages", ChatController, :messages)
       post("/chats/:id/messages", ChatController, :post_chat_message)
       post("/chats/:id/read", ChatController, :mark_as_read)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index b4b73da90..dda4f9e5b 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -153,6 +153,23 @@ test "it creates or returns a chat", %{conn: conn} do
     end
   end
 
+  describe "GET /api/v1/pleroma/chats/:id" do
+    setup do: oauth_access(["read:statuses"])
+
+    test "it returns a chat", %{conn: conn, user: user} do
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats/#{chat.id}")
+        |> json_response_and_validate_schema(200)
+
+      assert result["id"] == to_string(chat.id)
+    end
+  end
+
   describe "GET /api/v1/pleroma/chats" do
     setup do: oauth_access(["read:statuses"])
 

From 8cc8d960af87cdc1e2398a8470155b0930f43f5c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:27:40 +0200
Subject: [PATCH 087/283] Documentation: Add GET /chats/:id

---
 docs/API/chats.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 1f6175f77..ed160abd9 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -43,12 +43,17 @@ you can call:
 
 `POST /api/v1/pleroma/chats/by-account-id/{account_id}`
 
-The account id is the normal FlakeId of the usre
-
+The account id is the normal FlakeId of the user
 ```
 POST /api/v1/pleroma/chats/by-account-id/someflakeid
 ```
 
+If you already have the id of a chat, you can also use
+
+```
+GET /api/v1/pleroma/chats/:id
+```
+
 There will only ever be ONE Chat for you and a given recipient, so this call
 will return the same Chat if you already have one with that user.
 

From 1b1dfb54eb092921fe9dab2c49928e5b04fa049b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 10 May 2020 13:28:05 +0200
Subject: [PATCH 088/283] Credo fixes.

---
 test/web/pleroma_api/views/chat_view_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 8568d98c6..e24e29835 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -9,8 +9,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatView
 
   import Pleroma.Factory
 

From fdb98715b8e6ced7c4037b1292fb10980a994803 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 11 May 2020 10:58:14 +0200
Subject: [PATCH 089/283] Chat: Fix wrong query.

---
 lib/pleroma/chat.ex |  1 +
 test/chat_test.exs  | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 6a03ee3c1..4c92a58c7 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -29,6 +29,7 @@ defmodule Pleroma.Chat do
   def last_message_for_chat(chat) do
     messages_for_chat_query(chat)
     |> order_by(desc: :id)
+    |> limit(1)
     |> Repo.one()
   end
 
diff --git a/test/chat_test.exs b/test/chat_test.exs
index 943e48111..dfcb6422e 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -6,9 +6,26 @@ defmodule Pleroma.ChatTest do
   use Pleroma.DataCase, async: true
 
   alias Pleroma.Chat
+  alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
+  describe "messages" do
+    test "it returns the last message in a chat" do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, _message_1} = CommonAPI.post_chat_message(user, recipient, "hey")
+      {:ok, _message_2} = CommonAPI.post_chat_message(recipient, user, "ho")
+
+      {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
+
+      message = Chat.last_message_for_chat(chat)
+
+      assert message.data["content"] == "ho"
+    end
+  end
+
   describe "creation and getting" do
     test "it only works if the recipient is a valid user (for now)" do
       user = insert(:user)

From b5aa204eb8bf3f737d3d807a9924c0153d1b6d3e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 13:13:03 +0200
Subject: [PATCH 090/283] ChatController: Support deletion of chat messages.

---
 .../object_validators/delete_validator.ex     |  3 ++-
 .../web/api_spec/operations/chat_operation.ex | 25 +++++++++++++++++++
 .../controllers/chat_controller.ex            | 24 +++++++++++++++++-
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 24 ++++++++++++++++++
 5 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
index f42c03510..e5d08eb5c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
@@ -46,12 +46,13 @@ def add_deleted_activity_id(cng) do
     Answer
     Article
     Audio
+    ChatMessage
     Event
     Note
     Page
     Question
-    Video
     Tombstone
+    Video
   }
   def validate_data(cng) do
     cng
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index fe6c2f52f..8ba10c603 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -166,6 +166,31 @@ def post_chat_message_operation do
     }
   end
 
+  def delete_message_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "delete_message",
+      operationId: "ChatController.delete_message",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:message_id, :path, :string, "The ID of the message")
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The deleted ChatMessage",
+            "application/json",
+            ChatMessage
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write"]
+        }
+      ]
+    }
+  end
+
   def chats_response do
     %Schema{
       title: "ChatsResponse",
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 04f136dcd..8eed88752 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -4,6 +4,7 @@
 defmodule Pleroma.Web.PleromaAPI.ChatController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Activity
   alias Pleroma.Chat
   alias Pleroma.Object
   alias Pleroma.Pagination
@@ -22,7 +23,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["write:statuses"]} when action in [:post_chat_message, :create, :mark_as_read]
+    %{scopes: ["write:statuses"]}
+    when action in [:post_chat_message, :create, :mark_as_read, :delete_message]
   )
 
   plug(
@@ -34,6 +36,26 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
+  def delete_message(%{assigns: %{user: %{ap_id: actor} = user}} = conn, %{
+        message_id: id
+      }) do
+    with %Object{
+           data: %{
+             "actor" => ^actor,
+             "id" => object,
+             "to" => [recipient],
+             "type" => "ChatMessage"
+           }
+         } = message <- Object.get_by_id(id),
+         %Chat{} = chat <- Chat.get(user.id, recipient),
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object),
+         {:ok, _delete} <- CommonAPI.delete(activity.id, user) do
+      conn
+      |> put_view(ChatMessageView)
+      |> render("show.json", for: user, object: message, chat: chat)
+    end
+  end
+
   def post_chat_message(
         %{body_params: %{content: content} = params, assigns: %{user: %{id: user_id} = user}} =
           conn,
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 3b1834d97..0e4f45869 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -310,6 +310,7 @@ defmodule Pleroma.Web.Router do
       get("/chats/:id", ChatController, :show)
       get("/chats/:id/messages", ChatController, :messages)
       post("/chats/:id/messages", ChatController, :post_chat_message)
+      delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
       post("/chats/:id/read", ChatController, :mark_as_read)
     end
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index dda4f9e5b..86ccbb117 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -4,6 +4,7 @@
 defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
+  alias Pleroma.Object
   alias Pleroma.Chat
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
@@ -78,6 +79,29 @@ test "it works with an attachment", %{conn: conn, user: user} do
     end
   end
 
+  describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do
+    setup do: oauth_access(["write:statuses"])
+
+    test "it deletes a message for the author of the message", %{conn: conn, user: user} do
+      recipient = insert(:user)
+
+      {:ok, message} =
+        CommonAPI.post_chat_message(user, recipient, "Hello darkness my old friend")
+
+      object = Object.normalize(message, false)
+
+      chat = Chat.get(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
+        |> json_response_and_validate_schema(200)
+
+      assert result["id"] == to_string(object.id)
+    end
+  end
+
   describe "GET /api/v1/pleroma/chats/:id/messages" do
     setup do: oauth_access(["read:statuses"])
 

From ec72cba43ec4f45faadf1b06a6d014cd4136707e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 13:23:09 +0200
Subject: [PATCH 091/283] Chat Controller: Add basic error handling.

---
 .../web/pleroma_api/controllers/chat_controller.ex   |  5 +++--
 .../pleroma_api/controllers/chat_controller_test.exs | 12 ++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 8eed88752..4ce3e7419 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -18,8 +18,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   import Ecto.Query
   import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
 
-  # TODO
-  # - Error handling
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
 
   plug(
     OAuthScopesPlug,
@@ -53,6 +52,8 @@ def delete_message(%{assigns: %{user: %{ap_id: actor} = user}} = conn, %{
       conn
       |> put_view(ChatMessageView)
       |> render("show.json", for: user, object: message, chat: chat)
+    else
+      _e -> {:error, :could_not_delete}
     end
   end
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 86ccbb117..75d4903ed 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -88,6 +88,8 @@ test "it deletes a message for the author of the message", %{conn: conn, user: u
       {:ok, message} =
         CommonAPI.post_chat_message(user, recipient, "Hello darkness my old friend")
 
+      {:ok, other_message} = CommonAPI.post_chat_message(recipient, user, "nico nico ni")
+
       object = Object.normalize(message, false)
 
       chat = Chat.get(user.id, recipient.ap_id)
@@ -99,6 +101,16 @@ test "it deletes a message for the author of the message", %{conn: conn, user: u
         |> json_response_and_validate_schema(200)
 
       assert result["id"] == to_string(object.id)
+
+      object = Object.normalize(other_message, false)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
+        |> json_response(400)
+
+      assert result == %{"error" => "could_not_delete"}
     end
   end
 

From a61120f497e5d17be1207eacd11525edb7b6db36 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 13:25:25 +0200
Subject: [PATCH 092/283] Documention: Add chat message deletion docs

---
 docs/API/chats.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index ed160abd9..ad36961ae 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -192,6 +192,14 @@ Returned data:
 }
 ```
 
+### Deleting a chat message
+
+Deleting a chat message for given Chat id works like this:
+
+`DELETE /api/v1/pleroma/chats/{chat_id}/messages/{message_id}`
+
+Returned data is the deleted message.
+
 ### Notifications
 
 There's a new `pleroma:chat_mention` notification, which has this form:

From e44166b510f95bfb2e679b2d64bbf7e0facd0dd2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 14:44:11 +0200
Subject: [PATCH 093/283] Credo fixes.

---
 test/web/pleroma_api/controllers/chat_controller_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 75d4903ed..861ef10b0 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -4,8 +4,8 @@
 defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
-  alias Pleroma.Object
   alias Pleroma.Chat
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
 

From c0ea5c60e4e709d3d4415de42a65f878b55dc3bb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 16:43:04 +0200
Subject: [PATCH 094/283] ChatController: Don't return chats for user you've
 blocked.

---
 .../controllers/chat_controller.ex            |  5 +++-
 .../controllers/chat_controller_test.exs      | 23 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 4ce3e7419..496cb8e87 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -102,10 +102,13 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = para
     end
   end
 
-  def index(%{assigns: %{user: %{id: user_id}}} = conn, params) do
+  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
+    blocked_ap_ids = User.blocked_users_ap_ids(user)
+
     chats =
       from(c in Chat,
         where: c.user_id == ^user_id,
+        where: c.recipient not in ^blocked_ap_ids,
         order_by: [desc: c.updated_at]
       )
       |> Pagination.fetch_paginated(params |> stringify_keys)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 861ef10b0..037dd2297 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   alias Pleroma.Chat
   alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
 
@@ -209,6 +210,28 @@ test "it returns a chat", %{conn: conn, user: user} do
   describe "GET /api/v1/pleroma/chats" do
     setup do: oauth_access(["read:statuses"])
 
+    test "it does not return chats with users you blocked", %{conn: conn, user: user} do
+      recipient = insert(:user)
+
+      {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 1
+
+      User.block(user, recipient)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 0
+    end
+
     test "it paginates", %{conn: conn, user: user} do
       Enum.each(1..30, fn _ ->
         recipient = insert(:user)

From 06cad239e50cada3aec4fc3b4c494a70d328672c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 13 May 2020 14:05:22 +0200
Subject: [PATCH 095/283] InstanceView: Add pleroma chat messages to nodeinfo

---
 lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++-
 test/web/node_info_test.exs                         | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index a329ffc28..17cfc4fcf 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -68,7 +68,8 @@ def features do
       if Config.get([:instance, :safe_dm_mentions]) do
         "safe_dm_mentions"
       end,
-      "pleroma_emoji_reactions"
+      "pleroma_emoji_reactions",
+      "pleroma_chat_messages"
     ]
     |> Enum.filter(& &1)
   end
diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs
index 9bcc07b37..00925caad 100644
--- a/test/web/node_info_test.exs
+++ b/test/web/node_info_test.exs
@@ -145,7 +145,8 @@ test "it shows default features flags", %{conn: conn} do
       "shareable_emoji_packs",
       "multifetch",
       "pleroma_emoji_reactions",
-      "pleroma:api/v1/notifications:include_types_filter"
+      "pleroma:api/v1/notifications:include_types_filter",
+      "pleroma_chat_messages"
     ]
 
     assert MapSet.subset?(

From 0f0acc740d30c47d093f27875d4decf0693b2845 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 13 May 2020 15:31:28 +0200
Subject: [PATCH 096/283] Chat: Allow posting without content if an attachment
 is present.

---
 docs/API/chats.md                             |  5 ++-
 .../chat_message_validator.ex                 | 14 +++++++-
 .../web/api_spec/operations/chat_operation.ex | 12 ++++---
 .../web/api_spec/schemas/chat_message.ex      |  2 +-
 lib/pleroma/web/common_api/common_api.ex      | 17 +++++++---
 .../controllers/chat_controller.ex            |  7 ++--
 .../activity_pub/object_validator_test.exs    | 32 +++++++++++++++++++
 test/web/common_api/common_api_test.exs       | 23 +++++++++++++
 .../controllers/chat_controller_test.exs      | 18 +++++++++--
 9 files changed, 111 insertions(+), 19 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index ad36961ae..1ea18ff5f 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -166,11 +166,10 @@ Posting a chat message for given Chat id works like this:
 `POST /api/v1/pleroma/chats/{id}/messages`
 
 Parameters:
-- content: The text content of the message
+- content: The text content of the message. Optional if media is attached.
 - media_id: The id of an upload that will be attached to the message.
 
-Currently, no formatting beyond basic escaping and emoji is implemented, as well as no
-attachments. This will most probably change.
+Currently, no formatting beyond basic escaping and emoji is implemented.
 
 Returned data:
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index e40c80ab4..9c20c188a 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -61,12 +61,24 @@ def changeset(struct, data) do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["ChatMessage"])
-    |> validate_required([:id, :actor, :to, :type, :content, :published])
+    |> validate_required([:id, :actor, :to, :type, :published])
+    |> validate_content_or_attachment()
     |> validate_length(:to, is: 1)
     |> validate_length(:content, max: Pleroma.Config.get([:instance, :remote_limit]))
     |> validate_local_concern()
   end
 
+  def validate_content_or_attachment(cng) do
+    attachment = get_field(cng, :attachment)
+
+    if attachment do
+      cng
+    else
+      cng
+      |> validate_required([:content])
+    end
+  end
+
   @doc """
   Validates the following
   - If both users are in our system
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 8ba10c603..a1c5db5dc 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.Chat
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
 
@@ -149,14 +150,15 @@ def post_chat_message_operation do
       parameters: [
         Operation.parameter(:id, :path, :string, "The ID of the Chat")
       ],
-      requestBody: request_body("Parameters", chat_message_create(), required: true),
+      requestBody: request_body("Parameters", chat_message_create()),
       responses: %{
         200 =>
           Operation.response(
             "The newly created ChatMessage",
             "application/json",
             ChatMessage
-          )
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
       },
       security: [
         %{
@@ -292,10 +294,12 @@ def chat_message_create do
       description: "POST body for creating an chat message",
       type: :object,
       properties: %{
-        content: %Schema{type: :string, description: "The content of your message"},
+        content: %Schema{
+          type: :string,
+          description: "The content of your message. Optional if media_id is present"
+        },
         media_id: %Schema{type: :string, description: "The id of an upload"}
       },
-      required: [:content],
       example: %{
         "content" => "Hey wanna buy feet pics?",
         "media_id" => "134234"
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex
index 6e8f1a10a..3ee85aa76 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
       id: %Schema{type: :string},
       account_id: %Schema{type: :string, description: "The Mastodon API id of the actor"},
       chat_id: %Schema{type: :string},
-      content: %Schema{type: :string},
+      content: %Schema{type: :string, nullable: true},
       created_at: %Schema{type: :string, format: :"date-time"},
       emojis: %Schema{type: :array},
       attachment: %Schema{type: :object, nullable: true}
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 664175a4f..7008cea44 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -26,14 +26,14 @@ defmodule Pleroma.Web.CommonAPI do
   require Logger
 
   def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do
-    with :ok <- validate_chat_content_length(content),
-         maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]),
+    with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]),
+         :ok <- validate_chat_content_length(content, !!maybe_attachment),
          {_, {:ok, chat_message_data, _meta}} <-
            {:build_object,
             Builder.chat_message(
               user,
               recipient.ap_id,
-              content |> Formatter.html_escape("text/plain"),
+              content |> format_chat_content,
               attachment: maybe_attachment
             )},
          {_, {:ok, create_activity_data, _meta}} <-
@@ -47,7 +47,16 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
     end
   end
 
-  defp validate_chat_content_length(content) do
+  defp format_chat_content(nil), do: nil
+
+  defp format_chat_content(content) do
+    content |> Formatter.html_escape("text/plain")
+  end
+
+  defp validate_chat_content_length(_, true), do: :ok
+  defp validate_chat_content_length(nil, false), do: {:error, :no_content}
+
+  defp validate_chat_content_length(content, _) do
     if String.length(content) <= Pleroma.Config.get([:instance, :chat_limit]) do
       :ok
     else
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 496cb8e87..210c8ec4a 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -58,8 +58,7 @@ def delete_message(%{assigns: %{user: %{ap_id: actor} = user}} = conn, %{
   end
 
   def post_chat_message(
-        %{body_params: %{content: content} = params, assigns: %{user: %{id: user_id} = user}} =
-          conn,
+        %{body_params: params, assigns: %{user: %{id: user_id} = user}} = conn,
         %{
           id: id
         }
@@ -67,7 +66,9 @@ def post_chat_message(
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
          {:ok, activity} <-
-           CommonAPI.post_chat_message(user, recipient, content, media_id: params[:media_id]),
+           CommonAPI.post_chat_message(user, recipient, params[:content],
+             media_id: params[:media_id]
+           ),
          message <- Object.normalize(activity) do
       conn
       |> put_view(ChatMessageView)
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index d9f5a8fac..da33d3dbc 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -103,6 +103,38 @@ test "validates for a basic object with an attachment", %{
       assert object["attachment"]
     end
 
+    test "validates for a basic object with an attachment but without content", %{
+      valid_chat_message: valid_chat_message,
+      user: user
+    } do
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, attachment} = ActivityPub.upload(file, actor: user.ap_id)
+
+      valid_chat_message =
+        valid_chat_message
+        |> Map.put("attachment", attachment.data)
+        |> Map.delete("content")
+
+      assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+
+      assert object["attachment"]
+    end
+
+    test "does not validate if the message has no content", %{
+      valid_chat_message: valid_chat_message
+    } do
+      contentless =
+        valid_chat_message
+        |> Map.delete("content")
+
+      refute match?({:ok, _object, _meta}, ObjectValidator.validate(contentless, []))
+    end
+
     test "does not validate if the message is longer than the remote_limit", %{
       valid_chat_message: valid_chat_message
     } do
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index fd2c486a1..46ffd2888 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -27,6 +27,29 @@ defmodule Pleroma.Web.CommonAPITest do
   describe "posting chat messages" do
     setup do: clear_config([:instance, :chat_limit])
 
+    test "it posts a chat message without content but with an attachment" do
+      author = insert(:user)
+      recipient = insert(:user)
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, upload} = ActivityPub.upload(file, actor: author.ap_id)
+
+      {:ok, activity} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          nil,
+          media_id: upload.id
+        )
+
+      assert activity
+    end
+
     test "it posts a chat message" do
       author = insert(:user)
       recipient = insert(:user)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 037dd2297..d79aa3148 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -53,6 +53,20 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
       assert result["chat_id"] == chat.id |> to_string()
     end
 
+    test "it fails if there is no content", %{conn: conn, user: user} do
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages")
+        |> json_response_and_validate_schema(400)
+
+      assert result
+    end
+
     test "it works with an attachment", %{conn: conn, user: user} do
       file = %Plug.Upload{
         content_type: "image/jpg",
@@ -70,13 +84,11 @@ test "it works with an attachment", %{conn: conn, user: user} do
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{
-          "content" => "Hallo!!",
           "media_id" => to_string(upload.id)
         })
         |> json_response_and_validate_schema(200)
 
-      assert result["content"] == "Hallo!!"
-      assert result["chat_id"] == chat.id |> to_string()
+      assert result["attachment"]
     end
   end
 

From 3342846ac2bbd48e985cfeff26ba4593f4815879 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 14 May 2020 13:20:28 +0200
Subject: [PATCH 097/283] ChatView: Add update_at field.

---
 lib/pleroma/web/pleroma_api/views/chat_view.ex | 4 +++-
 test/web/pleroma_api/views/chat_view_test.exs  | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 21f0612ff..08d5110c3 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
 
   alias Pleroma.Chat
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
 
@@ -20,7 +21,8 @@ def render("show.json", %{chat: %Chat{} = chat} = opts) do
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
       unread: chat.unread,
       last_message:
-        last_message && ChatMessageView.render("show.json", chat: chat, object: last_message)
+        last_message && ChatMessageView.render("show.json", chat: chat, object: last_message),
+      updated_at: Utils.to_masto_date(chat.updated_at)
     }
   end
 
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index e24e29835..6062a0cfe 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   alias Pleroma.Chat
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
   alias Pleroma.Web.PleromaAPI.ChatView
@@ -26,7 +27,8 @@ test "it represents a chat" do
              id: "#{chat.id}",
              account: AccountView.render("show.json", user: recipient),
              unread: 0,
-             last_message: nil
+             last_message: nil,
+             updated_at: Utils.to_masto_date(chat.updated_at)
            }
 
     {:ok, chat_message_creation} = CommonAPI.post_chat_message(user, recipient, "hello")

From 1d18721a3c60aa0acc7d1ba858a92277e544a54a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 15 May 2020 13:18:41 +0200
Subject: [PATCH 098/283] Chats: Add updated_at to Schema and docs.

---
 docs/API/chats.md                        | 9 ++++++---
 lib/pleroma/web/api_spec/schemas/chat.ex | 6 ++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 1ea18ff5f..2e415e4da 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -68,7 +68,8 @@ Returned data:
   },
   "id" : "1",
   "unread" : 2,
-  "last_message" : {...} // The last message in that chat
+  "last_message" : {...}, // The last message in that chat
+  "updated_at": "2020-04-21T15:11:46.000Z"
 }
 ```
 
@@ -88,7 +89,8 @@ Returned data:
     ...
   },
   "id" : "1",
-  "unread" : 0
+  "unread" : 0,
+  "updated_at": "2020-04-21T15:11:46.000Z"
 }
 ```
 
@@ -112,7 +114,8 @@ Returned data:
       },
       "id" : "1",
       "unread" : 2,
-      "last_message" : {...} // The last message in that chat
+      "last_message" : {...}, // The last message in that chat
+      "updated_at": "2020-04-21T15:11:46.000Z"
    }
 ]
 ```
diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex
index c6ec07c88..b4986b734 100644
--- a/lib/pleroma/web/api_spec/schemas/chat.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat.ex
@@ -16,7 +16,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
       id: %Schema{type: :string},
       account: %Schema{type: :object},
       unread: %Schema{type: :integer},
-      last_message: ChatMessage
+      last_message: ChatMessage,
+      updated_at: %Schema{type: :string, format: :"date-time"}
     },
     example: %{
       "account" => %{
@@ -67,7 +68,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do
       },
       "id" => "1",
       "unread" => 2,
-      "last_message" => ChatMessage.schema().example()
+      "last_message" => ChatMessage.schema().example(),
+      "updated_at" => "2020-04-21T15:06:45.000Z"
     }
   })
 end

From baf051a59e8bfcb2e55b5e28e46e80d6961b9bb4 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 17 May 2020 12:22:26 +0200
Subject: [PATCH 099/283] SideEffects: Don't update unread count for actor in
 chatmessages.

---
 lib/pleroma/web/activity_pub/side_effects.ex |  6 +++++-
 test/web/activity_pub/side_effects_test.exs  | 21 ++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index c8b675d54..8e64b4615 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -117,7 +117,11 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       [[actor, recipient], [recipient, actor]]
       |> Enum.each(fn [user, other_user] ->
         if user.local do
-          Chat.bump_or_create(user.id, other_user.ap_id)
+          if user.ap_id == actor.ap_id do
+            Chat.get_or_create(user.id, other_user.ap_id)
+          else
+            Chat.bump_or_create(user.id, other_user.ap_id)
+          end
         end
       end)
 
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 148fa4442..37d7491ca 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -284,6 +284,27 @@ test "notifies the recipient" do
       assert Repo.get_by(Notification, user_id: recipient.id, activity_id: create_activity.id)
     end
 
+    test "it creates a Chat for the local users and bumps the unread count, except for the author" do
+      author = insert(:user, local: true)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      {:ok, _create_activity, _meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
+
+      chat = Chat.get(author.id, recipient.ap_id)
+      assert chat.unread == 0
+
+      chat = Chat.get(recipient.id, author.ap_id)
+      assert chat.unread == 1
+    end
+
     test "it creates a Chat for the local users and bumps the unread count" do
       author = insert(:user, local: false)
       recipient = insert(:user, local: true)

From e7bc2f980cce170731960e024614c497b821fe90 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 7 May 2020 13:44:38 +0300
Subject: [PATCH 100/283] account visibility

---
 lib/pleroma/user.ex                           | 52 ++++++++++------
 .../api_spec/operations/account_operation.ex  |  8 ++-
 .../controllers/account_controller.ex         | 21 +++++--
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 test/user_test.exs                            | 10 ++--
 .../controllers/account_controller_test.exs   | 59 ++++++++++++++-----
 6 files changed, 105 insertions(+), 47 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index cba391072..7a2558c29 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -262,37 +262,51 @@ def account_status(%User{deactivated: true}), do: :deactivated
   def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
 
   def account_status(%User{confirmation_pending: true}) do
-    case Config.get([:instance, :account_activation_required]) do
-      true -> :confirmation_pending
-      _ -> :active
+    if Config.get([:instance, :account_activation_required]) do
+      :confirmation_pending
+    else
+      :active
     end
   end
 
   def account_status(%User{}), do: :active
 
-  @spec visible_for?(User.t(), User.t() | nil) :: boolean()
-  def visible_for?(user, for_user \\ nil)
+  @spec visible_for(User.t(), User.t() | nil) ::
+          boolean()
+          | :invisible
+          | :restricted_unauthenticated
+          | :deactivated
+          | :confirmation_pending
+  def visible_for(user, for_user \\ nil)
 
-  def visible_for?(%User{invisible: true}, _), do: false
+  def visible_for(%User{invisible: true}, _), do: :invisible
 
-  def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true
+  def visible_for(%User{id: user_id}, %User{id: user_id}), do: true
 
-  def visible_for?(%User{local: local} = user, nil) do
-    cfg_key =
-      if local,
-        do: :local,
-        else: :remote
-
-    if Config.get([:restrict_unauthenticated, :profiles, cfg_key]),
-      do: false,
-      else: account_status(user) == :active
+  def visible_for(%User{} = user, nil) do
+    if restrict_unauthenticated?(user) do
+      :restrict_unauthenticated
+    else
+      visible_account_status(user)
+    end
   end
 
-  def visible_for?(%User{} = user, for_user) do
-    account_status(user) == :active || superuser?(for_user)
+  def visible_for(%User{} = user, for_user) do
+    superuser?(for_user) || visible_account_status(user)
   end
 
-  def visible_for?(_, _), do: false
+  def visible_for(_, _), do: false
+
+  defp restrict_unauthenticated?(%User{local: local}) do
+    config_key = if local, do: :local, else: :remote
+
+    Config.get([:restrict_unauthenticated, :profiles, config_key], false)
+  end
+
+  defp visible_account_status(user) do
+    status = account_status(user)
+    status in [:active, :password_reset_pending] || status
+  end
 
   @spec superuser?(User.t()) :: boolean()
   def superuser?(%User{local: true, is_admin: true}), do: true
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 934f6038e..43168acf7 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -102,7 +102,9 @@ def show_operation do
       parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}],
       responses: %{
         200 => Operation.response("Account", "application/json", Account),
-        404 => Operation.response("Error", "application/json", ApiError)
+        401 => Operation.response("Error", "application/json", ApiError),
+        404 => Operation.response("Error", "application/json", ApiError),
+        410 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
@@ -142,7 +144,9 @@ def statuses_operation do
         ] ++ pagination_params(),
       responses: %{
         200 => Operation.response("Statuses", "application/json", array_of_statuses()),
-        404 => Operation.response("Error", "application/json", ApiError)
+        401 => Operation.response("Error", "application/json", ApiError),
+        404 => Operation.response("Error", "application/json", ApiError),
+        410 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index ef41f9e96..ffa82731f 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -221,17 +221,17 @@ def relationships(%{assigns: %{user: _user}} = conn, _), do: json(conn, [])
   @doc "GET /api/v1/accounts/:id"
   def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do
     with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user),
-         true <- User.visible_for?(user, for_user) do
+         true <- User.visible_for(user, for_user) do
       render(conn, "show.json", user: user, for: for_user)
     else
-      _e -> render_error(conn, :not_found, "Can't find user")
+      error -> user_visibility_error(conn, error)
     end
   end
 
   @doc "GET /api/v1/accounts/:id/statuses"
   def statuses(%{assigns: %{user: reading_user}} = conn, params) do
     with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user),
-         true <- User.visible_for?(user, reading_user) do
+         true <- User.visible_for(user, reading_user) do
       params =
         params
         |> Map.delete(:tagged)
@@ -250,7 +250,20 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do
         as: :activity
       )
     else
-      _e -> render_error(conn, :not_found, "Can't find user")
+      error -> user_visibility_error(conn, error)
+    end
+  end
+
+  defp user_visibility_error(conn, error) do
+    case error do
+      :deactivated ->
+        render_error(conn, :gone, "")
+
+      :restrict_unauthenticated ->
+        render_error(conn, :unauthorized, "This API requires an authenticated user")
+
+      _ ->
+        render_error(conn, :not_found, "Can't find user")
     end
   end
 
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 45fffaad2..8e723d013 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -35,7 +35,7 @@ def render("index.json", %{users: users} = opts) do
   end
 
   def render("show.json", %{user: user} = opts) do
-    if User.visible_for?(user, opts[:for]) do
+    if User.visible_for(user, opts[:for]) == true do
       do_render("show.json", opts)
     else
       %{}
diff --git a/test/user_test.exs b/test/user_test.exs
index 6b9df60a4..3bfcfd10c 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1289,11 +1289,11 @@ test "returns false for a non-invisible user" do
     end
   end
 
-  describe "visible_for?/2" do
+  describe "visible_for/2" do
     test "returns true when the account is itself" do
       user = insert(:user, local: true)
 
-      assert User.visible_for?(user, user)
+      assert User.visible_for(user, user)
     end
 
     test "returns false when the account is unauthenticated and auth is required" do
@@ -1302,14 +1302,14 @@ test "returns false when the account is unauthenticated and auth is required" do
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true)
 
-      refute User.visible_for?(user, other_user)
+      refute User.visible_for(user, other_user) == true
     end
 
     test "returns true when the account is unauthenticated and auth is not required" do
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true)
 
-      assert User.visible_for?(user, other_user)
+      assert User.visible_for(user, other_user)
     end
 
     test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do
@@ -1318,7 +1318,7 @@ test "returns true when the account is unauthenticated and being viewed by a pri
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true, is_admin: true)
 
-      assert User.visible_for?(user, other_user)
+      assert User.visible_for(user, other_user)
     end
   end
 
diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index 280bd6aca..7dfea2f9e 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -127,6 +127,15 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do
                |> get("/api/v1/accounts/internal.fetch")
                |> json_response_and_validate_schema(404)
     end
+
+    test "returns 401 for deactivated user", %{conn: conn} do
+      user = insert(:user, deactivated: true)
+
+      assert %{} =
+               conn
+               |> get("/api/v1/accounts/#{user.id}")
+               |> json_response_and_validate_schema(:gone)
+    end
   end
 
   defp local_and_remote_users do
@@ -143,15 +152,15 @@ defp local_and_remote_users do
     setup do: clear_config([:restrict_unauthenticated, :profiles, :remote], true)
 
     test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{local.id}")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
 
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{remote.id}")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
     end
 
     test "if user is authenticated", %{local: local, remote: remote} do
@@ -173,8 +182,8 @@ test "if user is authenticated", %{local: local, remote: remote} do
     test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do
       res_conn = get(conn, "/api/v1/accounts/#{local.id}")
 
-      assert json_response_and_validate_schema(res_conn, :not_found) == %{
-               "error" => "Can't find user"
+      assert json_response_and_validate_schema(res_conn, :unauthorized) == %{
+               "error" => "This API requires an authenticated user"
              }
 
       res_conn = get(conn, "/api/v1/accounts/#{remote.id}")
@@ -203,8 +212,8 @@ test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} d
 
       res_conn = get(conn, "/api/v1/accounts/#{remote.id}")
 
-      assert json_response_and_validate_schema(res_conn, :not_found) == %{
-               "error" => "Can't find user"
+      assert json_response_and_validate_schema(res_conn, :unauthorized) == %{
+               "error" => "This API requires an authenticated user"
              }
     end
 
@@ -249,6 +258,24 @@ test "works with announces that are just addressed to public", %{conn: conn} do
       assert id == announce.id
     end
 
+    test "deactivated user", %{conn: conn} do
+      user = insert(:user, deactivated: true)
+
+      assert %{} ==
+               conn
+               |> get("/api/v1/accounts/#{user.id}/statuses")
+               |> json_response_and_validate_schema(:gone)
+    end
+
+    test "returns 404 when user is invisible", %{conn: conn} do
+      user = insert(:user, %{invisible: true})
+
+      assert %{"error" => "Can't find user"} =
+               conn
+               |> get("/api/v1/accounts/#{user.id}")
+               |> json_response_and_validate_schema(404)
+    end
+
     test "respects blocks", %{user: user_one, conn: conn} do
       user_two = insert(:user)
       user_three = insert(:user)
@@ -422,15 +449,15 @@ defp local_and_remote_activities(%{local: local, remote: remote}) do
     setup do: clear_config([:restrict_unauthenticated, :profiles, :remote], true)
 
     test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{local.id}/statuses")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
 
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{remote.id}/statuses")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
     end
 
     test "if user is authenticated", %{local: local, remote: remote} do
@@ -451,10 +478,10 @@ test "if user is authenticated", %{local: local, remote: remote} do
     setup do: clear_config([:restrict_unauthenticated, :profiles, :local], true)
 
     test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} do
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{local.id}/statuses")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
 
       res_conn = get(conn, "/api/v1/accounts/#{remote.id}/statuses")
       assert length(json_response_and_validate_schema(res_conn, 200)) == 1
@@ -481,10 +508,10 @@ test "if user is unauthenticated", %{conn: conn, local: local, remote: remote} d
       res_conn = get(conn, "/api/v1/accounts/#{local.id}/statuses")
       assert length(json_response_and_validate_schema(res_conn, 200)) == 1
 
-      assert %{"error" => "Can't find user"} ==
+      assert %{"error" => "This API requires an authenticated user"} ==
                conn
                |> get("/api/v1/accounts/#{remote.id}/statuses")
-               |> json_response_and_validate_schema(:not_found)
+               |> json_response_and_validate_schema(:unauthorized)
     end
 
     test "if user is authenticated", %{local: local, remote: remote} do

From b1aa402229b6422a5ab1aa7102c7a104e218d0e3 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 13 May 2020 11:11:10 +0300
Subject: [PATCH 101/283] removing 410 status

---
 lib/pleroma/web/api_spec/operations/account_operation.ex  | 6 ++----
 .../web/mastodon_api/controllers/account_controller.ex    | 3 ---
 .../mastodon_api/controllers/account_controller_test.exs  | 8 ++++----
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 43168acf7..74b395dfe 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -103,8 +103,7 @@ def show_operation do
       responses: %{
         200 => Operation.response("Account", "application/json", Account),
         401 => Operation.response("Error", "application/json", ApiError),
-        404 => Operation.response("Error", "application/json", ApiError),
-        410 => Operation.response("Error", "application/json", ApiError)
+        404 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
@@ -145,8 +144,7 @@ def statuses_operation do
       responses: %{
         200 => Operation.response("Statuses", "application/json", array_of_statuses()),
         401 => Operation.response("Error", "application/json", ApiError),
-        404 => Operation.response("Error", "application/json", ApiError),
-        410 => Operation.response("Error", "application/json", ApiError)
+        404 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index ffa82731f..1edc0d96a 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -256,9 +256,6 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do
 
   defp user_visibility_error(conn, error) do
     case error do
-      :deactivated ->
-        render_error(conn, :gone, "")
-
       :restrict_unauthenticated ->
         render_error(conn, :unauthorized, "This API requires an authenticated user")
 
diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index 7dfea2f9e..8700ab2f5 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -131,10 +131,10 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do
     test "returns 401 for deactivated user", %{conn: conn} do
       user = insert(:user, deactivated: true)
 
-      assert %{} =
+      assert %{"error" => "Can't find user"} =
                conn
                |> get("/api/v1/accounts/#{user.id}")
-               |> json_response_and_validate_schema(:gone)
+               |> json_response_and_validate_schema(:not_found)
     end
   end
 
@@ -261,10 +261,10 @@ test "works with announces that are just addressed to public", %{conn: conn} do
     test "deactivated user", %{conn: conn} do
       user = insert(:user, deactivated: true)
 
-      assert %{} ==
+      assert %{"error" => "Can't find user"} ==
                conn
                |> get("/api/v1/accounts/#{user.id}/statuses")
-               |> json_response_and_validate_schema(:gone)
+               |> json_response_and_validate_schema(:not_found)
     end
 
     test "returns 404 when user is invisible", %{conn: conn} do

From 1671864d886bf63d11bbf3d7303719e8744bfc32 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 15 May 2020 20:29:09 +0300
Subject: [PATCH 102/283] return :visible instead of boolean

---
 lib/pleroma/user.ex                           | 19 ++++++++++++++-----
 .../controllers/account_controller.ex         |  4 ++--
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 test/user_test.exs                            |  8 ++++----
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 7a2558c29..5052f7b97 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -272,7 +272,7 @@ def account_status(%User{confirmation_pending: true}) do
   def account_status(%User{}), do: :active
 
   @spec visible_for(User.t(), User.t() | nil) ::
-          boolean()
+          :visible
           | :invisible
           | :restricted_unauthenticated
           | :deactivated
@@ -281,7 +281,7 @@ def visible_for(user, for_user \\ nil)
 
   def visible_for(%User{invisible: true}, _), do: :invisible
 
-  def visible_for(%User{id: user_id}, %User{id: user_id}), do: true
+  def visible_for(%User{id: user_id}, %User{id: user_id}), do: :visible
 
   def visible_for(%User{} = user, nil) do
     if restrict_unauthenticated?(user) do
@@ -292,10 +292,14 @@ def visible_for(%User{} = user, nil) do
   end
 
   def visible_for(%User{} = user, for_user) do
-    superuser?(for_user) || visible_account_status(user)
+    if superuser?(for_user) do
+      :visible
+    else
+      visible_account_status(user)
+    end
   end
 
-  def visible_for(_, _), do: false
+  def visible_for(_, _), do: :invisible
 
   defp restrict_unauthenticated?(%User{local: local}) do
     config_key = if local, do: :local, else: :remote
@@ -305,7 +309,12 @@ defp restrict_unauthenticated?(%User{local: local}) do
 
   defp visible_account_status(user) do
     status = account_status(user)
-    status in [:active, :password_reset_pending] || status
+
+    if status in [:active, :password_reset_pending] do
+      :visible
+    else
+      status
+    end
   end
 
   @spec superuser?(User.t()) :: boolean()
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 1edc0d96a..8727faab7 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -221,7 +221,7 @@ def relationships(%{assigns: %{user: _user}} = conn, _), do: json(conn, [])
   @doc "GET /api/v1/accounts/:id"
   def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do
     with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user),
-         true <- User.visible_for(user, for_user) do
+         :visible <- User.visible_for(user, for_user) do
       render(conn, "show.json", user: user, for: for_user)
     else
       error -> user_visibility_error(conn, error)
@@ -231,7 +231,7 @@ def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do
   @doc "GET /api/v1/accounts/:id/statuses"
   def statuses(%{assigns: %{user: reading_user}} = conn, params) do
     with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user),
-         true <- User.visible_for(user, reading_user) do
+         :visible <- User.visible_for(user, reading_user) do
       params =
         params
         |> Map.delete(:tagged)
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 8e723d013..4a1508b22 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -35,7 +35,7 @@ def render("index.json", %{users: users} = opts) do
   end
 
   def render("show.json", %{user: user} = opts) do
-    if User.visible_for(user, opts[:for]) == true do
+    if User.visible_for(user, opts[:for]) == :visible do
       do_render("show.json", opts)
     else
       %{}
diff --git a/test/user_test.exs b/test/user_test.exs
index 3bfcfd10c..6865bd9be 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1293,7 +1293,7 @@ test "returns false for a non-invisible user" do
     test "returns true when the account is itself" do
       user = insert(:user, local: true)
 
-      assert User.visible_for(user, user)
+      assert User.visible_for(user, user) == :visible
     end
 
     test "returns false when the account is unauthenticated and auth is required" do
@@ -1302,14 +1302,14 @@ test "returns false when the account is unauthenticated and auth is required" do
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true)
 
-      refute User.visible_for(user, other_user) == true
+      refute User.visible_for(user, other_user) == :visible
     end
 
     test "returns true when the account is unauthenticated and auth is not required" do
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true)
 
-      assert User.visible_for(user, other_user)
+      assert User.visible_for(user, other_user) == :visible
     end
 
     test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do
@@ -1318,7 +1318,7 @@ test "returns true when the account is unauthenticated and being viewed by a pri
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true, is_admin: true)
 
-      assert User.visible_for(user, other_user)
+      assert User.visible_for(user, other_user) == :visible
     end
   end
 

From 0321a3e07814c3f225f19e0372b69a7813cef15e Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 18 May 2020 10:34:34 +0300
Subject: [PATCH 103/283] test naming fix

---
 test/web/mastodon_api/controllers/account_controller_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index 8700ab2f5..3008970af 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -128,7 +128,7 @@ test "returns 404 for internal.fetch actor", %{conn: conn} do
                |> json_response_and_validate_schema(404)
     end
 
-    test "returns 401 for deactivated user", %{conn: conn} do
+    test "returns 404 for deactivated user", %{conn: conn} do
       user = insert(:user, deactivated: true)
 
       assert %{"error" => "Can't find user"} =

From be4db41d713f981cc464e5fa7bc7191d3ff776d6 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 18 May 2020 18:45:33 +0200
Subject: [PATCH 104/283] ChatMessageValidator: Allow one message in an array,
 too.

---
 .../chat_message_validator.ex                 |  9 +++++
 .../activity_pub/object_validator_test.exs    | 35 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 9c20c188a..138736f23 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -47,9 +47,18 @@ def cast_data(data) do
   def fix(data) do
     data
     |> fix_emoji()
+    |> fix_attachment()
     |> Map.put_new("actor", data["attributedTo"])
   end
 
+  # Throws everything but the first one away
+  def fix_attachment(%{"attachment" => [attachment | _]} = data) do
+    data
+    |> Map.put("attachment", attachment)
+  end
+
+  def fix_attachment(data), do: data
+
   def changeset(struct, data) do
     data = fix(data)
 
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index da33d3dbc..a79e50a29 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -13,6 +13,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   import Pleroma.Factory
 
   describe "attachments" do
+    test "works with honkerific attachments" do
+      attachment = %{
+        "mediaType" => "image/jpeg",
+        "name" => "298p3RG7j27tfsZ9RQ.jpg",
+        "summary" => "298p3RG7j27tfsZ9RQ.jpg",
+        "type" => "Document",
+        "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg"
+      }
+
+      assert {:ok, attachment} =
+               AttachmentValidator.cast_and_validate(attachment)
+               |> Ecto.Changeset.apply_action(:insert)
+    end
+
     test "it turns mastodon attachments into our attachments" do
       attachment = %{
         "url" =>
@@ -103,6 +117,27 @@ test "validates for a basic object with an attachment", %{
       assert object["attachment"]
     end
 
+    test "validates for a basic object with an attachment in an array", %{
+      valid_chat_message: valid_chat_message,
+      user: user
+    } do
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, attachment} = ActivityPub.upload(file, actor: user.ap_id)
+
+      valid_chat_message =
+        valid_chat_message
+        |> Map.put("attachment", [attachment.data])
+
+      assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+
+      assert object["attachment"]
+    end
+
     test "validates for a basic object with an attachment but without content", %{
       valid_chat_message: valid_chat_message,
       user: user

From d19c7167704308df093f060082639c0a15996af7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 18 May 2020 20:17:28 +0200
Subject: [PATCH 105/283] AttachmentValidator: Handle empty mediatypes

---
 .../object_validators/attachment_validator.ex | 14 +++++--
 .../activity_pub/object_validator_test.exs    |  4 +-
 .../transmogrifier/chat_message_test.exs      | 37 +++++++++++++++++++
 3 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index 16ed49051..c4b502cb9 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   @primary_key false
   embedded_schema do
     field(:type, :string)
-    field(:mediaType, :string)
+    field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
 
     embeds_many(:url, UrlObjectValidator)
@@ -41,8 +41,16 @@ def changeset(struct, data) do
   end
 
   def fix_media_type(data) do
-    data
-    |> Map.put_new("mediaType", data["mimeType"])
+    data =
+      data
+      |> Map.put_new("mediaType", data["mimeType"])
+
+    if data["mediaType"] == "" do
+      data
+      |> Map.put("mediaType", "application/octet-stream")
+    else
+      data
+    end
   end
 
   def fix_url(data) do
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index a79e50a29..ed6b84e8e 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -15,8 +15,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidatorTest do
   describe "attachments" do
     test "works with honkerific attachments" do
       attachment = %{
-        "mediaType" => "image/jpeg",
-        "name" => "298p3RG7j27tfsZ9RQ.jpg",
+        "mediaType" => "",
+        "name" => "",
         "summary" => "298p3RG7j27tfsZ9RQ.jpg",
         "type" => "Document",
         "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg"
diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index 85644d787..820090de3 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -13,6 +13,43 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   describe "handle_incoming" do
+    test "handles this" do
+      data = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "actor" => "https://honk.tedunangst.com/u/tedu",
+        "id" => "https://honk.tedunangst.com/u/tedu/honk/x6gt8X8PcyGkQcXxzg1T",
+        "object" => %{
+          "attachment" => [
+            %{
+              "mediaType" => "image/jpeg",
+              "name" => "298p3RG7j27tfsZ9RQ.jpg",
+              "summary" => "298p3RG7j27tfsZ9RQ.jpg",
+              "type" => "Document",
+              "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg"
+            }
+          ],
+          "attributedTo" => "https://honk.tedunangst.com/u/tedu",
+          "content" => "",
+          "id" => "https://honk.tedunangst.com/u/tedu/chonk/26L4wl5yCbn4dr4y1b",
+          "published" => "2020-05-18T01:13:03Z",
+          "to" => [
+            "https://dontbulling.me/users/lain"
+          ],
+          "type" => "ChatMessage"
+        },
+        "published" => "2020-05-18T01:13:03Z",
+        "to" => [
+          "https://dontbulling.me/users/lain"
+        ],
+        "type" => "Create"
+      }
+
+      _user = insert(:user, ap_id: data["actor"])
+      _user = insert(:user, ap_id: hd(data["to"]))
+
+      assert {:ok, _activity} = Transmogrifier.handle_incoming(data)
+    end
+
     test "it rejects messages that don't contain content" do
       data =
         File.read!("test/fixtures/create-chat-message.json")

From cc0d462e91dd29c834c56b82e02022e1babda369 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 21 May 2020 15:08:56 +0200
Subject: [PATCH 106/283] Attachments: Have the mediaType on the root, too.

---
 lib/pleroma/upload.ex                         |  1 +
 .../activity_pub/object_validator_test.exs    | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 1be1a3a5b..797555bff 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -67,6 +67,7 @@ def store(upload, opts \\ []) do
       {:ok,
        %{
          "type" => opts.activity_type,
+         "mediaType" => upload.content_type,
          "url" => [
            %{
              "type" => "Link",
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index ed6b84e8e..f9990bd2c 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -25,6 +25,8 @@ test "works with honkerific attachments" do
       assert {:ok, attachment} =
                AttachmentValidator.cast_and_validate(attachment)
                |> Ecto.Changeset.apply_action(:insert)
+
+      assert attachment.mediaType == "application/octet-stream"
     end
 
     test "it turns mastodon attachments into our attachments" do
@@ -48,6 +50,27 @@ test "it turns mastodon attachments into our attachments" do
                  mediaType: "image/jpeg"
                }
              ] = attachment.url
+
+      assert attachment.mediaType == "image/jpeg"
+    end
+
+    test "it handles our own uploads" do
+      user = insert(:user)
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, attachment} = ActivityPub.upload(file, actor: user.ap_id)
+
+      {:ok, attachment} =
+        attachment.data
+        |> AttachmentValidator.cast_and_validate()
+        |> Ecto.Changeset.apply_action(:insert)
+
+      assert attachment.mediaType == "image/jpeg"
     end
   end
 

From c4a5cead51770f0d54cb77805b7e2bd705f251d9 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 21 May 2020 15:17:39 +0200
Subject: [PATCH 107/283] UploadTest: Fix test.

---
 test/upload_test.exs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/upload_test.exs b/test/upload_test.exs
index 060a940bb..2abf0edec 100644
--- a/test/upload_test.exs
+++ b/test/upload_test.exs
@@ -54,6 +54,7 @@ test "it returns file" do
                 %{
                   "name" => "image.jpg",
                   "type" => "Document",
+                  "mediaType" => "image/jpeg",
                   "url" => [
                     %{
                       "href" => "http://localhost:4001/media/post-process-file.jpg",

From dbd07d29a358a446d87078d60b993a59b757ad1d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 25 May 2020 17:27:45 +0200
Subject: [PATCH 108/283] Streamer: Don't crash on streaming chat notifications

---
 lib/pleroma/web/common_api/common_api.ex |  9 +++++----
 test/web/streamer/streamer_test.exs      | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index c08edbc5f..764fa4f4f 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -467,12 +467,13 @@ def remove_mute(user, activity) do
     {:ok, activity}
   end
 
-  def thread_muted?(%{id: nil} = _user, _activity), do: false
-
-  def thread_muted?(user, activity) do
-    ThreadMute.exists?(user.id, activity.data["context"])
+  def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}})
+      when is_binary("context") do
+    ThreadMute.exists?(user_id, context)
   end
 
+  def thread_muted?(_, _), do: false
+
   def report(user, data) do
     with {:ok, account} <- get_reported_account(data.account_id),
          {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]),
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index cb4595bb6..115ba4703 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -126,6 +126,21 @@ test "it sends notify to in the 'user:notification' stream", %{user: user, notif
       refute Streamer.filtered_by_user?(user, notify)
     end
 
+    test "it sends chat message notifications to the 'user:notification' stream", %{user: user} do
+      other_user = insert(:user)
+
+      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+
+      notify =
+        Repo.get_by(Pleroma.Notification, user_id: user.id, activity_id: create_activity.id)
+        |> Repo.preload(:activity)
+
+      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.stream("user:notification", notify)
+      assert_receive {:render_with_user, _, _, ^notify}
+      refute Streamer.filtered_by_user?(user, notify)
+    end
+
     test "it doesn't send notify to the 'user:notification' stream when a user is blocked", %{
       user: user
     } do

From c86a88edec75223f650faa2bb442c09aa95ad694 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 15:24:41 +0200
Subject: [PATCH 109/283] Streamer: Add a chat message stream.

---
 lib/pleroma/web/streamer/streamer.ex   | 23 ++++++++++++++++++++++-
 lib/pleroma/web/views/streamer_view.ex | 19 +++++++++++++++++++
 test/web/streamer/streamer_test.exs    | 24 ++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 49a400df7..331490a78 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.Streamer do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
@@ -22,7 +23,7 @@ defmodule Pleroma.Web.Streamer do
   def registry, do: @registry
 
   @public_streams ["public", "public:local", "public:media", "public:local:media"]
-  @user_streams ["user", "user:notification", "direct"]
+  @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
 
   @doc "Expands and authorizes a stream, and registers the process for streaming."
   @spec get_topic_and_add_socket(stream :: String.t(), User.t() | nil, Map.t() | nil) ::
@@ -200,6 +201,26 @@ defp do_stream(topic, %Notification{} = item)
     end)
   end
 
+  defp do_stream(topic, %{data: %{"type" => "ChatMessage"}} = object)
+       when topic in ["user", "user:pleroma_chat"] do
+    recipients = [object.data["actor"] | object.data["to"]]
+
+    topics =
+      %{ap_id: recipients, local: true}
+      |> Pleroma.User.Query.build()
+      |> Repo.all()
+      |> Enum.map(fn %{id: id} = user -> {user, "#{topic}:#{id}"} end)
+
+    Enum.each(topics, fn {user, topic} ->
+      Registry.dispatch(@registry, topic, fn list ->
+        Enum.each(list, fn {pid, _auth} ->
+          text = StreamerView.render("chat_update.json", object, user, recipients)
+          send(pid, {:text, text})
+        end)
+      end)
+    end)
+  end
+
   defp do_stream("user", item) do
     Logger.debug("Trying to push to users")
 
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 237b29ded..949e2ed37 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -6,11 +6,30 @@ defmodule Pleroma.Web.StreamerView do
   use Pleroma.Web, :view
 
   alias Pleroma.Activity
+  alias Pleroma.Chat
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.User
   alias Pleroma.Web.MastodonAPI.NotificationView
 
+  def render("chat_update.json", object, user, recipients) do
+    chat = Chat.get(user.id, hd(recipients -- [user.ap_id]))
+
+    representation =
+      Pleroma.Web.PleromaAPI.ChatMessageView.render(
+        "show.json",
+        %{object: object, chat: chat}
+      )
+
+    %{
+      event: "pleroma:chat_update",
+      payload:
+        representation
+        |> Jason.encode!()
+    }
+    |> Jason.encode!()
+  end
+
   def render("update.json", %Activity{} = activity, %User{} = user) do
     %{
       event: "update",
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index 115ba4703..ffbff35ca 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -9,9 +9,11 @@ defmodule Pleroma.Web.StreamerTest do
 
   alias Pleroma.Conversation.Participation
   alias Pleroma.List
+  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Streamer
+  alias Pleroma.Web.StreamerView
 
   @moduletag needs_streamer: true, capture_log: true
 
@@ -126,6 +128,28 @@ test "it sends notify to in the 'user:notification' stream", %{user: user, notif
       refute Streamer.filtered_by_user?(user, notify)
     end
 
+    test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} do
+      other_user = insert(:user)
+
+      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+      object = Object.normalize(create_activity, false)
+      Streamer.get_topic_and_add_socket("user:pleroma_chat", user)
+      Streamer.stream("user:pleroma_chat", object)
+      text = StreamerView.render("chat_update.json", object, user, [user.ap_id, other_user.ap_id])
+      assert_receive {:text, ^text}
+    end
+
+    test "it sends chat messages to the 'user' stream", %{user: user} do
+      other_user = insert(:user)
+
+      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+      object = Object.normalize(create_activity, false)
+      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.stream("user", object)
+      text = StreamerView.render("chat_update.json", object, user, [user.ap_id, other_user.ap_id])
+      assert_receive {:text, ^text}
+    end
+
     test "it sends chat message notifications to the 'user:notification' stream", %{user: user} do
       other_user = insert(:user)
 

From 863c02b25d1c6128fab88c33d2c4c3565a6c378f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 15:44:03 +0200
Subject: [PATCH 110/283] SideEffects: Stream out chat messages.

---
 lib/pleroma/web/activity_pub/side_effects.ex |  2 ++
 test/web/activity_pub/side_effects_test.exs  | 21 ++++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index f0f0659c2..a4de8691e 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.Streamer
 
   def handle(object, meta \\ [])
 
@@ -126,6 +127,7 @@ def handle(object, meta) do
 
   def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
+      Streamer.stream(["user", "user:pleroma_chat"], object)
       actor = User.get_cached_by_ap_id(object.data["actor"])
       recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
 
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index fb4411c07..210ba6ef0 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -309,6 +309,27 @@ test "notifies the recipient" do
       assert Repo.get_by(Notification, user_id: recipient.id, activity_id: create_activity.id)
     end
 
+    test "it streams the created ChatMessage" do
+      author = insert(:user, local: true)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      with_mock Pleroma.Web.Streamer, [], stream: fn _, _ -> nil end do
+        {:ok, _create_activity, _meta} =
+          SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
+
+        object = Object.normalize(create_activity, false)
+
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], object))
+      end
+    end
+
     test "it creates a Chat for the local users and bumps the unread count, except for the author" do
       author = insert(:user, local: true)
       recipient = insert(:user, local: true)

From 767ce8b8030562935ccd9f7c3d9ed83af0735db0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 16:02:45 +0200
Subject: [PATCH 111/283] StreamerView: Actually send Chats, not ChatMessages.

---
 lib/pleroma/web/pleroma_api/views/chat_view.ex | 2 +-
 lib/pleroma/web/views/streamer_view.ex         | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 08d5110c3..223b64987 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
 
-    last_message = Chat.last_message_for_chat(chat)
+    last_message = opts[:message] || Chat.last_message_for_chat(chat)
 
     %{
       id: chat.id |> to_string(),
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 949e2ed37..5e953d770 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -16,9 +16,9 @@ def render("chat_update.json", object, user, recipients) do
     chat = Chat.get(user.id, hd(recipients -- [user.ap_id]))
 
     representation =
-      Pleroma.Web.PleromaAPI.ChatMessageView.render(
+      Pleroma.Web.PleromaAPI.ChatView.render(
         "show.json",
-        %{object: object, chat: chat}
+        %{message: object, chat: chat}
       )
 
     %{

From b08baf905b09ac49ed908eff8b43593d890612dd Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 16:03:55 +0200
Subject: [PATCH 112/283] Docs: Document streaming differences

---
 docs/API/differences_in_mastoapi_responses.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md
index e65fd5da4..a9d1f2f38 100644
--- a/docs/API/differences_in_mastoapi_responses.md
+++ b/docs/API/differences_in_mastoapi_responses.md
@@ -226,3 +226,7 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 Has these additional fields under the `pleroma` object:
 
 - `unread_count`: contains number unread notifications
+
+## Streaming
+
+There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.

From 3898dd69a69d3af9793f2e1d442b409c84b319a8 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 16:05:02 +0200
Subject: [PATCH 113/283] SideEffects: Ensure a chat is present before
 streaming something out.

---
 lib/pleroma/web/activity_pub/side_effects.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a4de8691e..02296b210 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -127,7 +127,6 @@ def handle(object, meta) do
 
   def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
-      Streamer.stream(["user", "user:pleroma_chat"], object)
       actor = User.get_cached_by_ap_id(object.data["actor"])
       recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
 
@@ -142,6 +141,7 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
         end
       end)
 
+      Streamer.stream(["user", "user:pleroma_chat"], object)
       {:ok, object, meta}
     end
   end

From 32431ad1ee88d260b720fab05fce76eb75bfe107 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 29 May 2020 16:07:40 +0200
Subject: [PATCH 114/283] Docs: Also add the streaming docs to the Chat api
 doc.

---
 docs/API/chats.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 2e415e4da..2eca5adf6 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -220,3 +220,7 @@ There's a new `pleroma:chat_mention` notification, which has this form:
   "created_at": "somedate"
 }
 ```
+
+### Streaming
+
+There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.

From 2c9465cc51160546ae054d1a1912fbb8e9add8e8 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 30 May 2020 12:17:18 +0200
Subject: [PATCH 115/283] SafeText: Let through basic html.

---
 .../object_validators/types/safe_text.ex           |  2 +-
 test/web/activity_pub/object_validator_test.exs    | 14 ++++++++++++++
 .../object_validators/types/safe_text_test.exs     |  7 +++++++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
index 822e8d2c1..95c948123 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do
   def type, do: :string
 
   def cast(str) when is_binary(str) do
-    {:ok, HTML.strip_tags(str)}
+    {:ok, HTML.filter_tags(str)}
   end
 
   def cast(_), do: :error
diff --git a/test/web/activity_pub/object_validator_test.exs b/test/web/activity_pub/object_validator_test.exs
index 929fdbc9b..31224abe0 100644
--- a/test/web/activity_pub/object_validator_test.exs
+++ b/test/web/activity_pub/object_validator_test.exs
@@ -113,6 +113,20 @@ test "it is invalid if the object data has a different `to` or `actor` field" do
       %{user: user, recipient: recipient, valid_chat_message: valid_chat_message}
     end
 
+    test "let's through some basic html", %{user: user, recipient: recipient} do
+      {:ok, valid_chat_message, _} =
+        Builder.chat_message(
+          user,
+          recipient.ap_id,
+          "hey <a href='https://example.org'>example</a> <script>alert('uguu')</script>"
+        )
+
+      assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
+
+      assert object["content"] ==
+               "hey <a href=\"https://example.org\">example</a> alert(&#39;uguu&#39;)"
+    end
+
     test "validates for a basic object we build", %{valid_chat_message: valid_chat_message} do
       assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, [])
 
diff --git a/test/web/activity_pub/object_validators/types/safe_text_test.exs b/test/web/activity_pub/object_validators/types/safe_text_test.exs
index 59ed0a1fe..d4a574554 100644
--- a/test/web/activity_pub/object_validators/types/safe_text_test.exs
+++ b/test/web/activity_pub/object_validators/types/safe_text_test.exs
@@ -17,6 +17,13 @@ test "it removes html tags from text" do
     assert {:ok, "hey look xss alert(&#39;foo&#39;)"} == SafeText.cast(text)
   end
 
+  test "it keeps basic html tags" do
+    text = "hey <a href='http://gensokyo.2hu'>look</a> xss <script>alert('foo')</script>"
+
+    assert {:ok, "hey <a href=\"http://gensokyo.2hu\">look</a> xss alert(&#39;foo&#39;)"} ==
+             SafeText.cast(text)
+  end
+
   test "errors for non-text" do
     assert :error == SafeText.cast(1)
   end

From 8bdf18d7c10f0e740b2f5e0fa5063c522b8b3872 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 30 May 2020 12:30:31 +0200
Subject: [PATCH 116/283] CommonAPI: Linkify chat messages.

---
 lib/pleroma/web/common_api/common_api.ex |  7 ++++++-
 test/web/common_api/common_api_test.exs  | 23 +++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 764fa4f4f..173353aa5 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -50,7 +50,12 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
   defp format_chat_content(nil), do: nil
 
   defp format_chat_content(content) do
-    content |> Formatter.html_escape("text/plain")
+    {text, _, _} =
+      content
+      |> Formatter.html_escape("text/plain")
+      |> Formatter.linkify()
+
+    text
   end
 
   defp validate_chat_content_length(_, true), do: :ok
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 9e129e5a7..41c6909de 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -50,6 +50,29 @@ test "it posts a chat message without content but with an attachment" do
       assert activity
     end
 
+    test "it linkifies" do
+      author = insert(:user)
+      recipient = insert(:user)
+
+      other_user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          "https://example.org is the site of @#{other_user.nickname} #2hu"
+        )
+
+      assert other_user.ap_id not in activity.recipients
+
+      object = Object.normalize(activity, false)
+
+      assert object.data["content"] ==
+               "<a href=\"https://example.org\" rel=\"ugc\">https://example.org</a> is the site of <span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{
+                 other_user.id
+               }\" href=\"#{other_user.ap_id}\" rel=\"ugc\">@<span>#{other_user.nickname}</span></a></span> <a class=\"hashtag\" data-tag=\"2hu\" href=\"http://localhost:4001/tag/2hu\">#2hu</a>"
+    end
+
     test "it posts a chat message" do
       author = insert(:user)
       recipient = insert(:user)

From af9090238e1f71e6b081fbd09c09a5975d2ed99e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 1 Jun 2020 15:14:22 +0200
Subject: [PATCH 117/283] CommonAPI: Newlines -> br for chat messages.

---
 lib/pleroma/web/common_api/common_api.ex |  3 +++
 test/web/common_api/common_api_test.exs  | 20 ++++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 173353aa5..e0987b1a7 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -54,6 +54,9 @@ defp format_chat_content(content) do
       content
       |> Formatter.html_escape("text/plain")
       |> Formatter.linkify()
+      |> (fn {text, mentions, tags} ->
+            {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
+          end).()
 
     text
   end
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 41c6909de..611a9ae66 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -50,6 +50,26 @@ test "it posts a chat message without content but with an attachment" do
       assert activity
     end
 
+    test "it adds html newlines" do
+      author = insert(:user)
+      recipient = insert(:user)
+
+      other_user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post_chat_message(
+          author,
+          recipient,
+          "uguu\nuguuu"
+        )
+
+      assert other_user.ap_id not in activity.recipients
+
+      object = Object.normalize(activity, false)
+
+      assert object.data["content"] == "uguu<br/>uguuu"
+    end
+
     test "it linkifies" do
       author = insert(:user)
       recipient = insert(:user)

From 805ab86933d90d4284c83e4a8ebfd6bf4b0395b3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 13:24:34 +0200
Subject: [PATCH 118/283] Notifications: Make notifications save their type.

---
 lib/pleroma/following_relationship.ex         | 11 ++--
 lib/pleroma/notification.ex                   | 61 ++++++++++++++++++-
 .../web/activity_pub/transmogrifier.ex        |  3 +
 .../operations/notification_operation.ex      |  1 +
 lib/pleroma/web/common_api/common_api.ex      |  1 +
 .../mastodon_api/views/notification_view.ex   | 20 +-----
 ...200602094828_add_type_to_notifications.exs |  9 +++
 test/notification_test.exs                    | 11 +++-
 8 files changed, 90 insertions(+), 27 deletions(-)
 create mode 100644 priv/repo/migrations/20200602094828_add_type_to_notifications.exs

diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex
index 3a3082e72..0343a20d4 100644
--- a/lib/pleroma/following_relationship.ex
+++ b/lib/pleroma/following_relationship.ex
@@ -62,10 +62,13 @@ def update(%User{} = follower, %User{} = following, state) do
         follow(follower, following, state)
 
       following_relationship ->
-        following_relationship
-        |> cast(%{state: state}, [:state])
-        |> validate_required([:state])
-        |> Repo.update()
+        {:ok, relationship} =
+          following_relationship
+          |> cast(%{state: state}, [:state])
+          |> validate_required([:state])
+          |> Repo.update()
+
+        {:ok, relationship}
     end
   end
 
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index efafbce48..41ac53505 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -30,12 +30,26 @@ defmodule Pleroma.Notification do
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
+    field(:type, :string)
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
 
     timestamps()
   end
 
+  def update_notification_type(user, activity) do
+    with %__MODULE__{} = notification <-
+           Repo.get_by(__MODULE__, user_id: user.id, activity_id: activity.id) do
+      type =
+        activity
+        |> type_from_activity()
+
+      notification
+      |> changeset(%{type: type})
+      |> Repo.update()
+    end
+  end
+
   @spec unread_notifications_count(User.t()) :: integer()
   def unread_notifications_count(%User{id: user_id}) do
     from(q in __MODULE__,
@@ -46,7 +60,7 @@ def unread_notifications_count(%User{id: user_id}) do
 
   def changeset(%Notification{} = notification, attrs) do
     notification
-    |> cast(attrs, [:seen])
+    |> cast(attrs, [:seen, :type])
   end
 
   @spec last_read_query(User.t()) :: Ecto.Queryable.t()
@@ -330,12 +344,55 @@ defp do_create_notifications(%Activity{} = activity) do
     {:ok, notifications}
   end
 
+  defp type_from_activity(%{data: %{"type" => type}} = activity) do
+    case type do
+      "Follow" ->
+        if Activity.follow_accepted?(activity) do
+          "follow"
+        else
+          "follow_request"
+        end
+
+      "Announce" ->
+        "reblog"
+
+      "Like" ->
+        "favourite"
+
+      "Move" ->
+        "move"
+
+      "EmojiReact" ->
+        "pleroma:emoji_reaction"
+
+      "Create" ->
+        activity
+        |> type_from_activity_object()
+
+      t ->
+        raise "No notification type for activity type #{t}"
+    end
+  end
+
+  defp type_from_activity_object(%{data: %{"type" => "Create"}} = activity) do
+    object = Object.normalize(activity, false)
+
+    case object.data["type"] do
+      "ChatMessage" -> "pleroma:chat_mention"
+      _ -> "mention"
+    end
+  end
+
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
     unless skip?(activity, user) do
       {:ok, %{notification: notification}} =
         Multi.new()
-        |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity})
+        |> Multi.insert(:notification, %Notification{
+          user_id: user.id,
+          activity: activity,
+          type: type_from_activity(activity)
+        })
         |> Marker.multi_set_last_read_id(user, "notifications")
         |> Repo.transaction()
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 4ac0d43fc..886403fcd 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
@@ -595,6 +596,8 @@ def handle_incoming(
       User.update_follower_count(followed)
       User.update_following_count(follower)
 
+      Notification.update_notification_type(followed, follow_activity)
+
       ActivityPub.accept(%{
         to: follow_activity.data["to"],
         type: "Accept",
diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index 46e72f8bf..c966b553a 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -185,6 +185,7 @@ defp notification_type do
         "mention",
         "poll",
         "pleroma:emoji_reaction",
+        "pleroma:chat_mention",
         "move",
         "follow_request"
       ],
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e0987b1a7..5a194910d 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -121,6 +121,7 @@ def accept_follow_request(follower, followed) do
              object: follow_activity.data["id"],
              type: "Accept"
            }) do
+      Notification.update_notification_type(followed, follow_activity)
       {:ok, follower}
     end
   end
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 07d55a3e9..c090be8ad 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -81,22 +81,6 @@ def render(
       end
     end
 
-    # This returns the notification type by activity, but both chats and statuses
-    # are in "Create" activities.
-    mastodon_type =
-      case Activity.mastodon_notification_type(activity) do
-        "mention" ->
-          object = Object.normalize(activity)
-
-          case object do
-            %{data: %{"type" => "ChatMessage"}} -> "pleroma:chat_mention"
-            _ -> "mention"
-          end
-
-        type ->
-          type
-      end
-
     # Note: :relationships contain user mutes (needed for :muted flag in :status)
     status_render_opts = %{relationships: opts[:relationships]}
 
@@ -107,7 +91,7 @@ def render(
            ) do
       response = %{
         id: to_string(notification.id),
-        type: mastodon_type,
+        type: notification.type,
         created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
         account: account,
         pleroma: %{
@@ -115,7 +99,7 @@ def render(
         }
       }
 
-      case mastodon_type do
+      case notification.type do
         "mention" ->
           put_status(response, activity, reading_user, status_render_opts)
 
diff --git a/priv/repo/migrations/20200602094828_add_type_to_notifications.exs b/priv/repo/migrations/20200602094828_add_type_to_notifications.exs
new file mode 100644
index 000000000..19c733628
--- /dev/null
+++ b/priv/repo/migrations/20200602094828_add_type_to_notifications.exs
@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.AddTypeToNotifications do
+  use Ecto.Migration
+
+  def change do
+    alter table(:notifications) do
+      add(:type, :string)
+    end
+  end
+end
diff --git a/test/notification_test.exs b/test/notification_test.exs
index 37c255fee..421b7fc40 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -31,6 +31,7 @@ test "creates a notification for an emoji reaction" do
       {:ok, [notification]} = Notification.create_notifications(activity)
 
       assert notification.user_id == user.id
+      assert notification.type == "pleroma:emoji_reaction"
     end
 
     test "notifies someone when they are directly addressed" do
@@ -48,6 +49,7 @@ test "notifies someone when they are directly addressed" do
       notified_ids = Enum.sort([notification.user_id, other_notification.user_id])
       assert notified_ids == [other_user.id, third_user.id]
       assert notification.activity_id == activity.id
+      assert notification.type == "mention"
       assert other_notification.activity_id == activity.id
 
       assert [%Pleroma.Marker{unread_count: 2}] =
@@ -335,9 +337,12 @@ test "it creates `follow_request` notification for pending Follow activity" do
       # After request is accepted, the same notification is rendered with type "follow":
       assert {:ok, _} = CommonAPI.accept_follow_request(user, followed_user)
 
-      notification_id = notification.id
-      assert [%{id: ^notification_id}] = Notification.for_user(followed_user)
-      assert %{type: "follow"} = NotificationView.render("show.json", render_opts)
+      notification =
+        Repo.get(Notification, notification.id)
+        |> Repo.preload(:activity)
+
+      assert %{type: "follow"} =
+               NotificationView.render("show.json", notification: notification, for: followed_user)
     end
 
     test "it doesn't create a notification for follow-unfollow-follow chains" do

From 127ccc4e1c76c2782b26a0cfbb154bc1317f31b3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 14:05:53 +0200
Subject: [PATCH 119/283] NotificationController: Don't return chat_mentions by
 default.

---
 docs/API/chats.md                             |  2 +-
 .../controllers/notification_controller.ex    | 14 ++++++++++++-
 lib/pleroma/web/mastodon_api/mastodon_api.ex  | 15 ++-----------
 .../notification_controller_test.exs          | 21 +++++++++++++++++++
 4 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 2eca5adf6..d1d39f495 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -204,7 +204,7 @@ Returned data is the deleted message.
 
 ### Notifications
 
-There's a new `pleroma:chat_mention` notification, which has this form:
+There's a new `pleroma:chat_mention` notification, which has this form. It is not given out in the notifications endpoint by default, you need to explicitly request it with `include_types[]=pleroma:chat_mention`:
 
 ```json
 {
diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
index bcd12c73f..e25cef30b 100644
--- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
@@ -42,8 +42,20 @@ def index(conn, %{account_id: account_id} = params) do
     end
   end
 
+  @default_notification_types ~w{
+    mention
+    follow
+    follow_request
+    reblog
+    favourite
+    move
+    pleroma:emoji_reaction
+  }
   def index(%{assigns: %{user: user}} = conn, params) do
-    params = Map.new(params, fn {k, v} -> {to_string(k), v} end)
+    params =
+      Map.new(params, fn {k, v} -> {to_string(k), v} end)
+      |> Map.put_new("include_types", @default_notification_types)
+
     notifications = MastodonAPI.get_notifications(user, params)
 
     conn
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex
index 70da64a7a..694bf5ca8 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
   import Ecto.Query
   import Ecto.Changeset
 
-  alias Pleroma.Activity
   alias Pleroma.Notification
   alias Pleroma.Pagination
   alias Pleroma.ScheduledActivity
@@ -82,15 +81,11 @@ defp cast_params(params) do
   end
 
   defp restrict(query, :include_types, %{include_types: mastodon_types = [_ | _]}) do
-    ap_types = convert_and_filter_mastodon_types(mastodon_types)
-
-    where(query, [q, a], fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data))
+    where(query, [n], n.type in ^mastodon_types)
   end
 
   defp restrict(query, :exclude_types, %{exclude_types: mastodon_types = [_ | _]}) do
-    ap_types = convert_and_filter_mastodon_types(mastodon_types)
-
-    where(query, [q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data))
+    where(query, [n], n.type not in ^mastodon_types)
   end
 
   defp restrict(query, :account_ap_id, %{account_ap_id: account_ap_id}) do
@@ -98,10 +93,4 @@ defp restrict(query, :account_ap_id, %{account_ap_id: account_ap_id}) do
   end
 
   defp restrict(query, _, _), do: query
-
-  defp convert_and_filter_mastodon_types(types) do
-    types
-    |> Enum.map(&Activity.from_mastodon_notification_type/1)
-    |> Enum.filter(& &1)
-  end
 end
diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs
index e278d61f5..698c99711 100644
--- a/test/web/mastodon_api/controllers/notification_controller_test.exs
+++ b/test/web/mastodon_api/controllers/notification_controller_test.exs
@@ -54,6 +54,27 @@ test "list of notifications" do
     assert response == expected_response
   end
 
+  test "by default, does not contain pleroma:chat_mention" do
+    %{user: user, conn: conn} = oauth_access(["read:notifications"])
+    other_user = insert(:user)
+
+    {:ok, _activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+
+    result =
+      conn
+      |> get("/api/v1/notifications")
+      |> json_response_and_validate_schema(200)
+
+    assert [] == result
+
+    result =
+      conn
+      |> get("/api/v1/notifications?include_types[]=pleroma:chat_mention")
+      |> json_response_and_validate_schema(200)
+
+    assert [_] = result
+  end
+
   test "getting a single notification" do
     %{user: user, conn: conn} = oauth_access(["read:notifications"])
     other_user = insert(:user)

From 37542a9dfa99cc4324f211b45254acea758ac1ae Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 14:22:16 +0200
Subject: [PATCH 120/283] Activity: Remove notifications-related functions.

---
 lib/pleroma/activity.ex                       | 36 -------------------
 .../mastodon_api/views/notification_view.ex   | 15 ++++----
 lib/pleroma/web/push/impl.ex                  | 14 +++-----
 test/web/push/impl_test.exs                   | 25 +++++++------
 4 files changed, 26 insertions(+), 64 deletions(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 6213d0eb7..da1be20b3 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -24,16 +24,6 @@ defmodule Pleroma.Activity do
 
   @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
-  # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
-  @mastodon_notification_types %{
-    "Create" => "mention",
-    "Follow" => ["follow", "follow_request"],
-    "Announce" => "reblog",
-    "Like" => "favourite",
-    "Move" => "move",
-    "EmojiReact" => "pleroma:emoji_reaction"
-  }
-
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -300,32 +290,6 @@ def follow_accepted?(
 
   def follow_accepted?(_), do: false
 
-  @spec mastodon_notification_type(Activity.t()) :: String.t() | nil
-
-  for {ap_type, type} <- @mastodon_notification_types, not is_list(type) do
-    def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
-      do: unquote(type)
-  end
-
-  def mastodon_notification_type(%Activity{data: %{"type" => "Follow"}} = activity) do
-    if follow_accepted?(activity) do
-      "follow"
-    else
-      "follow_request"
-    end
-  end
-
-  def mastodon_notification_type(%Activity{}), do: nil
-
-  @spec from_mastodon_notification_type(String.t()) :: String.t() | nil
-  @doc "Converts Mastodon notification type to AR activity type"
-  def from_mastodon_notification_type(type) do
-    with {k, _v} <-
-           Enum.find(@mastodon_notification_types, fn {_k, v} -> type in List.wrap(v) end) do
-      k
-    end
-  end
-
   def all_by_actor_and_id(actor, status_ids \\ [])
   def all_by_actor_and_id(_actor, []), do: []
 
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index c090be8ad..af15bba48 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -16,18 +16,17 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.PleromaAPI.ChatMessageView
 
+  @parent_types ~w{Like Announce EmojiReact}
+
   def render("index.json", %{notifications: notifications, for: reading_user} = opts) do
     activities = Enum.map(notifications, & &1.activity)
 
     parent_activities =
       activities
-      |> Enum.filter(
-        &(Activity.mastodon_notification_type(&1) in [
-            "favourite",
-            "reblog",
-            "pleroma:emoji_reaction"
-          ])
-      )
+      |> Enum.filter(fn
+        %{data: %{"type" => type}} ->
+          type in @parent_types
+      end)
       |> Enum.map(& &1.data["object"])
       |> Activity.create_by_object_ap_id()
       |> Activity.with_preloaded_object(:left)
@@ -44,7 +43,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op
         true ->
           move_activities_targets =
             activities
-            |> Enum.filter(&(Activity.mastodon_notification_type(&1) == "move"))
+            |> Enum.filter(&(&1.data["type"] == "Move"))
             |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"]))
 
           actors =
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 691725702..125f33755 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -16,8 +16,6 @@ defmodule Pleroma.Web.Push.Impl do
   require Logger
   import Ecto.Query
 
-  defdelegate mastodon_notification_type(activity), to: Activity
-
   @types ["Create", "Follow", "Announce", "Like", "Move"]
 
   @doc "Performs sending notifications for user subscriptions"
@@ -31,7 +29,7 @@ def perform(
       when activity_type in @types do
     actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
 
-    mastodon_type = mastodon_notification_type(notification.activity)
+    mastodon_type = notification.type
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
     object = Object.normalize(activity)
@@ -116,7 +114,7 @@ def build_content(
   end
 
   def build_content(notification, actor, object, mastodon_type) do
-    mastodon_type = mastodon_type || mastodon_notification_type(notification.activity)
+    mastodon_type = mastodon_type || notification.type
 
     %{
       title: format_title(notification, mastodon_type),
@@ -151,7 +149,7 @@ def format_body(
         mastodon_type
       )
       when type in ["Follow", "Like"] do
-    mastodon_type = mastodon_type || mastodon_notification_type(notification.activity)
+    mastodon_type = mastodon_type || notification.type
 
     case mastodon_type do
       "follow" -> "@#{actor.nickname} has followed you"
@@ -166,10 +164,8 @@ def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_typ
     "New Direct Message"
   end
 
-  def format_title(%{activity: activity}, mastodon_type) do
-    mastodon_type = mastodon_type || mastodon_notification_type(activity)
-
-    case mastodon_type do
+  def format_title(%{type: type}, mastodon_type) do
+    case mastodon_type || type do
       "mention" -> "New Mention"
       "follow" -> "New Follower"
       "follow_request" -> "New Follow Request"
diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs
index a826b24c9..26c65bc82 100644
--- a/test/web/push/impl_test.exs
+++ b/test/web/push/impl_test.exs
@@ -60,7 +60,8 @@ test "performs sending notifications" do
     notif =
       insert(:notification,
         user: user,
-        activity: activity
+        activity: activity,
+        type: "mention"
       )
 
     assert Impl.perform(notif) == {:ok, [:ok, :ok]}
@@ -126,7 +127,7 @@ test "renders title and body for create activity" do
            ) ==
              "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
 
-    assert Impl.format_title(%{activity: activity}) ==
+    assert Impl.format_title(%{activity: activity, type: "mention"}) ==
              "New Mention"
   end
 
@@ -136,9 +137,10 @@ test "renders title and body for follow activity" do
     {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
     object = Object.normalize(activity, false)
 
-    assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has followed you"
+    assert Impl.format_body(%{activity: activity, type: "follow"}, user, object) ==
+             "@Bob has followed you"
 
-    assert Impl.format_title(%{activity: activity}) ==
+    assert Impl.format_title(%{activity: activity, type: "follow"}) ==
              "New Follower"
   end
 
@@ -157,7 +159,7 @@ test "renders title and body for announce activity" do
     assert Impl.format_body(%{activity: announce_activity}, user, object) ==
              "@#{user.nickname} repeated: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
 
-    assert Impl.format_title(%{activity: announce_activity}) ==
+    assert Impl.format_title(%{activity: announce_activity, type: "reblog"}) ==
              "New Repeat"
   end
 
@@ -173,9 +175,10 @@ test "renders title and body for like activity" do
     {:ok, activity} = CommonAPI.favorite(user, activity.id)
     object = Object.normalize(activity)
 
-    assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has favorited your post"
+    assert Impl.format_body(%{activity: activity, type: "favourite"}, user, object) ==
+             "@Bob has favorited your post"
 
-    assert Impl.format_title(%{activity: activity}) ==
+    assert Impl.format_title(%{activity: activity, type: "favourite"}) ==
              "New Favorite"
   end
 
@@ -218,7 +221,7 @@ test "hides details for notifications when privacy option enabled" do
           status: "<Lorem ipsum dolor sit amet."
         })
 
-      notif = insert(:notification, user: user2, activity: activity)
+      notif = insert(:notification, user: user2, activity: activity, type: "mention")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
       object = Object.normalize(activity)
@@ -229,7 +232,7 @@ test "hides details for notifications when privacy option enabled" do
 
       {:ok, activity} = CommonAPI.favorite(user, activity.id)
 
-      notif = insert(:notification, user: user2, activity: activity)
+      notif = insert(:notification, user: user2, activity: activity, type: "favourite")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
       object = Object.normalize(activity)
@@ -268,7 +271,7 @@ test "returns regular content for notifications with privacy option disabled" do
             "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
         })
 
-      notif = insert(:notification, user: user2, activity: activity)
+      notif = insert(:notification, user: user2, activity: activity, type: "mention")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
       object = Object.normalize(activity)
@@ -281,7 +284,7 @@ test "returns regular content for notifications with privacy option disabled" do
 
       {:ok, activity} = CommonAPI.favorite(user, activity.id)
 
-      notif = insert(:notification, user: user2, activity: activity)
+      notif = insert(:notification, user: user2, activity: activity, type: "favourite")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
       object = Object.normalize(activity)

From 38dce485c47e9315663c5c9cfd67dab4164b1bbe Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 14:49:56 +0200
Subject: [PATCH 121/283] Notification: Add function to backfill notification
 types

---
 lib/pleroma/notification.ex | 20 ++++++++++++++++++++
 test/notification_test.exs  | 28 ++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 41ac53505..c8b964400 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -37,6 +37,26 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
+  def fill_in_notification_types() do
+    query =
+      from(n in __MODULE__,
+        where: is_nil(n.type),
+        preload: :activity
+      )
+
+    query
+    |> Repo.all()
+    |> Enum.each(fn notification ->
+      type =
+        notification.activity
+        |> type_from_activity()
+
+      notification
+      |> changeset(%{type: type})
+      |> Repo.update()
+    end)
+  end
+
   def update_notification_type(user, activity) do
     with %__MODULE__{} = notification <-
            Repo.get_by(__MODULE__, user_id: user.id, activity_id: activity.id) do
diff --git a/test/notification_test.exs b/test/notification_test.exs
index 421b7fc40..6bc2b6904 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -20,6 +20,34 @@ defmodule Pleroma.NotificationTest do
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
 
+  describe "fill_in_notification_types" do
+    test "it fills in missing notification types" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, post} = CommonAPI.post(user, %{status: "yeah, @#{other_user.nickname}"})
+      {:ok, chat} = CommonAPI.post_chat_message(user, other_user, "yo")
+      {:ok, react} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
+      {:ok, like} = CommonAPI.favorite(other_user, post.id)
+
+      assert {4, nil} = Repo.update_all(Notification, set: [type: nil])
+
+      Notification.fill_in_notification_types()
+
+      assert %{type: "mention"} =
+               Repo.get_by(Notification, user_id: other_user.id, activity_id: post.id)
+
+      assert %{type: "favourite"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: like.id)
+
+      assert %{type: "pleroma:emoji_reaction"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: react.id)
+
+      assert %{type: "pleroma:chat_mention"} =
+               Repo.get_by(Notification, user_id: other_user.id, activity_id: chat.id)
+    end
+  end
+
   describe "create_notifications" do
     test "creates a notification for an emoji reaction" do
       user = insert(:user)

From 6cd2fa2a4cbffaaab7c911f1051d4917e8a06c78 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 15:13:19 +0200
Subject: [PATCH 122/283] Migrations: Add a migration to backfill notification
 types.

---
 lib/pleroma/notification.ex                   | 23 +++++++++++++++----
 ...0602125218_backfill_notification_types.exs | 10 ++++++++
 2 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 priv/repo/migrations/20200602125218_backfill_notification_types.exs

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index c8b964400..d89ee4645 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -49,7 +49,7 @@ def fill_in_notification_types() do
     |> Enum.each(fn notification ->
       type =
         notification.activity
-        |> type_from_activity()
+        |> type_from_activity(no_cachex: true)
 
       notification
       |> changeset(%{type: type})
@@ -364,10 +364,23 @@ defp do_create_notifications(%Activity{} = activity) do
     {:ok, notifications}
   end
 
-  defp type_from_activity(%{data: %{"type" => type}} = activity) do
+  defp type_from_activity(%{data: %{"type" => type}} = activity, opts \\ []) do
     case type do
       "Follow" ->
-        if Activity.follow_accepted?(activity) do
+        accepted_function =
+          if Keyword.get(opts, :no_cachex, false) do
+            # A special function to make this usable in a migration.
+            fn activity ->
+              with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]),
+                   %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do
+                Pleroma.FollowingRelationship.following?(follower, followed)
+              end
+            end
+          else
+            &Activity.follow_accepted?/1
+          end
+
+        if accepted_function.(activity) do
           "follow"
         else
           "follow_request"
@@ -394,8 +407,10 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do
     end
   end
 
+  defp type_from_activity_object(%{data: %{"type" => "Create", "object" => %{}}}), do: "mention"
+
   defp type_from_activity_object(%{data: %{"type" => "Create"}} = activity) do
-    object = Object.normalize(activity, false)
+    object = Object.get_by_ap_id(activity.data["object"])
 
     case object.data["type"] do
       "ChatMessage" -> "pleroma:chat_mention"
diff --git a/priv/repo/migrations/20200602125218_backfill_notification_types.exs b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
new file mode 100644
index 000000000..493c0280c
--- /dev/null
+++ b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
@@ -0,0 +1,10 @@
+defmodule Pleroma.Repo.Migrations.BackfillNotificationTypes do
+  use Ecto.Migration
+
+  def up do
+    Pleroma.Notification.fill_in_notification_types()
+  end
+
+  def down do
+  end
+end

From 2c6ebe709a9fb84bedb5d50c24715fd4532272f9 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 2 Jun 2020 15:14:52 +0200
Subject: [PATCH 123/283] Credo fixes

---
 lib/pleroma/notification.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index d89ee4645..0f33d282d 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -37,7 +37,7 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
-  def fill_in_notification_types() do
+  def fill_in_notification_types do
     query =
       from(n in __MODULE__,
         where: is_nil(n.type),

From aa22fce8f46cf2e7f871b3584fbfff7ac2ebe4c2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 12:30:12 +0200
Subject: [PATCH 124/283] ChatMessageReference: Introduce and switch in chat
 controller.

---
 lib/pleroma/chat.ex                           |  5 ++
 lib/pleroma/chat_message_reference.ex         | 80 +++++++++++++++++++
 lib/pleroma/web/activity_pub/side_effects.ex  |  9 ++-
 .../controllers/chat_controller.ex            | 60 ++++++++------
 ...view.ex => chat_message_reference_view.ex} | 21 +++--
 ...02150528_create_chat_message_reference.exs | 20 +++++
 test/web/activity_pub/side_effects_test.exs   | 13 ++-
 .../controllers/chat_controller_test.exs      | 22 +++--
 ...s => chat_message_reference_view_test.exs} | 20 +++--
 9 files changed, 205 insertions(+), 45 deletions(-)
 create mode 100644 lib/pleroma/chat_message_reference.ex
 rename lib/pleroma/web/pleroma_api/views/{chat_message_view.ex => chat_message_reference_view.ex} (67%)
 create mode 100644 priv/repo/migrations/20200602150528_create_chat_message_reference.exs
 rename test/web/pleroma_api/views/{chat_message_view_test.exs => chat_message_reference_view_test.exs} (67%)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 4c92a58c7..211b872f9 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -72,6 +72,11 @@ def creation_cng(struct, params) do
     |> unique_constraint(:user_id, name: :chats_user_id_recipient_index)
   end
 
+  def get_by_id(id) do
+    __MODULE__
+    |> Repo.get(id)
+  end
+
   def get(user_id, recipient) do
     __MODULE__
     |> Repo.get_by(user_id: user_id, recipient: recipient)
diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
new file mode 100644
index 000000000..e9ca3dfe8
--- /dev/null
+++ b/lib/pleroma/chat_message_reference.ex
@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ChatMessageReference do
+  @moduledoc """
+  A reference that builds a relation between an AP chat message that a user can see and whether it has been seen
+  by them, or should be displayed to them. Used to build the chat view that is presented to the user.
+  """
+
+  use Ecto.Schema
+
+  alias Pleroma.Chat
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+
+  schema "chat_message_references" do
+    belongs_to(:object, Object)
+    belongs_to(:chat, Chat)
+
+    field(:seen, :boolean, default: false)
+
+    timestamps()
+  end
+
+  def changeset(struct, params) do
+    struct
+    |> cast(params, [:object_id, :chat_id, :seen])
+    |> validate_required([:object_id, :chat_id, :seen])
+  end
+
+  def get_by_id(id) do
+    __MODULE__
+    |> Repo.get(id)
+    |> Repo.preload(:object)
+  end
+
+  def delete(cm_ref) do
+    cm_ref
+    |> Repo.delete()
+  end
+
+  def delete_for_object(%{id: object_id}) do
+    from(cr in __MODULE__,
+      where: cr.object_id == ^object_id
+    )
+    |> Repo.delete_all()
+  end
+
+  def for_chat_and_object(%{id: chat_id}, %{id: object_id}) do
+    __MODULE__
+    |> Repo.get_by(chat_id: chat_id, object_id: object_id)
+    |> Repo.preload(:object)
+  end
+
+  def for_chat_query(chat) do
+    from(cr in __MODULE__,
+      where: cr.chat_id == ^chat.id,
+      order_by: [desc: :id],
+      preload: [:object]
+    )
+  end
+
+  def create(chat, object, seen) do
+    params = %{
+      chat_id: chat.id,
+      object_id: object.id,
+      seen: seen
+    }
+
+    %__MODULE__{}
+    |> changeset(params)
+    |> Repo.insert()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a34bf6a05..cda52b00e 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -104,6 +105,8 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
               Object.decrease_replies_count(in_reply_to)
             end
 
+            ChatMessageReference.delete_for_object(deleted_object)
+
             ActivityPub.stream_out(object)
             ActivityPub.stream_out_participations(deleted_object, user)
             :ok
@@ -137,9 +140,11 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       |> Enum.each(fn [user, other_user] ->
         if user.local do
           if user.ap_id == actor.ap_id do
-            Chat.get_or_create(user.id, other_user.ap_id)
+            {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+            ChatMessageReference.create(chat, object, true)
           else
-            Chat.bump_or_create(user.id, other_user.ap_id)
+            {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+            ChatMessageReference.create(chat, object, false)
           end
         end
       end)
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 210c8ec4a..c54681054 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -6,14 +6,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
   alias Pleroma.Web.PleromaAPI.ChatView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   import Ecto.Query
   import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]
@@ -35,28 +36,38 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation
 
-  def delete_message(%{assigns: %{user: %{ap_id: actor} = user}} = conn, %{
-        message_id: id
+  def delete_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
+        message_id: message_id,
+        id: chat_id
       }) do
-    with %Object{
-           data: %{
-             "actor" => ^actor,
-             "id" => object,
-             "to" => [recipient],
-             "type" => "ChatMessage"
-           }
-         } = message <- Object.get_by_id(id),
-         %Chat{} = chat <- Chat.get(user.id, recipient),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object),
-         {:ok, _delete} <- CommonAPI.delete(activity.id, user) do
+    with %ChatMessageReference{} = cm_ref <-
+           ChatMessageReference.get_by_id(message_id),
+         ^chat_id <- cm_ref.chat_id |> to_string(),
+         %Chat{user_id: ^user_id} <- Chat.get_by_id(chat_id),
+         {:ok, _} <- remove_or_delete(cm_ref, user) do
       conn
-      |> put_view(ChatMessageView)
-      |> render("show.json", for: user, object: message, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("show.json", chat_message_reference: cm_ref)
     else
-      _e -> {:error, :could_not_delete}
+      _e ->
+        {:error, :could_not_delete}
     end
   end
 
+  defp remove_or_delete(
+         %{object: %{data: %{"actor" => actor, "id" => id}}},
+         %{ap_id: actor} = user
+       ) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+      CommonAPI.delete(activity.id, user)
+    end
+  end
+
+  defp remove_or_delete(cm_ref, _) do
+    cm_ref
+    |> ChatMessageReference.delete()
+  end
+
   def post_chat_message(
         %{body_params: params, assigns: %{user: %{id: user_id} = user}} = conn,
         %{
@@ -69,10 +80,11 @@ def post_chat_message(
            CommonAPI.post_chat_message(user, recipient, params[:content],
              media_id: params[:media_id]
            ),
-         message <- Object.normalize(activity) do
+         message <- Object.normalize(activity, false),
+         cm_ref <- ChatMessageReference.for_chat_and_object(chat, message) do
       conn
-      |> put_view(ChatMessageView)
-      |> render("show.json", for: user, object: message, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("show.json", for: user, chat_message_reference: cm_ref)
     end
   end
 
@@ -87,14 +99,14 @@ def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
 
   def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
-      messages =
+      cm_refs =
         chat
-        |> Chat.messages_for_chat_query()
+        |> ChatMessageReference.for_chat_query()
         |> Pagination.fetch_paginated(params |> stringify_keys())
 
       conn
-      |> put_view(ChatMessageView)
-      |> render("index.json", for: user, objects: messages, chat: chat)
+      |> put_view(ChatMessageReferenceView)
+      |> render("index.json", for: user, chat_message_references: cm_refs)
     else
       _ ->
         conn
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
similarity index 67%
rename from lib/pleroma/web/pleroma_api/views/chat_message_view.ex
rename to lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
index b088a8734..ff170e162 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
@@ -2,10 +2,9 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
+defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Chat
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
@@ -13,8 +12,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageView do
   def render(
         "show.json",
         %{
-          object: %{id: id, data: %{"type" => "ChatMessage"} = chat_message},
-          chat: %Chat{id: chat_id}
+          chat_message_reference: %{
+            id: id,
+            object: %{data: chat_message},
+            chat_id: chat_id,
+            seen: seen
+          }
         }
       ) do
     %{
@@ -26,11 +29,17 @@ def render(
       emojis: StatusView.build_emojis(chat_message["emoji"]),
       attachment:
         chat_message["attachment"] &&
-          StatusView.render("attachment.json", attachment: chat_message["attachment"])
+          StatusView.render("attachment.json", attachment: chat_message["attachment"]),
+      seen: seen
     }
   end
 
   def render("index.json", opts) do
-    render_many(opts[:objects], __MODULE__, "show.json", Map.put(opts, :as, :object))
+    render_many(
+      opts[:chat_message_references],
+      __MODULE__,
+      "show.json",
+      Map.put(opts, :as, :chat_message_reference)
+    )
   end
 end
diff --git a/priv/repo/migrations/20200602150528_create_chat_message_reference.exs b/priv/repo/migrations/20200602150528_create_chat_message_reference.exs
new file mode 100644
index 000000000..6f9148b7c
--- /dev/null
+++ b/priv/repo/migrations/20200602150528_create_chat_message_reference.exs
@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.CreateChatMessageReference do
+  use Ecto.Migration
+
+  def change do
+    create table(:chat_message_references, primary_key: false) do
+      add(:id, :uuid, primary_key: true)
+      add(:chat_id, references(:chats, on_delete: :delete_all), null: false)
+      add(:object_id, references(:objects, on_delete: :delete_all), null: false)
+      add(:seen, :boolean, default: false, null: false)
+
+      timestamps()
+    end
+
+    create(index(:chat_message_references, [:chat_id, "id desc"]))
+  end
+end
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 210ba6ef0..ff6b3ac15 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -330,7 +331,7 @@ test "it streams the created ChatMessage" do
       end
     end
 
-    test "it creates a Chat for the local users and bumps the unread count, except for the author" do
+    test "it creates a Chat and ChatMessageReferences for the local users and bumps the unread count, except for the author" do
       author = insert(:user, local: true)
       recipient = insert(:user, local: true)
 
@@ -347,8 +348,18 @@ test "it creates a Chat for the local users and bumps the unread count, except f
       chat = Chat.get(author.id, recipient.ap_id)
       assert chat.unread == 0
 
+      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+
+      assert cm_ref.object.data["content"] == "hey"
+      assert cm_ref.seen == true
+
       chat = Chat.get(recipient.id, author.ap_id)
       assert chat.unread == 1
+
+      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+
+      assert cm_ref.object.data["content"] == "hey"
+      assert cm_ref.seen == false
     end
 
     test "it creates a Chat for the local users and bumps the unread count" do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index d79aa3148..bd4024c09 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,6 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -95,7 +96,7 @@ test "it works with an attachment", %{conn: conn, user: user} do
   describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do
     setup do: oauth_access(["write:statuses"])
 
-    test "it deletes a message for the author of the message", %{conn: conn, user: user} do
+    test "it deletes a message from the chat", %{conn: conn, user: user} do
       recipient = insert(:user)
 
       {:ok, message} =
@@ -107,23 +108,32 @@ test "it deletes a message for the author of the message", %{conn: conn, user: u
 
       chat = Chat.get(user.id, recipient.ap_id)
 
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
+      # Deleting your own message removes the message and the reference
       result =
         conn
         |> put_req_header("content-type", "application/json")
-        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{cm_ref.id}")
         |> json_response_and_validate_schema(200)
 
-      assert result["id"] == to_string(object.id)
+      assert result["id"] == cm_ref.id
+      refute ChatMessageReference.get_by_id(cm_ref.id)
+      assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id)
 
+      # Deleting other people's messages just removes the reference
       object = Object.normalize(other_message, false)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
       result =
         conn
         |> put_req_header("content-type", "application/json")
-        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{object.id}")
-        |> json_response(400)
+        |> delete("/api/v1/pleroma/chats/#{chat.id}/messages/#{cm_ref.id}")
+        |> json_response_and_validate_schema(200)
 
-      assert result == %{"error" => "could_not_delete"}
+      assert result["id"] == cm_ref.id
+      refute ChatMessageReference.get_by_id(cm_ref.id)
+      assert Object.get_by_id(object.id)
     end
   end
 
diff --git a/test/web/pleroma_api/views/chat_message_view_test.exs b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
similarity index 67%
rename from test/web/pleroma_api/views/chat_message_view_test.exs
rename to test/web/pleroma_api/views/chat_message_reference_view_test.exs
index d7a2d10a5..00024d52c 100644
--- a/test/web/pleroma_api/views/chat_message_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -2,14 +2,15 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageViewTest do
+defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   import Pleroma.Factory
 
@@ -30,25 +31,32 @@ test "it displays a chat message" do
 
     object = Object.normalize(activity)
 
-    chat_message = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-    assert chat_message[:id] == object.id |> to_string()
+    chat_message = ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+
+    assert chat_message[:id] == cm_ref.id
     assert chat_message[:content] == "kippis :firefox:"
     assert chat_message[:account_id] == user.id
     assert chat_message[:chat_id]
     assert chat_message[:created_at]
+    assert chat_message[:seen] == true
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
 
     {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk", media_id: upload.id)
 
     object = Object.normalize(activity)
 
-    chat_message_two = ChatMessageView.render("show.json", object: object, for: user, chat: chat)
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-    assert chat_message_two[:id] == object.id |> to_string()
+    chat_message_two =
+      ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+
+    assert chat_message_two[:id] == cm_ref.id
     assert chat_message_two[:content] == "gkgkgk"
     assert chat_message_two[:account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
     assert chat_message_two[:attachment]
+    assert chat_message_two[:seen] == false
   end
 end

From f3ccd50a33c9eec3661bf2116fe38542f04986aa Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 12:49:53 +0200
Subject: [PATCH 125/283] ChatMessageReferences: Adjust views

---
 lib/pleroma/chat_message_reference.ex                   | 7 +++++++
 lib/pleroma/web/mastodon_api/views/notification_view.ex | 8 +++++---
 lib/pleroma/web/pleroma_api/views/chat_view.ex          | 8 +++++---
 lib/pleroma/web/views/streamer_view.ex                  | 8 +++++++-
 test/web/mastodon_api/views/notification_view_test.exs  | 7 +++++--
 test/web/pleroma_api/views/chat_view_test.exs           | 7 +++++--
 6 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
index e9ca3dfe8..6808d1365 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat_message_reference.ex
@@ -66,6 +66,13 @@ def for_chat_query(chat) do
     )
   end
 
+  def last_message_for_chat(chat) do
+    chat
+    |> for_chat_query()
+    |> limit(1)
+    |> Repo.one()
+  end
+
   def create(chat, object, seen) do
     params = %{
       chat_id: chat.id,
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index af15bba48..2ae82eb2d 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   use Pleroma.Web, :view
 
   alias Pleroma.Activity
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
@@ -14,7 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   @parent_types ~w{Like Announce EmojiReact}
 
@@ -138,8 +139,9 @@ defp put_chat_message(response, activity, reading_user, opts) do
     object = Object.normalize(activity)
     author = User.get_cached_by_ap_id(object.data["actor"])
     chat = Pleroma.Chat.get(reading_user.id, author.ap_id)
-    render_opts = Map.merge(opts, %{object: object, for: reading_user, chat: chat})
-    chat_message_render = ChatMessageView.render("show.json", render_opts)
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+    render_opts = Map.merge(opts, %{for: reading_user, chat_message_reference: cm_ref})
+    chat_message_render = ChatMessageReferenceView.render("show.json", render_opts)
 
     Map.put(response, :chat_message, chat_message_render)
   end
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 223b64987..331c1d282 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -6,22 +6,24 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   use Pleroma.Web, :view
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
 
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
 
-    last_message = opts[:message] || Chat.last_message_for_chat(chat)
+    last_message = opts[:last_message] || ChatMessageReference.last_message_for_chat(chat)
 
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
       unread: chat.unread,
       last_message:
-        last_message && ChatMessageView.render("show.json", chat: chat, object: last_message),
+        last_message &&
+          ChatMessageReferenceView.render("show.json", chat_message_reference: last_message),
       updated_at: Utils.to_masto_date(chat.updated_at)
     }
   end
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 5e953d770..616e0c4f2 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.StreamerView do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.User
@@ -15,10 +16,15 @@ defmodule Pleroma.Web.StreamerView do
   def render("chat_update.json", object, user, recipients) do
     chat = Chat.get(user.id, hd(recipients -- [user.ap_id]))
 
+    # Explicitly giving the cmr for the object here, so we don't accidentally
+    # send a later 'last_message' that was inserted between inserting this and
+    # streaming it out
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
     representation =
       Pleroma.Web.PleromaAPI.ChatView.render(
         "show.json",
-        %{message: object, chat: chat}
+        %{last_message: cm_ref, chat: chat}
       )
 
     %{
diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs
index 384fe7253..c5691341a 100644
--- a/test/web/mastodon_api/views/notification_view_test.exs
+++ b/test/web/mastodon_api/views/notification_view_test.exs
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -16,7 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
   import Pleroma.Factory
 
   defp test_notifications_rendering(notifications, user, expected_result) do
@@ -44,13 +45,15 @@ test "ChatMessage notification" do
     object = Object.normalize(activity)
     chat = Chat.get(recipient.id, user.ap_id)
 
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
     expected = %{
       id: to_string(notification.id),
       pleroma: %{is_seen: false},
       type: "pleroma:chat_mention",
       account: AccountView.render("show.json", %{user: user, for: recipient}),
       chat_message:
-        ChatMessageView.render("show.json", %{object: object, for: recipient, chat: chat}),
+        ChatMessageReferenceView.render("show.json", %{chat_message_reference: cm_ref}),
       created_at: Utils.to_masto_date(notification.inserted_at)
     }
 
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index 6062a0cfe..f3bd12616 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -6,11 +6,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.PleromaAPI.ChatMessageView
+  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
   alias Pleroma.Web.PleromaAPI.ChatView
 
   import Pleroma.Factory
@@ -39,7 +40,9 @@ test "it represents a chat" do
 
     represented_chat = ChatView.render("show.json", chat: chat)
 
+    cm_ref = ChatMessageReference.for_chat_and_object(chat, chat_message)
+
     assert represented_chat[:last_message] ==
-             ChatMessageView.render("show.json", chat: chat, object: chat_message)
+             ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
   end
 end

From 2591745fc2417771f96340ed3f36177c0da194c3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 12:56:39 +0200
Subject: [PATCH 126/283] ChatMessageReferences: Move tests

---
 lib/pleroma/chat.ex                  | 34 ----------------------------
 test/chat_message_reference_test.exs | 29 ++++++++++++++++++++++++
 test/chat_test.exs                   | 16 -------------
 3 files changed, 29 insertions(+), 50 deletions(-)
 create mode 100644 test/chat_message_reference_test.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 211b872f9..65938c7a4 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -6,9 +6,7 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query
 
-  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -26,38 +24,6 @@ defmodule Pleroma.Chat do
     timestamps()
   end
 
-  def last_message_for_chat(chat) do
-    messages_for_chat_query(chat)
-    |> order_by(desc: :id)
-    |> limit(1)
-    |> Repo.one()
-  end
-
-  def messages_for_chat_query(chat) do
-    chat =
-      chat
-      |> Repo.preload(:user)
-
-    from(o in Object,
-      where: fragment("?->>'type' = ?", o.data, "ChatMessage"),
-      where:
-        fragment(
-          """
-          (?->>'actor' = ? and ?->'to' = ?) 
-          OR (?->>'actor' = ? and ?->'to' = ?) 
-          """,
-          o.data,
-          ^chat.user.ap_id,
-          o.data,
-          ^[chat.recipient],
-          o.data,
-          ^chat.recipient,
-          o.data,
-          ^[chat.user.ap_id]
-        )
-    )
-  end
-
   def creation_cng(struct, params) do
     struct
     |> cast(params, [:user_id, :recipient, :unread])
diff --git a/test/chat_message_reference_test.exs b/test/chat_message_reference_test.exs
new file mode 100644
index 000000000..963a0e225
--- /dev/null
+++ b/test/chat_message_reference_test.exs
@@ -0,0 +1,29 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ChatMessageReferencTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  describe "messages" do
+    test "it returns the last message in a chat" do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, _message_1} = CommonAPI.post_chat_message(user, recipient, "hey")
+      {:ok, _message_2} = CommonAPI.post_chat_message(recipient, user, "ho")
+
+      {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
+
+      message = ChatMessageReference.last_message_for_chat(chat)
+
+      assert message.object.data["content"] == "ho"
+    end
+  end
+end
diff --git a/test/chat_test.exs b/test/chat_test.exs
index dfcb6422e..42e01fe27 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -10,22 +10,6 @@ defmodule Pleroma.ChatTest do
 
   import Pleroma.Factory
 
-  describe "messages" do
-    test "it returns the last message in a chat" do
-      user = insert(:user)
-      recipient = insert(:user)
-
-      {:ok, _message_1} = CommonAPI.post_chat_message(user, recipient, "hey")
-      {:ok, _message_2} = CommonAPI.post_chat_message(recipient, user, "ho")
-
-      {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
-
-      message = Chat.last_message_for_chat(chat)
-
-      assert message.data["content"] == "ho"
-    end
-  end
-
   describe "creation and getting" do
     test "it only works if the recipient is a valid user (for now)" do
       user = insert(:user)

From 6413e06a861bd383196c79d7754a67d96cd5e2a4 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 13:13:44 +0200
Subject: [PATCH 127/283] Migrations: Add unique index to
 ChatMessageReferences.

---
 ...add_unique_index_to_chat_message_references.exs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs

diff --git a/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
new file mode 100644
index 000000000..1101be94f
--- /dev/null
+++ b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.BackfillChatMessageReferences do
+  use Ecto.Migration
+
+  alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
+  import Ecto.Query
+
+  def change do
+    create(unique_index(:chat_message_references, [:object_id, :chat_id]))
+  end
+end

From 73127cff750736c5ebe15606db2f928a8924499a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 13:17:29 +0200
Subject: [PATCH 128/283] Credo fixes.

---
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index c54681054..f22f33de9 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -13,8 +13,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.ChatView
 
   import Ecto.Query
   import Pleroma.Web.ActivityPub.ObjectValidator, only: [stringify_keys: 1]

From 8edead7c1dc33457dc30b301b544d71482ef0f28 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 13:19:38 +0200
Subject: [PATCH 129/283] Migration: Remove superfluous imports

---
 ...3105113_add_unique_index_to_chat_message_references.exs | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
index 1101be94f..623ac6c85 100644
--- a/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
+++ b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
@@ -1,13 +1,6 @@
 defmodule Pleroma.Repo.Migrations.BackfillChatMessageReferences do
   use Ecto.Migration
 
-  alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
-  alias Pleroma.Object
-  alias Pleroma.Repo
-
-  import Ecto.Query
-
   def change do
     create(unique_index(:chat_message_references, [:object_id, :chat_id]))
   end

From 7f5c5b11a5baeddec36ccc01b4954ac8aa9f8590 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 14:26:50 +0200
Subject: [PATCH 130/283] Chats: Remove `unread` from the db, calculate from
 unseen messages.

---
 lib/pleroma/chat.ex                              | 13 +++----------
 lib/pleroma/chat_message_reference.ex            | 16 ++++++++++++++++
 lib/pleroma/web/activity_pub/side_effects.ex     |  9 ++-------
 .../pleroma_api/controllers/chat_controller.ex   |  2 +-
 lib/pleroma/web/pleroma_api/views/chat_view.ex   |  2 +-
 .../20200603120448_remove_unread_from_chats.exs  |  9 +++++++++
 test/chat_test.exs                               |  6 +-----
 test/web/activity_pub/side_effects_test.exs      |  2 --
 .../controllers/chat_controller_test.exs         | 11 +++++++----
 9 files changed, 40 insertions(+), 30 deletions(-)
 create mode 100644 priv/repo/migrations/20200603120448_remove_unread_from_chats.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 65938c7a4..5aefddc5e 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -19,14 +19,13 @@ defmodule Pleroma.Chat do
   schema "chats" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     field(:recipient, :string)
-    field(:unread, :integer, default: 0, read_after_writes: true)
 
     timestamps()
   end
 
   def creation_cng(struct, params) do
     struct
-    |> cast(params, [:user_id, :recipient, :unread])
+    |> cast(params, [:user_id, :recipient])
     |> validate_change(:recipient, fn
       :recipient, recipient ->
         case User.get_cached_by_ap_id(recipient) do
@@ -61,16 +60,10 @@ def get_or_create(user_id, recipient) do
 
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
-    |> creation_cng(%{user_id: user_id, recipient: recipient, unread: 1})
+    |> creation_cng(%{user_id: user_id, recipient: recipient})
     |> Repo.insert(
-      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()], inc: [unread: 1]],
+      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
       conflict_target: [:user_id, :recipient]
     )
   end
-
-  def mark_as_read(chat) do
-    chat
-    |> change(%{unread: 0})
-    |> Repo.update()
-  end
 end
diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
index 6808d1365..ad174b294 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat_message_reference.ex
@@ -84,4 +84,20 @@ def create(chat, object, seen) do
     |> changeset(params)
     |> Repo.insert()
   end
+
+  def unread_count_for_chat(chat) do
+    chat
+    |> for_chat_query()
+    |> where([cmr], cmr.seen == false)
+    |> Repo.aggregate(:count)
+  end
+
+  def set_all_seen_for_chat(chat) do
+    chat
+    |> for_chat_query()
+    |> exclude(:order_by)
+    |> exclude(:preload)
+    |> where([cmr], cmr.seen == false)
+    |> Repo.update_all(set: [seen: true])
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index cda52b00e..884d399d0 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -139,13 +139,8 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       [[actor, recipient], [recipient, actor]]
       |> Enum.each(fn [user, other_user] ->
         if user.local do
-          if user.ap_id == actor.ap_id do
-            {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
-            ChatMessageReference.create(chat, object, true)
-          else
-            {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-            ChatMessageReference.create(chat, object, false)
-          end
+          {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+          ChatMessageReference.create(chat, object, user.ap_id == actor.ap_id)
         end
       end)
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index f22f33de9..29922da99 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -90,7 +90,7 @@ def post_chat_message(
 
   def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
-         {:ok, chat} <- Chat.mark_as_read(chat) do
+         {_n, _} <- ChatMessageReference.set_all_seen_for_chat(chat) do
       conn
       |> put_view(ChatView)
       |> render("show.json", chat: chat)
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 331c1d282..c903a71fd 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -20,7 +20,7 @@ def render("show.json", %{chat: %Chat{} = chat} = opts) do
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
-      unread: chat.unread,
+      unread: ChatMessageReference.unread_count_for_chat(chat),
       last_message:
         last_message &&
           ChatMessageReferenceView.render("show.json", chat_message_reference: last_message),
diff --git a/priv/repo/migrations/20200603120448_remove_unread_from_chats.exs b/priv/repo/migrations/20200603120448_remove_unread_from_chats.exs
new file mode 100644
index 000000000..6322137d5
--- /dev/null
+++ b/priv/repo/migrations/20200603120448_remove_unread_from_chats.exs
@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.RemoveUnreadFromChats do
+  use Ecto.Migration
+
+  def change do
+    alter table(:chats) do
+      remove(:unread, :integer, default: 0)
+    end
+  end
+end
diff --git a/test/chat_test.exs b/test/chat_test.exs
index 42e01fe27..332f2180a 100644
--- a/test/chat_test.exs
+++ b/test/chat_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.ChatTest do
   use Pleroma.DataCase, async: true
 
   alias Pleroma.Chat
-  alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
@@ -35,7 +34,6 @@ test "it returns and bumps a chat for a user and recipient if it already exists"
       {:ok, chat_two} = Chat.bump_or_create(user.id, other_user.ap_id)
 
       assert chat.id == chat_two.id
-      assert chat_two.unread == 2
     end
 
     test "it returns a chat for a user and recipient if it already exists" do
@@ -48,15 +46,13 @@ test "it returns a chat for a user and recipient if it already exists" do
       assert chat.id == chat_two.id
     end
 
-    test "a returning chat will have an updated `update_at` field and an incremented unread count" do
+    test "a returning chat will have an updated `update_at` field" do
       user = insert(:user)
       other_user = insert(:user)
 
       {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-      assert chat.unread == 1
       :timer.sleep(1500)
       {:ok, chat_two} = Chat.bump_or_create(user.id, other_user.ap_id)
-      assert chat_two.unread == 2
 
       assert chat.id == chat_two.id
       assert chat.updated_at != chat_two.updated_at
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index ff6b3ac15..f2fa062b4 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -346,7 +346,6 @@ test "it creates a Chat and ChatMessageReferences for the local users and bumps
         SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
       chat = Chat.get(author.id, recipient.ap_id)
-      assert chat.unread == 0
 
       [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
 
@@ -354,7 +353,6 @@ test "it creates a Chat and ChatMessageReferences for the local users and bumps
       assert cm_ref.seen == true
 
       chat = Chat.get(recipient.id, author.ap_id)
-      assert chat.unread == 1
 
       [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index bd4024c09..e62b71799 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -19,9 +19,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
     test "it marks all messages in a chat as read", %{conn: conn, user: user} do
       other_user = insert(:user)
 
-      {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+      {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      object = Object.normalize(create, false)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert chat.unread == 1
+      assert cm_ref.seen == false
 
       result =
         conn
@@ -30,9 +33,9 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
 
       assert result["unread"] == 0
 
-      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert chat.unread == 0
+      assert cm_ref.seen == true
     end
   end
 

From 1e9efcf7c3de2aa4d57d4292dfa5843761bff111 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 14:27:54 +0200
Subject: [PATCH 131/283] Migrations: Fix migration module name

---
 ...200603105113_add_unique_index_to_chat_message_references.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
index 623ac6c85..fdf85132e 100644
--- a/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
+++ b/priv/repo/migrations/20200603105113_add_unique_index_to_chat_message_references.exs
@@ -1,4 +1,4 @@
-defmodule Pleroma.Repo.Migrations.BackfillChatMessageReferences do
+defmodule Pleroma.Repo.Migrations.AddUniqueIndexToChatMessageReferences do
   use Ecto.Migration
 
   def change do

From 7b79871e9721dca9b134598c182df890b909047c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 14:32:19 +0200
Subject: [PATCH 132/283] Migrations: Add chat_id, seen index to
 ChatMessageReferences

This ensures fast count of unseen messages
---
 ...732_add_seen_index_to_chat_message_references.exs | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 priv/repo/migrations/20200603122732_add_seen_index_to_chat_message_references.exs

diff --git a/priv/repo/migrations/20200603122732_add_seen_index_to_chat_message_references.exs b/priv/repo/migrations/20200603122732_add_seen_index_to_chat_message_references.exs
new file mode 100644
index 000000000..a5065d612
--- /dev/null
+++ b/priv/repo/migrations/20200603122732_add_seen_index_to_chat_message_references.exs
@@ -0,0 +1,12 @@
+defmodule Pleroma.Repo.Migrations.AddSeenIndexToChatMessageReferences do
+  use Ecto.Migration
+
+  def change do
+    create(
+      index(:chat_message_references, [:chat_id],
+        where: "seen = false",
+        name: "unseen_messages_count_index"
+      )
+    )
+  end
+end

From 903955b189561d3a95d5955feda723999078b894 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 14:40:44 +0200
Subject: [PATCH 133/283] FollowingRelationship: Remove meaningless change

---
 lib/pleroma/following_relationship.ex | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex
index 0343a20d4..3a3082e72 100644
--- a/lib/pleroma/following_relationship.ex
+++ b/lib/pleroma/following_relationship.ex
@@ -62,13 +62,10 @@ def update(%User{} = follower, %User{} = following, state) do
         follow(follower, following, state)
 
       following_relationship ->
-        {:ok, relationship} =
-          following_relationship
-          |> cast(%{state: state}, [:state])
-          |> validate_required([:state])
-          |> Repo.update()
-
-        {:ok, relationship}
+        following_relationship
+        |> cast(%{state: state}, [:state])
+        |> validate_required([:state])
+        |> Repo.update()
     end
   end
 

From fb4ae9c720054372c1f0e41e3227fb8ad24e6c2d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 16:45:04 +0200
Subject: [PATCH 134/283] Streamer, SideEffects: Stream out
 ChatMessageReferences

Saves us a few calles to fetch things from the DB that we already
have.
---
 lib/pleroma/web/activity_pub/side_effects.ex |  8 +++-
 lib/pleroma/web/streamer/streamer.ex         | 21 +++------
 lib/pleroma/web/views/streamer_view.ex       | 46 +++++++++-----------
 test/web/activity_pub/side_effects_test.exs  |  5 +--
 test/web/streamer/streamer_test.exs          | 28 +++++++++---
 5 files changed, 58 insertions(+), 50 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 884d399d0..0c5709356 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -140,11 +140,15 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       |> Enum.each(fn [user, other_user] ->
         if user.local do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-          ChatMessageReference.create(chat, object, user.ap_id == actor.ap_id)
+          {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id == actor.ap_id)
+
+          Streamer.stream(
+            ["user", "user:pleroma_chat"],
+            {user, %{cm_ref | chat: chat, object: object}}
+          )
         end
       end)
 
-      Streamer.stream(["user", "user:pleroma_chat"], object)
       {:ok, object, meta}
     end
   end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 2201cbfef..5e37e2cf2 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -6,11 +6,11 @@ defmodule Pleroma.Web.Streamer do
   require Logger
 
   alias Pleroma.Activity
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
-  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
@@ -201,22 +201,15 @@ defp do_stream(topic, %Notification{} = item)
     end)
   end
 
-  defp do_stream(topic, %{data: %{"type" => "ChatMessage"}} = object)
+  defp do_stream(topic, {user, %ChatMessageReference{} = cm_ref})
        when topic in ["user", "user:pleroma_chat"] do
-    recipients = [object.data["actor"] | object.data["to"]]
+    topic = "#{topic}:#{user.id}"
 
-    topics =
-      %{ap_id: recipients, local: true}
-      |> Pleroma.User.Query.build()
-      |> Repo.all()
-      |> Enum.map(fn %{id: id} = user -> {user, "#{topic}:#{id}"} end)
+    text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
 
-    Enum.each(topics, fn {user, topic} ->
-      Registry.dispatch(@registry, topic, fn list ->
-        Enum.each(list, fn {pid, _auth} ->
-          text = StreamerView.render("chat_update.json", object, user, recipients)
-          send(pid, {:text, text})
-        end)
+    Registry.dispatch(@registry, topic, fn list ->
+      Enum.each(list, fn {pid, _auth} ->
+        send(pid, {:text, text})
       end)
     end)
   end
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index 616e0c4f2..a6efd0109 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -6,36 +6,11 @@ defmodule Pleroma.Web.StreamerView do
   use Pleroma.Web, :view
 
   alias Pleroma.Activity
-  alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.User
   alias Pleroma.Web.MastodonAPI.NotificationView
 
-  def render("chat_update.json", object, user, recipients) do
-    chat = Chat.get(user.id, hd(recipients -- [user.ap_id]))
-
-    # Explicitly giving the cmr for the object here, so we don't accidentally
-    # send a later 'last_message' that was inserted between inserting this and
-    # streaming it out
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
-
-    representation =
-      Pleroma.Web.PleromaAPI.ChatView.render(
-        "show.json",
-        %{last_message: cm_ref, chat: chat}
-      )
-
-    %{
-      event: "pleroma:chat_update",
-      payload:
-        representation
-        |> Jason.encode!()
-    }
-    |> Jason.encode!()
-  end
-
   def render("update.json", %Activity{} = activity, %User{} = user) do
     %{
       event: "update",
@@ -76,6 +51,27 @@ def render("update.json", %Activity{} = activity) do
     |> Jason.encode!()
   end
 
+  def render("chat_update.json", %{chat_message_reference: cm_ref}) do
+    # Explicitly giving the cmr for the object here, so we don't accidentally
+    # send a later 'last_message' that was inserted between inserting this and
+    # streaming it out
+    Logger.debug("Trying to stream out #{inspect(cm_ref)}")
+
+    representation =
+      Pleroma.Web.PleromaAPI.ChatView.render(
+        "show.json",
+        %{last_message: cm_ref, chat: cm_ref.chat}
+      )
+
+    %{
+      event: "pleroma:chat_update",
+      payload:
+        representation
+        |> Jason.encode!()
+    }
+    |> Jason.encode!()
+  end
+
   def render("conversation.json", %Participation{} = participation) do
     %{
       event: "conversation",
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index f2fa062b4..92c266d84 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -325,9 +325,8 @@ test "it streams the created ChatMessage" do
         {:ok, _create_activity, _meta} =
           SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
-        object = Object.normalize(create_activity, false)
-
-        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], object))
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], {author, :_}))
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], {recipient, :_}))
       end
     end
 
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index bcb05a02d..893ae5449 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -7,6 +7,8 @@ defmodule Pleroma.Web.StreamerTest do
 
   import Pleroma.Factory
 
+  alias Pleroma.Chat
+  alias Pleroma.ChatMessageReference
   alias Pleroma.Conversation.Participation
   alias Pleroma.List
   alias Pleroma.Object
@@ -150,22 +152,36 @@ test "it sends notify to in the 'user:notification' stream", %{user: user, notif
     test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} do
       other_user = insert(:user)
 
-      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
       object = Object.normalize(create_activity, false)
+      chat = Chat.get(user.id, other_user.ap_id)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = %{cm_ref | chat: chat, object: object}
+
       Streamer.get_topic_and_add_socket("user:pleroma_chat", user)
-      Streamer.stream("user:pleroma_chat", object)
-      text = StreamerView.render("chat_update.json", object, user, [user.ap_id, other_user.ap_id])
+      Streamer.stream("user:pleroma_chat", {user, cm_ref})
+
+      text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
+
+      assert text =~ "hey cirno"
       assert_receive {:text, ^text}
     end
 
     test "it sends chat messages to the 'user' stream", %{user: user} do
       other_user = insert(:user)
 
-      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
+      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
       object = Object.normalize(create_activity, false)
+      chat = Chat.get(user.id, other_user.ap_id)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = %{cm_ref | chat: chat, object: object}
+
       Streamer.get_topic_and_add_socket("user", user)
-      Streamer.stream("user", object)
-      text = StreamerView.render("chat_update.json", object, user, [user.ap_id, other_user.ap_id])
+      Streamer.stream("user", {user, cm_ref})
+
+      text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
+
+      assert text =~ "hey cirno"
       assert_receive {:text, ^text}
     end
 

From c020fd435216012f08812efdb9ee0c05352cec10 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 18:58:58 +0200
Subject: [PATCH 135/283] ChatMessageReferenceView: Return read status as
 `unread`.

---
 .../web/pleroma_api/views/chat_message_reference_view.ex      | 2 +-
 .../pleroma_api/views/chat_message_reference_view_test.exs    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
index ff170e162..f9405aec5 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
@@ -30,7 +30,7 @@ def render(
       attachment:
         chat_message["attachment"] &&
           StatusView.render("attachment.json", attachment: chat_message["attachment"]),
-      seen: seen
+      unread: !seen
     }
   end
 
diff --git a/test/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
index 00024d52c..b53bd3490 100644
--- a/test/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -40,7 +40,7 @@ test "it displays a chat message" do
     assert chat_message[:account_id] == user.id
     assert chat_message[:chat_id]
     assert chat_message[:created_at]
-    assert chat_message[:seen] == true
+    assert chat_message[:unread] == false
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
 
     {:ok, activity} = CommonAPI.post_chat_message(recipient, user, "gkgkgk", media_id: upload.id)
@@ -57,6 +57,6 @@ test "it displays a chat message" do
     assert chat_message_two[:account_id] == recipient.id
     assert chat_message_two[:chat_id] == chat_message[:chat_id]
     assert chat_message_two[:attachment]
-    assert chat_message_two[:seen] == false
+    assert chat_message_two[:unread] == true
   end
 end

From b3407344d3acafa4a1271289d985632c058e7a6e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 19:21:23 +0200
Subject: [PATCH 136/283] ChatController: Add function to mark single message
 as read.

---
 lib/pleroma/chat_message_reference.ex         |  6 ++++
 .../web/api_spec/operations/chat_operation.ex | 31 +++++++++++++++++--
 .../controllers/chat_controller.ex            | 23 +++++++++++++-
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/chat_controller_test.exs      | 28 +++++++++++++++++
 5 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
index ad174b294..9b00443f5 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat_message_reference.ex
@@ -92,6 +92,12 @@ def unread_count_for_chat(chat) do
     |> Repo.aggregate(:count)
   end
 
+  def mark_as_read(cm_ref) do
+    cm_ref
+    |> changeset(%{seen: true})
+    |> Repo.update()
+  end
+
   def set_all_seen_for_chat(chat) do
     chat
     |> for_chat_query()
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index a1c5db5dc..6ad325113 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -39,6 +39,31 @@ def mark_as_read_operation do
     }
   end
 
+  def mark_message_as_read_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Mark one message in the chat as read",
+      operationId: "ChatController.mark_message_as_read",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:message_id, :path, :string, "The ID of the message")
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The read ChatMessage",
+            "application/json",
+            ChatMessage
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write"]
+        }
+      ]
+    }
+  end
+
   def show_operation do
     %Operation{
       tags: ["chat"],
@@ -274,7 +299,8 @@ def chat_messages_response do
           "content" => "Check this out :firefox:",
           "id" => "13",
           "chat_id" => "1",
-          "actor_id" => "someflakeid"
+          "actor_id" => "someflakeid",
+          "unread" => false
         },
         %{
           "actor_id" => "someflakeid",
@@ -282,7 +308,8 @@ def chat_messages_response do
           "id" => "12",
           "chat_id" => "1",
           "emojis" => [],
-          "created_at" => "2020-04-21T15:06:45.000Z"
+          "created_at" => "2020-04-21T15:06:45.000Z",
+          "unread" => false
         }
       ]
     }
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 29922da99..01d47045d 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -24,7 +24,13 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:statuses"]}
-    when action in [:post_chat_message, :create, :mark_as_read, :delete_message]
+    when action in [
+           :post_chat_message,
+           :create,
+           :mark_as_read,
+           :mark_message_as_read,
+           :delete_message
+         ]
   )
 
   plug(
@@ -88,6 +94,21 @@ def post_chat_message(
     end
   end
 
+  def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
+        id: chat_id,
+        message_id: message_id
+      }) do
+    with %ChatMessageReference{} = cm_ref <-
+           ChatMessageReference.get_by_id(message_id),
+         ^chat_id <- cm_ref.chat_id |> to_string(),
+         %Chat{user_id: ^user_id} <- Chat.get_by_id(chat_id),
+         {:ok, cm_ref} <- ChatMessageReference.mark_as_read(cm_ref) do
+      conn
+      |> put_view(ChatMessageReferenceView)
+      |> render("show.json", for: user, chat_message_reference: cm_ref)
+    end
+  end
+
   def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
          {_n, _} <- ChatMessageReference.set_all_seen_for_chat(chat) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index fef277ac6..fd2dc82ca 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -313,6 +313,7 @@ defmodule Pleroma.Web.Router do
       post("/chats/:id/messages", ChatController, :post_chat_message)
       delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
       post("/chats/:id/read", ChatController, :mark_as_read)
+      post("/chats/:id/messages/:message_id/read", ChatController, :mark_message_as_read)
 
       get("/conversations/:id/statuses", ConversationController, :statuses)
       get("/conversations/:id", ConversationController, :show)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index e62b71799..e7892142a 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -13,6 +13,33 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
 
   import Pleroma.Factory
 
+  describe "POST /api/v1/pleroma/chats/:id/messages/:message_id/read" do
+    setup do: oauth_access(["write:statuses"])
+
+    test "it marks one message as read", %{conn: conn, user: user} do
+      other_user = insert(:user)
+
+      {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
+      {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      object = Object.normalize(create, false)
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
+      assert cm_ref.seen == false
+
+      result =
+        conn
+        |> post("/api/v1/pleroma/chats/#{chat.id}/messages/#{cm_ref.id}/read")
+        |> json_response_and_validate_schema(200)
+
+      assert result["unread"] == false
+
+      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+
+      assert cm_ref.seen == true
+    end
+  end
+
   describe "POST /api/v1/pleroma/chats/:id/read" do
     setup do: oauth_access(["write:statuses"])
 
@@ -20,6 +47,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
       other_user = insert(:user)
 
       {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
+      {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
       object = Object.normalize(create, false)
       cm_ref = ChatMessageReference.for_chat_and_object(chat, object)

From 286bd8eb83e3fd9a2546e27c5e5d98f5316934a0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 19:24:37 +0200
Subject: [PATCH 137/283] Docs: Add `mark_message_as_read` to docs

---
 docs/API/chats.md | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index d1d39f495..c0ef75664 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -94,6 +94,15 @@ Returned data:
 }
 ```
 
+### Marking a single chat message as read
+
+To set the `unread` property of a message to `false`
+
+`POST /api/v1/pleroma/chats/:id/messages/:message_id/read`
+
+Returned data:
+
+The modified chat message
 
 ### Getting a list of Chats
 
@@ -149,7 +158,8 @@ Returned data:
         "visible_in_picker": false
       }
     ],
-    "id": "13"
+    "id": "13",
+    "unread": true
   },
   {
     "account_id": "someflakeid",
@@ -157,7 +167,8 @@ Returned data:
     "content": "Whats' up?",
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
-    "id": "12"
+    "id": "12",
+    "unread": false
   }
 ]
 ```
@@ -190,7 +201,8 @@ Returned data:
       "visible_in_picker": false
     }
   ],
-  "id": "13"
+  "id": "13",
+  "unread": false
 }
 ```
 
@@ -215,7 +227,8 @@ There's a new `pleroma:chat_mention` notification, which has this form. It is no
     "chat_id": "1",
     "id": "10",
     "content": "Hello",
-    "account_id": "someflakeid"
+    "account_id": "someflakeid",
+    "unread": false
   },
   "created_at": "somedate"
 }

From e213e3157737f87513999ef2aa00dffa735a8ada Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 19:25:57 +0200
Subject: [PATCH 138/283] Changelog: Add chats to changelog.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 839bf90ab..1cf2210f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
+- Chats: Added support for federated chats. For details, see the docs.
 - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
 - Instance: Add `background_image` to configuration and `/api/v1/instance`
 - Instance: Extend `/api/v1/instance` with Pleroma-specific information.

From e46aecda55b20c0d48463fb2a5c0040d4fc34e97 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 3 Jun 2020 20:51:59 +0200
Subject: [PATCH 139/283] Notification: Fix notifications backfill for
 compacted activities

---
 lib/pleroma/notification.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 0f33d282d..455d214bf 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -412,7 +412,7 @@ defp type_from_activity_object(%{data: %{"type" => "Create", "object" => %{}}}),
   defp type_from_activity_object(%{data: %{"type" => "Create"}} = activity) do
     object = Object.get_by_ap_id(activity.data["object"])
 
-    case object.data["type"] do
+    case object && object.data["type"] do
       "ChatMessage" -> "pleroma:chat_mention"
       _ -> "mention"
     end

From 5d7dda883e76041025384e453da74110c550aa3b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 14:46:41 +0200
Subject: [PATCH 140/283] SideEffectsTest: More tests.

---
 test/web/activity_pub/side_effects_test.exs | 22 ++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 92c266d84..82d72119e 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -321,7 +321,27 @@ test "it streams the created ChatMessage" do
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      with_mock Pleroma.Web.Streamer, [], stream: fn _, _ -> nil end do
+      with_mock Pleroma.Web.Streamer, [],
+        stream: fn _, payload ->
+          case payload do
+            {^author, cm_ref} ->
+              assert cm_ref.seen == true
+
+            {^recipient, cm_ref} ->
+              assert cm_ref.seen == false
+
+              view =
+                Pleroma.Web.PleromaAPI.ChatView.render("show.json",
+                  last_message: cm_ref,
+                  chat: cm_ref.chat
+                )
+
+              assert view.unread == 1
+
+            _ ->
+              nil
+          end
+        end do
         {:ok, _create_activity, _meta} =
           SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 

From b952f3f37907c735e3426ba43d01027f6f49c5b5 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 14:49:10 +0200
Subject: [PATCH 141/283] WebPush: Push out chat message notications.

---
 lib/pleroma/web/push/impl.ex         |  3 ++-
 lib/pleroma/web/push/subscription.ex |  2 +-
 test/web/push/impl_test.exs          | 17 +++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 125f33755..006a242af 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -32,7 +32,7 @@ def perform(
     mastodon_type = notification.type
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, false)
     user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 
@@ -171,6 +171,7 @@ def format_title(%{type: type}, mastodon_type) do
       "follow_request" -> "New Follow Request"
       "reblog" -> "New Repeat"
       "favourite" -> "New Favorite"
+      "pleroma:chat_mention" -> "New Chat Message"
       type -> "New #{String.capitalize(type || "event")}"
     end
   end
diff --git a/lib/pleroma/web/push/subscription.ex b/lib/pleroma/web/push/subscription.ex
index 3e401a490..5b5aa0d59 100644
--- a/lib/pleroma/web/push/subscription.ex
+++ b/lib/pleroma/web/push/subscription.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Push.Subscription do
     timestamps()
   end
 
-  @supported_alert_types ~w[follow favourite mention reblog]a
+  @supported_alert_types ~w[follow favourite mention reblog pleroma:chat_mention]a
 
   defp alerts(%{data: %{alerts: alerts}}) do
     alerts = Map.take(alerts, @supported_alert_types)
diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs
index 26c65bc82..8fb7faaa5 100644
--- a/test/web/push/impl_test.exs
+++ b/test/web/push/impl_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Push.ImplTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
@@ -196,6 +197,22 @@ test "renders title for create activity with direct visibility" do
   end
 
   describe "build_content/3" do
+    test "builds content for chat messages" do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, chat} = CommonAPI.post_chat_message(user, recipient, "hey")
+      object = Object.normalize(chat, false)
+      [notification] = Notification.for_user(recipient)
+
+      res = Impl.build_content(notification, user, object)
+
+      assert res == %{
+               body: "@#{user.nickname}: hey",
+               title: "New Chat Message"
+             }
+    end
+
     test "hides details for notifications when privacy option enabled" do
       user = insert(:user, nickname: "Bob")
       user2 = insert(:user, nickname: "Rob", notification_settings: %{privacy_option: true})

From 6e103a18af6cfd7f454a911e2f0e1ae35cd45aa4 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 14:49:36 +0200
Subject: [PATCH 142/283] Docs: Document WebPush changes.

---
 docs/API/chats.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index c0ef75664..abeee698f 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -237,3 +237,7 @@ There's a new `pleroma:chat_mention` notification, which has this form. It is no
 ### Streaming
 
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
+
+### Web Push
+
+If you want to receive push messages for this type, you'll need to add the `pleroma:chat_mention` type to your alerts in the push subscription.

From 00748e9650e911d828dfe6f769ac20a6b31c8b69 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 17:14:42 +0200
Subject: [PATCH 143/283] ChatMessageReferences: Change seen -> unread

---
 lib/pleroma/chat_message_reference.ex         | 18 +++++------
 lib/pleroma/web/activity_pub/side_effects.ex  |  2 +-
 .../views/chat_message_reference_view.ex      |  4 +--
 ...n_to_unread_in_chat_message_references.exs | 30 +++++++++++++++++++
 test/web/activity_pub/side_effects_test.exs   |  8 ++---
 .../controllers/chat_controller_test.exs      |  8 ++---
 6 files changed, 50 insertions(+), 20 deletions(-)
 create mode 100644 priv/repo/migrations/20200604150318_migrate_seen_to_unread_in_chat_message_references.exs

diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
index 9b00443f5..fc2aaae7a 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat_message_reference.ex
@@ -23,15 +23,15 @@ defmodule Pleroma.ChatMessageReference do
     belongs_to(:object, Object)
     belongs_to(:chat, Chat)
 
-    field(:seen, :boolean, default: false)
+    field(:unread, :boolean, default: true)
 
     timestamps()
   end
 
   def changeset(struct, params) do
     struct
-    |> cast(params, [:object_id, :chat_id, :seen])
-    |> validate_required([:object_id, :chat_id, :seen])
+    |> cast(params, [:object_id, :chat_id, :unread])
+    |> validate_required([:object_id, :chat_id, :unread])
   end
 
   def get_by_id(id) do
@@ -73,11 +73,11 @@ def last_message_for_chat(chat) do
     |> Repo.one()
   end
 
-  def create(chat, object, seen) do
+  def create(chat, object, unread) do
     params = %{
       chat_id: chat.id,
       object_id: object.id,
-      seen: seen
+      unread: unread
     }
 
     %__MODULE__{}
@@ -88,13 +88,13 @@ def create(chat, object, seen) do
   def unread_count_for_chat(chat) do
     chat
     |> for_chat_query()
-    |> where([cmr], cmr.seen == false)
+    |> where([cmr], cmr.unread == true)
     |> Repo.aggregate(:count)
   end
 
   def mark_as_read(cm_ref) do
     cm_ref
-    |> changeset(%{seen: true})
+    |> changeset(%{unread: false})
     |> Repo.update()
   end
 
@@ -103,7 +103,7 @@ def set_all_seen_for_chat(chat) do
     |> for_chat_query()
     |> exclude(:order_by)
     |> exclude(:preload)
-    |> where([cmr], cmr.seen == false)
-    |> Repo.update_all(set: [seen: true])
+    |> where([cmr], cmr.unread == true)
+    |> Repo.update_all(set: [unread: false])
   end
 end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0c5709356..e9f109d80 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -140,7 +140,7 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       |> Enum.each(fn [user, other_user] ->
         if user.local do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-          {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id == actor.ap_id)
+          {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
           Streamer.stream(
             ["user", "user:pleroma_chat"],
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
index f9405aec5..592bb17f0 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
@@ -16,7 +16,7 @@ def render(
             id: id,
             object: %{data: chat_message},
             chat_id: chat_id,
-            seen: seen
+            unread: unread
           }
         }
       ) do
@@ -30,7 +30,7 @@ def render(
       attachment:
         chat_message["attachment"] &&
           StatusView.render("attachment.json", attachment: chat_message["attachment"]),
-      unread: !seen
+      unread: unread
     }
   end
 
diff --git a/priv/repo/migrations/20200604150318_migrate_seen_to_unread_in_chat_message_references.exs b/priv/repo/migrations/20200604150318_migrate_seen_to_unread_in_chat_message_references.exs
new file mode 100644
index 000000000..fd6bc7bc7
--- /dev/null
+++ b/priv/repo/migrations/20200604150318_migrate_seen_to_unread_in_chat_message_references.exs
@@ -0,0 +1,30 @@
+defmodule Pleroma.Repo.Migrations.MigrateSeenToUnreadInChatMessageReferences do
+  use Ecto.Migration
+
+  def change do
+    drop(
+      index(:chat_message_references, [:chat_id],
+        where: "seen = false",
+        name: "unseen_messages_count_index"
+      )
+    )
+
+    alter table(:chat_message_references) do
+      add(:unread, :boolean, default: true)
+    end
+
+    execute("update chat_message_references set unread = not seen")
+
+    alter table(:chat_message_references) do
+      modify(:unread, :boolean, default: true, null: false)
+      remove(:seen, :boolean, default: false, null: false)
+    end
+
+    create(
+      index(:chat_message_references, [:chat_id],
+        where: "unread = true",
+        name: "unread_messages_count_index"
+      )
+    )
+  end
+end
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 82d72119e..40df664eb 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -325,10 +325,10 @@ test "it streams the created ChatMessage" do
         stream: fn _, payload ->
           case payload do
             {^author, cm_ref} ->
-              assert cm_ref.seen == true
+              assert cm_ref.unread == false
 
             {^recipient, cm_ref} ->
-              assert cm_ref.seen == false
+              assert cm_ref.unread == true
 
               view =
                 Pleroma.Web.PleromaAPI.ChatView.render("show.json",
@@ -369,14 +369,14 @@ test "it creates a Chat and ChatMessageReferences for the local users and bumps
       [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
 
       assert cm_ref.object.data["content"] == "hey"
-      assert cm_ref.seen == true
+      assert cm_ref.unread == false
 
       chat = Chat.get(recipient.id, author.ap_id)
 
       [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
 
       assert cm_ref.object.data["content"] == "hey"
-      assert cm_ref.seen == false
+      assert cm_ref.unread == true
     end
 
     test "it creates a Chat for the local users and bumps the unread count" do
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index e7892142a..7af6dec1c 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -25,7 +25,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
       object = Object.normalize(create, false)
       cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert cm_ref.seen == false
+      assert cm_ref.unread == true
 
       result =
         conn
@@ -36,7 +36,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
 
       cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert cm_ref.seen == true
+      assert cm_ref.unread == false
     end
   end
 
@@ -52,7 +52,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
       object = Object.normalize(create, false)
       cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert cm_ref.seen == false
+      assert cm_ref.unread == true
 
       result =
         conn
@@ -63,7 +63,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
 
       cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
 
-      assert cm_ref.seen == true
+      assert cm_ref.unread == false
     end
   end
 

From 41503b167335a5f54eb122ecfd945018c9b50f90 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 15:16:10 +0000
Subject: [PATCH 144/283] Apply suggestion to
 test/web/activity_pub/transmogrifier/chat_message_test.exs

---
 test/web/activity_pub/transmogrifier/chat_message_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/activity_pub/transmogrifier/chat_message_test.exs b/test/web/activity_pub/transmogrifier/chat_message_test.exs
index 820090de3..d6736dc3e 100644
--- a/test/web/activity_pub/transmogrifier/chat_message_test.exs
+++ b/test/web/activity_pub/transmogrifier/chat_message_test.exs
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   describe "handle_incoming" do
-    test "handles this" do
+    test "handles chonks with attachment" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
         "actor" => "https://honk.tedunangst.com/u/tedu",

From 9a53f619e03cd515460a7b1570d339e4554e1740 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 15:16:15 +0000
Subject: [PATCH 145/283] Apply suggestion to
 test/chat_message_reference_test.exs

---
 test/chat_message_reference_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/chat_message_reference_test.exs b/test/chat_message_reference_test.exs
index 963a0e225..66bf493b4 100644
--- a/test/chat_message_reference_test.exs
+++ b/test/chat_message_reference_test.exs
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ChatMessageReferencTest do
+defmodule Pleroma.ChatMessageReferenceTest do
   use Pleroma.DataCase, async: true
 
   alias Pleroma.Chat

From 56dfa0e0fb0ca34054930e64e092f4a3a1b87fd1 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 19:22:49 +0200
Subject: [PATCH 146/283] Transmogrifier: Update notification after accepting.

---
 lib/pleroma/web/activity_pub/transmogrifier.ex           | 1 +
 .../activity_pub/transmogrifier/follow_handling_test.exs | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 886403fcd..b2461de2b 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -538,6 +538,7 @@ def handle_incoming(
            {_, false} <- {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked},
            {_, false} <- {:user_locked, User.locked?(followed)},
            {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)},
+           _ <- Notification.update_notification_type(followed, activity),
            {_, {:ok, _}} <-
              {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")},
            {:ok, _relationship} <-
diff --git a/test/web/activity_pub/transmogrifier/follow_handling_test.exs b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
index 967389fae..6b003b51c 100644
--- a/test/web/activity_pub/transmogrifier/follow_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.Transmogrifier.FollowHandlingTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Notification
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -57,9 +58,12 @@ test "it works for incoming follow requests" do
       activity = Repo.get(Activity, activity.id)
       assert activity.data["state"] == "accept"
       assert User.following?(User.get_cached_by_ap_id(data["actor"]), user)
+
+      [notification] = Notification.for_user(user)
+      assert notification.type == "follow"
     end
 
-    test "with locked accounts, it does not create a follow or an accept" do
+    test "with locked accounts, it does create a Follow, but not an Accept" do
       user = insert(:user, locked: true)
 
       data =
@@ -81,6 +85,9 @@ test "with locked accounts, it does not create a follow or an accept" do
         |> Repo.all()
 
       assert Enum.empty?(accepts)
+
+      [notification] = Notification.for_user(user)
+      assert notification.type == "follow_request"
     end
 
     test "it works for follow requests when you are already followed, creating a new accept activity" do

From d44da91bbf50ae91e8246ebe3669cfaf1fabda1b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 20:28:33 +0200
Subject: [PATCH 147/283] SubscriptionOperation: Let chat mentions through.

---
 .../web/api_spec/operations/subscription_operation.ex       | 5 +++++
 .../controllers/subscription_controller_test.exs            | 6 ++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/subscription_operation.ex b/lib/pleroma/web/api_spec/operations/subscription_operation.ex
index c575a87e6..775dd795d 100644
--- a/lib/pleroma/web/api_spec/operations/subscription_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/subscription_operation.ex
@@ -141,6 +141,11 @@ defp create_request do
                   allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive poll notifications?"
+                },
+                "pleroma:chat_mention": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive chat notifications?"
                 }
               }
             }
diff --git a/test/web/mastodon_api/controllers/subscription_controller_test.exs b/test/web/mastodon_api/controllers/subscription_controller_test.exs
index 4aa260663..d36bb1ae8 100644
--- a/test/web/mastodon_api/controllers/subscription_controller_test.exs
+++ b/test/web/mastodon_api/controllers/subscription_controller_test.exs
@@ -58,7 +58,9 @@ test "successful creation", %{conn: conn} do
       result =
         conn
         |> post("/api/v1/push/subscription", %{
-          "data" => %{"alerts" => %{"mention" => true, "test" => true}},
+          "data" => %{
+            "alerts" => %{"mention" => true, "test" => true, "pleroma:chat_mention" => true}
+          },
           "subscription" => @sub
         })
         |> json_response_and_validate_schema(200)
@@ -66,7 +68,7 @@ test "successful creation", %{conn: conn} do
       [subscription] = Pleroma.Repo.all(Subscription)
 
       assert %{
-               "alerts" => %{"mention" => true},
+               "alerts" => %{"mention" => true, "pleroma:chat_mention" => true},
                "endpoint" => subscription.endpoint,
                "id" => to_string(subscription.id),
                "server_key" => @server_key

From aa2ac76510d95f2412e23f3739e8e1ae4402643f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 4 Jun 2020 20:40:46 +0200
Subject: [PATCH 148/283] Notification: Don't break on figuring out the type of
 old EmojiReactions

---
 lib/pleroma/notification.ex |  4 ++++
 test/notification_test.exs  | 17 ++++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 455d214bf..e5b880b10 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -398,6 +398,10 @@ defp type_from_activity(%{data: %{"type" => type}} = activity, opts \\ []) do
       "EmojiReact" ->
         "pleroma:emoji_reaction"
 
+      # Compatibility with old reactions
+      "EmojiReaction" ->
+        "pleroma:emoji_reaction"
+
       "Create" ->
         activity
         |> type_from_activity_object()
diff --git a/test/notification_test.exs b/test/notification_test.exs
index 6bc2b6904..f2115a29e 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -8,8 +8,10 @@ defmodule Pleroma.NotificationTest do
   import Pleroma.Factory
   import Mock
 
+  alias Pleroma.Activity
   alias Pleroma.FollowingRelationship
   alias Pleroma.Notification
+  alias Pleroma.Repo
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -29,8 +31,18 @@ test "it fills in missing notification types" do
       {:ok, chat} = CommonAPI.post_chat_message(user, other_user, "yo")
       {:ok, react} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
       {:ok, like} = CommonAPI.favorite(other_user, post.id)
+      {:ok, react_2} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
 
-      assert {4, nil} = Repo.update_all(Notification, set: [type: nil])
+      data =
+        react_2.data
+        |> Map.put("type", "EmojiReaction")
+
+      {:ok, react_2} =
+        react_2
+        |> Activity.change(%{data: data})
+        |> Repo.update()
+
+      assert {5, nil} = Repo.update_all(Notification, set: [type: nil])
 
       Notification.fill_in_notification_types()
 
@@ -43,6 +55,9 @@ test "it fills in missing notification types" do
       assert %{type: "pleroma:emoji_reaction"} =
                Repo.get_by(Notification, user_id: user.id, activity_id: react.id)
 
+      assert %{type: "pleroma:emoji_reaction"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: react_2.id)
+
       assert %{type: "pleroma:chat_mention"} =
                Repo.get_by(Notification, user_id: other_user.id, activity_id: chat.id)
     end

From cc8a7dc205a4516452c48659e6bf081f3f730496 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 12:01:33 +0200
Subject: [PATCH 149/283] SideEffects / ChatView: Add an unread cache.

This is to prevent wrong values in the stream.
---
 lib/pleroma/web/activity_pub/side_effects.ex  |  5 ++++
 .../web/pleroma_api/views/chat_view.ex        |  2 +-
 lib/pleroma/web/streamer/streamer.ex          | 28 +++++--------------
 lib/pleroma/web/views/streamer_view.ex        |  2 ++
 test/web/pleroma_api/views/chat_view_test.exs | 15 ++++++++++
 5 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index e9f109d80..992c04ac1 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -142,6 +142,11 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
           {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
+          # We add a cache of the unread value here so that it doesn't change when being streamed out
+          chat =
+            chat
+            |> Map.put(:unread, ChatMessageReference.unread_count_for_chat(chat))
+
           Streamer.stream(
             ["user", "user:pleroma_chat"],
             {user, %{cm_ref | chat: chat, object: object}}
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index c903a71fd..91d50dd1e 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -20,7 +20,7 @@ def render("show.json", %{chat: %Chat{} = chat} = opts) do
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
-      unread: ChatMessageReference.unread_count_for_chat(chat),
+      unread: Map.get(chat, :unread) || ChatMessageReference.unread_count_for_chat(chat),
       last_message:
         last_message &&
           ChatMessageReferenceView.render("show.json", chat_message_reference: last_message),
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 5e37e2cf2..b22297955 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -90,34 +90,20 @@ def remove_socket(topic) do
     if should_env_send?(), do: Registry.unregister(@registry, topic)
   end
 
-  def stream(topics, item) when is_list(topics) do
+  def stream(topics, items) do
     if should_env_send?() do
-      Enum.each(topics, fn t ->
-        spawn(fn -> do_stream(t, item) end)
+      List.wrap(topics)
+      |> Enum.each(fn topic ->
+        List.wrap(items)
+        |> Enum.each(fn item ->
+          spawn(fn -> do_stream(topic, item) end)
+        end)
       end)
     end
 
     :ok
   end
 
-  def stream(topic, items) when is_list(items) do
-    if should_env_send?() do
-      Enum.each(items, fn i ->
-        spawn(fn -> do_stream(topic, i) end)
-      end)
-
-      :ok
-    end
-  end
-
-  def stream(topic, item) do
-    if should_env_send?() do
-      spawn(fn -> do_stream(topic, item) end)
-    end
-
-    :ok
-  end
-
   def filtered_by_user?(%User{} = user, %Activity{} = item) do
     %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} =
       User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute])
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index a6efd0109..b000e7ce0 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -55,6 +55,8 @@ def render("chat_update.json", %{chat_message_reference: cm_ref}) do
     # Explicitly giving the cmr for the object here, so we don't accidentally
     # send a later 'last_message' that was inserted between inserting this and
     # streaming it out
+    # 
+    # It also contains the chat with a cache of the correct unread count
     Logger.debug("Trying to stream out #{inspect(cm_ref)}")
 
     representation =
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index f3bd12616..f77584dd1 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -16,6 +16,21 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
 
   import Pleroma.Factory
 
+  test "giving a chat with an 'unread' field, it uses that" do
+    user = insert(:user)
+    recipient = insert(:user)
+
+    {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
+
+    chat =
+      chat
+      |> Map.put(:unread, 5)
+
+    represented_chat = ChatView.render("show.json", chat: chat)
+
+    assert represented_chat[:unread] == 5
+  end
+
   test "it represents a chat" do
     user = insert(:user)
     recipient = insert(:user)

From 0efa8aa0b9567f42b1af63e2b93a9c51e9a0fb11 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 12:26:07 +0200
Subject: [PATCH 150/283] Transmogrifier: For follows, create notifications
 last.

As the notification type changes depending on the follow state,
the notification should not be created and streamed out before the
state settles. For this reason, the notification creation has been
delayed until it's clear if the user has been followed or not.

This is a bit hacky but it will be properly rewritten using the
pipeline soon.
---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 12 +++++++-----
 lib/pleroma/web/activity_pub/transmogrifier.ex |  5 +++--
 .../transmogrifier/follow_handling_test.exs    | 18 ++++++++++++++++++
 3 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 568db2348..4f7043c92 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -363,19 +363,21 @@ def update(%{to: to, cc: cc, actor: actor, object: object} = params) do
     end
   end
 
-  @spec follow(User.t(), User.t(), String.t() | nil, boolean()) ::
+  @spec follow(User.t(), User.t(), String.t() | nil, boolean(), keyword()) ::
           {:ok, Activity.t()} | {:error, any()}
-  def follow(follower, followed, activity_id \\ nil, local \\ true) do
+  def follow(follower, followed, activity_id \\ nil, local \\ true, opts \\ []) do
     with {:ok, result} <-
-           Repo.transaction(fn -> do_follow(follower, followed, activity_id, local) end) do
+           Repo.transaction(fn -> do_follow(follower, followed, activity_id, local, opts) end) do
       result
     end
   end
 
-  defp do_follow(follower, followed, activity_id, local) do
+  defp do_follow(follower, followed, activity_id, local, opts) do
+    skip_notify_and_stream = Keyword.get(opts, :skip_notify_and_stream, false)
+
     with data <- make_follow_data(follower, followed, activity_id),
          {:ok, activity} <- insert(data, local),
-         _ <- notify_and_stream(activity),
+         _ <- skip_notify_and_stream || notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index b2461de2b..50f3216f3 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -533,12 +533,12 @@ def handle_incoming(
            User.get_cached_by_ap_id(Containment.get_actor(%{"actor" => followed})),
          {:ok, %User{} = follower} <-
            User.get_or_fetch_by_ap_id(Containment.get_actor(%{"actor" => follower})),
-         {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
+         {:ok, activity} <-
+           ActivityPub.follow(follower, followed, id, false, skip_notify_and_stream: true) do
       with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]),
            {_, false} <- {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked},
            {_, false} <- {:user_locked, User.locked?(followed)},
            {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)},
-           _ <- Notification.update_notification_type(followed, activity),
            {_, {:ok, _}} <-
              {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")},
            {:ok, _relationship} <-
@@ -577,6 +577,7 @@ def handle_incoming(
           :noop
       end
 
+      ActivityPub.notify_and_stream(activity)
       {:ok, activity}
     else
       _e ->
diff --git a/test/web/activity_pub/transmogrifier/follow_handling_test.exs b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
index 6b003b51c..06c39eed6 100644
--- a/test/web/activity_pub/transmogrifier/follow_handling_test.exs
+++ b/test/web/activity_pub/transmogrifier/follow_handling_test.exs
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.FollowHandlingTest do
 
   import Pleroma.Factory
   import Ecto.Query
+  import Mock
 
   setup_all do
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -151,6 +152,23 @@ test "it rejects incoming follow requests from blocked users when deny_follow_bl
       assert activity.data["state"] == "reject"
     end
 
+    test "it rejects incoming follow requests if the following errors for some reason" do
+      user = insert(:user)
+
+      data =
+        File.read!("test/fixtures/mastodon-follow-activity.json")
+        |> Poison.decode!()
+        |> Map.put("object", user.ap_id)
+
+      with_mock Pleroma.User, [:passthrough], follow: fn _, _ -> {:error, :testing} end do
+        {:ok, %Activity{data: %{"id" => id}}} = Transmogrifier.handle_incoming(data)
+
+        %Activity{} = activity = Activity.get_by_ap_id(id)
+
+        assert activity.data["state"] == "reject"
+      end
+    end
+
     test "it works for incoming follow requests from hubzilla" do
       user = insert(:user)
 

From f3ea6ee2c82b2d63991d3e658566298c722ac0af Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 12:45:25 +0200
Subject: [PATCH 151/283] Credo fixes.

---
 lib/pleroma/web/activity_pub/side_effects.ex | 3 ++-
 lib/pleroma/web/views/streamer_view.ex       | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 992c04ac1..e7d050e81 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -142,7 +142,8 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
           {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
-          # We add a cache of the unread value here so that it doesn't change when being streamed out
+          # We add a cache of the unread value here so that it 
+          # doesn't change when being streamed out
           chat =
             chat
             |> Map.put(:unread, ChatMessageReference.unread_count_for_chat(chat))
diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex
index b000e7ce0..476a33245 100644
--- a/lib/pleroma/web/views/streamer_view.ex
+++ b/lib/pleroma/web/views/streamer_view.ex
@@ -55,7 +55,7 @@ def render("chat_update.json", %{chat_message_reference: cm_ref}) do
     # Explicitly giving the cmr for the object here, so we don't accidentally
     # send a later 'last_message' that was inserted between inserting this and
     # streaming it out
-    # 
+    #
     # It also contains the chat with a cache of the correct unread count
     Logger.debug("Trying to stream out #{inspect(cm_ref)}")
 

From 65689ba9bd44e291fc626cce2bd5136b93a5da90 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 13:10:48 +0200
Subject: [PATCH 152/283] If Credo fixes is so good, why is there no Credo
 fixes 2?

---
 lib/pleroma/web/activity_pub/side_effects.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index e7d050e81..b3aacff40 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -142,7 +142,7 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
           {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
-          # We add a cache of the unread value here so that it 
+          # We add a cache of the unread value here so that it
           # doesn't change when being streamed out
           chat =
             chat

From 115d08a7542b92c5e1d889da41c0ee6837a1235e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 16:47:02 +0200
Subject: [PATCH 153/283] Pipeline: Add a side effects step after the
 transaction finishes

This is to run things like streaming notifications out, which will
sometimes need data that is created by the transaction, but is
streamed out asynchronously.
---
 lib/pleroma/notification.ex                  | 26 ++++--
 lib/pleroma/web/activity_pub/pipeline.ex     |  4 +
 lib/pleroma/web/activity_pub/side_effects.ex | 30 ++++++-
 test/web/activity_pub/pipeline_test.exs      |  9 +-
 test/web/activity_pub/side_effects_test.exs  | 86 +++++++++++++++++---
 test/web/common_api/common_api_test.exs      | 43 ++++++++--
 6 files changed, 169 insertions(+), 29 deletions(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index e5b880b10..49e27c05a 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -334,30 +334,34 @@ def dismiss(%{id: user_id} = _user, id) do
     end
   end
 
-  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
+  def create_notifications(activity, options \\ [])
+
+  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do
     object = Object.normalize(activity, false)
 
     if object && object.data["type"] == "Answer" do
       {:ok, []}
     else
-      do_create_notifications(activity)
+      do_create_notifications(activity, options)
     end
   end
 
-  def create_notifications(%Activity{data: %{"type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
       when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
-    do_create_notifications(activity)
+    do_create_notifications(activity, options)
   end
 
-  def create_notifications(_), do: {:ok, []}
+  def create_notifications(_, _), do: {:ok, []}
+
+  defp do_create_notifications(%Activity{} = activity, options) do
+    do_send = Keyword.get(options, :do_send, true)
 
-  defp do_create_notifications(%Activity{} = activity) do
     {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
     potential_receivers = enabled_receivers ++ disabled_receivers
 
     notifications =
       Enum.map(potential_receivers, fn user ->
-        do_send = user in enabled_receivers
+        do_send = do_send && user in enabled_receivers
         create_notification(activity, user, do_send)
       end)
 
@@ -623,4 +627,12 @@ def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity,
   end
 
   def skip?(_, _, _), do: false
+
+  def for_user_and_activity(user, activity) do
+    from(n in __MODULE__,
+      where: n.user_id == ^user.id,
+      where: n.activity_id == ^activity.id
+    )
+    |> Repo.one()
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 0c54c4b23..6875c47f6 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -17,6 +17,10 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
           {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}
   def common_pipeline(object, meta) do
     case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do
+      {:ok, {:ok, activity, meta}} ->
+        SideEffects.handle_after_transaction(meta)
+        {:ok, activity, meta}
+
       {:ok, value} ->
         value
 
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index b3aacff40..10136789a 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Streamer
+  alias Pleroma.Web.Push
 
   def handle(object, meta \\ [])
 
@@ -37,7 +38,12 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
     with {:ok, _object, _meta} <- handle_object_creation(meta[:object_data], meta) do
-      Notification.create_notifications(activity)
+      {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+
+      meta =
+        meta
+        |> add_notifications(notifications)
+
       {:ok, activity, meta}
     else
       e -> Repo.rollback(e)
@@ -200,4 +206,26 @@ def handle_undoing(
   end
 
   def handle_undoing(object), do: {:error, ["don't know how to handle", object]}
+
+  defp send_notifications(meta) do
+    Keyword.get(meta, :created_notifications, [])
+    |> Enum.each(fn notification ->
+      Streamer.stream(["user", "user:notification"], notification)
+      Push.send(notification)
+    end)
+
+    meta
+  end
+
+  defp add_notifications(meta, notifications) do
+    existing = Keyword.get(meta, :created_notifications, [])
+
+    meta
+    |> Keyword.put(:created_notifications, notifications ++ existing)
+  end
+
+  def handle_after_transaction(meta) do
+    meta
+    |> send_notifications()
+  end
 end
diff --git a/test/web/activity_pub/pipeline_test.exs b/test/web/activity_pub/pipeline_test.exs
index 26557720b..8deb64501 100644
--- a/test/web/activity_pub/pipeline_test.exs
+++ b/test/web/activity_pub/pipeline_test.exs
@@ -33,7 +33,10 @@ test "it goes through validation, filtering, persisting, side effects and federa
         {
           Pleroma.Web.ActivityPub.SideEffects,
           [],
-          [handle: fn o, m -> {:ok, o, m} end]
+          [
+            handle: fn o, m -> {:ok, o, m} end,
+            handle_after_transaction: fn m -> m end
+          ]
         },
         {
           Pleroma.Web.Federator,
@@ -71,7 +74,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
         {
           Pleroma.Web.ActivityPub.SideEffects,
           [],
-          [handle: fn o, m -> {:ok, o, m} end]
+          [handle: fn o, m -> {:ok, o, m} end, handle_after_transaction: fn m -> m end]
         },
         {
           Pleroma.Web.Federator,
@@ -110,7 +113,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
         {
           Pleroma.Web.ActivityPub.SideEffects,
           [],
-          [handle: fn o, m -> {:ok, o, m} end]
+          [handle: fn o, m -> {:ok, o, m} end, handle_after_transaction: fn m -> m end]
         },
         {
           Pleroma.Web.Federator,
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 40df664eb..43ffe1337 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -22,6 +22,47 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
   import Pleroma.Factory
   import Mock
 
+  describe "handle_after_transaction" do
+    test "it streams out notifications" do
+      author = insert(:user, local: true)
+      recipient = insert(:user, local: true)
+
+      {:ok, chat_message_data, _meta} = Builder.chat_message(author, recipient.ap_id, "hey")
+
+      {:ok, create_activity_data, _meta} =
+        Builder.create(author, chat_message_data["id"], [recipient.ap_id])
+
+      {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
+
+      {:ok, _create_activity, meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
+
+      assert [notification] = meta[:created_notifications]
+
+      with_mocks([
+        {
+          Pleroma.Web.Streamer,
+          [],
+          [
+            stream: fn _, _ -> nil end
+          ]
+        },
+        {
+          Pleroma.Web.Push,
+          [],
+          [
+            send: fn _ -> nil end
+          ]
+        }
+      ]) do
+        SideEffects.handle_after_transaction(meta)
+
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+        assert called(Pleroma.Web.Push.send(notification))
+      end
+    end
+  end
+
   describe "delete objects" do
     setup do
       user = insert(:user)
@@ -361,22 +402,47 @@ test "it creates a Chat and ChatMessageReferences for the local users and bumps
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      {:ok, _create_activity, _meta} =
-        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
+      with_mocks([
+        {
+          Pleroma.Web.Streamer,
+          [],
+          [
+            stream: fn _, _ -> nil end
+          ]
+        },
+        {
+          Pleroma.Web.Push,
+          [],
+          [
+            send: fn _ -> nil end
+          ]
+        }
+      ]) do
+        {:ok, _create_activity, meta} =
+          SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
-      chat = Chat.get(author.id, recipient.ap_id)
+        # The notification gets created
+        assert [notification] = meta[:created_notifications]
+        assert notification.activity_id == create_activity.id
 
-      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+        # But it is not sent out
+        refute called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+        refute called(Pleroma.Web.Push.send(notification))
 
-      assert cm_ref.object.data["content"] == "hey"
-      assert cm_ref.unread == false
+        chat = Chat.get(author.id, recipient.ap_id)
 
-      chat = Chat.get(recipient.id, author.ap_id)
+        [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
 
-      [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+        assert cm_ref.object.data["content"] == "hey"
+        assert cm_ref.unread == false
 
-      assert cm_ref.object.data["content"] == "hey"
-      assert cm_ref.unread == true
+        chat = Chat.get(recipient.id, author.ap_id)
+
+        [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+
+        assert cm_ref.object.data["content"] == "hey"
+        assert cm_ref.unread == true
+      end
     end
 
     test "it creates a Chat for the local users and bumps the unread count" do
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 611a9ae66..63b59820e 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPITest do
   alias Pleroma.Activity
   alias Pleroma.Chat
   alias Pleroma.Conversation.Participation
+  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -39,15 +40,41 @@ test "it posts a chat message without content but with an attachment" do
 
       {:ok, upload} = ActivityPub.upload(file, actor: author.ap_id)
 
-      {:ok, activity} =
-        CommonAPI.post_chat_message(
-          author,
-          recipient,
-          nil,
-          media_id: upload.id
-        )
+      with_mocks([
+        {
+          Pleroma.Web.Streamer,
+          [],
+          [
+            stream: fn _, _ ->
+              nil
+            end
+          ]
+        },
+        {
+          Pleroma.Web.Push,
+          [],
+          [
+            send: fn _ -> nil end
+          ]
+        }
+      ]) do
+        {:ok, activity} =
+          CommonAPI.post_chat_message(
+            author,
+            recipient,
+            nil,
+            media_id: upload.id
+          )
 
-      assert activity
+        notification =
+          Notification.for_user_and_activity(recipient, activity)
+          |> Repo.preload(:activity)
+
+        assert called(Pleroma.Web.Push.send(notification))
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+
+        assert activity
+      end
     end
 
     test "it adds html newlines" do

From f24d2f714f44175cae9fcd878de1629ee32be73c Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 5 Jun 2020 17:18:48 +0200
Subject: [PATCH 154/283] Credo fixes

---
 lib/pleroma/web/activity_pub/side_effects.ex   | 2 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 10136789a..5258212ec 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -15,8 +15,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Streamer
   alias Pleroma.Web.Push
+  alias Pleroma.Web.Streamer
 
   def handle(object, meta \\ [])
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index f97ab510e..d2347cdc9 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
   alias Pleroma.FollowingRelationship
-  alias Pleroma.Notification
   alias Pleroma.Maps
+  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
   alias Pleroma.Repo

From 4e8c0eecd5179428a47795380a9da1ab0419e024 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 09:46:07 +0200
Subject: [PATCH 155/283] WebPush: Don't break on contentless chat messages.

---
 lib/pleroma/web/push/impl.ex |  7 +++++++
 test/web/push/impl_test.exs  | 25 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 006a242af..cdb827e76 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -124,6 +124,13 @@ def build_content(notification, actor, object, mastodon_type) do
 
   def format_body(activity, actor, object, mastodon_type \\ nil)
 
+  def format_body(_activity, actor, %{data: %{"type" => "ChatMessage", "content" => content}}, _) do
+    case content do
+      nil -> "@#{actor.nickname}: (Attachment)"
+      content -> "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+    end
+  end
+
   def format_body(
         %{activity: %{data: %{"type" => "Create"}}},
         actor,
diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs
index 8fb7faaa5..b48952b29 100644
--- a/test/web/push/impl_test.exs
+++ b/test/web/push/impl_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Push.ImplTest do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Push.Impl
   alias Pleroma.Web.Push.Subscription
@@ -213,6 +214,30 @@ test "builds content for chat messages" do
              }
     end
 
+    test "builds content for chat messages with no content" do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
+
+      {:ok, chat} = CommonAPI.post_chat_message(user, recipient, nil, media_id: upload.id)
+      object = Object.normalize(chat, false)
+      [notification] = Notification.for_user(recipient)
+
+      res = Impl.build_content(notification, user, object)
+
+      assert res == %{
+               body: "@#{user.nickname}: (Attachment)",
+               title: "New Chat Message"
+             }
+    end
+
     test "hides details for notifications when privacy option enabled" do
       user = insert(:user, nickname: "Bob")
       user2 = insert(:user, nickname: "Rob", notification_settings: %{privacy_option: true})

From c5e3f2454c736e09de5c433a2bf578e8eb0e70c3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 10:35:38 +0200
Subject: [PATCH 156/283] Docs: Unify parameters in examples.

---
 docs/API/chats.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index abeee698f..761047336 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -41,7 +41,7 @@ This is the overview of using the API. The API is also documented via OpenAPI, s
 To create or get an existing Chat for a certain recipient (identified by Account ID)
 you can call:
 
-`POST /api/v1/pleroma/chats/by-account-id/{account_id}`
+`POST /api/v1/pleroma/chats/by-account-id/:account_id`
 
 The account id is the normal FlakeId of the user
 ```
@@ -136,7 +136,7 @@ The usual pagination options are implemented.
 
 For a given Chat id, you can get the associated messages with
 
-`GET /api/v1/pleroma/chats/{id}/messages`
+`GET /api/v1/pleroma/chats/:id/messages`
 
 This will return all messages, sorted by most recent to least recent. The usual
 pagination options are implemented.
@@ -177,7 +177,7 @@ Returned data:
 
 Posting a chat message for given Chat id works like this:
 
-`POST /api/v1/pleroma/chats/{id}/messages`
+`POST /api/v1/pleroma/chats/:id/messages`
 
 Parameters:
 - content: The text content of the message. Optional if media is attached.
@@ -210,7 +210,7 @@ Returned data:
 
 Deleting a chat message for given Chat id works like this:
 
-`DELETE /api/v1/pleroma/chats/{chat_id}/messages/{message_id}`
+`DELETE /api/v1/pleroma/chats/:chat_id/messages/:message_id`
 
 Returned data is the deleted message.
 

From 239d03499ebe9196c099b6c8ded05f1f6634c09d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 10:38:45 +0200
Subject: [PATCH 157/283] Chat: creation_cng -> changeset

Make our usage of this more uniform.
---
 lib/pleroma/chat.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 5aefddc5e..4fe31de94 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Chat do
     timestamps()
   end
 
-  def creation_cng(struct, params) do
+  def changeset(struct, params) do
     struct
     |> cast(params, [:user_id, :recipient])
     |> validate_change(:recipient, fn
@@ -49,7 +49,7 @@ def get(user_id, recipient) do
 
   def get_or_create(user_id, recipient) do
     %__MODULE__{}
-    |> creation_cng(%{user_id: user_id, recipient: recipient})
+    |> changeset(%{user_id: user_id, recipient: recipient})
     |> Repo.insert(
       # Need to set something, otherwise we get nothing back at all
       on_conflict: [set: [recipient: recipient]],
@@ -60,7 +60,7 @@ def get_or_create(user_id, recipient) do
 
   def bump_or_create(user_id, recipient) do
     %__MODULE__{}
-    |> creation_cng(%{user_id: user_id, recipient: recipient})
+    |> changeset(%{user_id: user_id, recipient: recipient})
     |> Repo.insert(
       on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
       conflict_target: [:user_id, :recipient]

From 137adef6e061a1d7d7fc704feac27ebf5319a768 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 10:42:24 +0200
Subject: [PATCH 158/283] ChatMessageReference: Use FlakeId.Ecto.Type

No need for compat because this is brand new.
---
 lib/pleroma/chat_message_reference.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat_message_reference.ex
index fc2aaae7a..6e836cad9 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat_message_reference.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.ChatMessageReference do
   import Ecto.Changeset
   import Ecto.Query
 
-  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+  @primary_key {:id, FlakeId.Ecto.Type, autogenerate: true}
 
   schema "chat_message_references" do
     belongs_to(:object, Object)

From ca0e6e702be3714bb40ff0fb48e9c08aaf322fff Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 11:51:10 +0200
Subject: [PATCH 159/283] ChatMessageReference -> Chat.MessageReference

---
 .../message_reference.ex}                     |  2 +-
 lib/pleroma/web/activity_pub/side_effects.ex  |  8 ++---
 .../mastodon_api/views/notification_view.ex   |  8 ++---
 .../controllers/chat_controller.ex            | 30 +++++++++----------
 .../message_reference_view.ex}                |  2 +-
 .../web/pleroma_api/views/chat_view.ex        | 10 +++----
 lib/pleroma/web/streamer/streamer.ex          |  4 +--
 .../message_reference_test.exs}               |  6 ++--
 test/web/activity_pub/side_effects_test.exs   |  8 ++---
 .../views/notification_view_test.exs          |  9 +++---
 .../controllers/chat_controller_test.exs      | 18 +++++------
 .../message_reference_view_test.exs}          | 15 +++++-----
 test/web/pleroma_api/views/chat_view_test.exs |  8 ++---
 test/web/streamer/streamer_test.exs           |  6 ++--
 14 files changed, 66 insertions(+), 68 deletions(-)
 rename lib/pleroma/{chat_message_reference.ex => chat/message_reference.ex} (98%)
 rename lib/pleroma/web/pleroma_api/views/{chat_message_reference_view.ex => chat/message_reference_view.ex} (95%)
 rename test/{chat_message_reference_test.exs => chat/message_reference_test.exs} (82%)
 rename test/web/pleroma_api/views/{chat_message_reference_view_test.exs => chat/message_reference_view_test.exs} (77%)

diff --git a/lib/pleroma/chat_message_reference.ex b/lib/pleroma/chat/message_reference.ex
similarity index 98%
rename from lib/pleroma/chat_message_reference.ex
rename to lib/pleroma/chat/message_reference.ex
index 6e836cad9..4b201db2e 100644
--- a/lib/pleroma/chat_message_reference.ex
+++ b/lib/pleroma/chat/message_reference.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ChatMessageReference do
+defmodule Pleroma.Chat.MessageReference do
   @moduledoc """
   A reference that builds a relation between an AP chat message that a user can see and whether it has been seen
   by them, or should be displayed to them. Used to build the chat view that is presented to the user.
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 5258212ec..1e9d6c2fc 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -111,7 +111,7 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
               Object.decrease_replies_count(in_reply_to)
             end
 
-            ChatMessageReference.delete_for_object(deleted_object)
+            MessageReference.delete_for_object(deleted_object)
 
             ActivityPub.stream_out(object)
             ActivityPub.stream_out_participations(deleted_object, user)
@@ -146,13 +146,13 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       |> Enum.each(fn [user, other_user] ->
         if user.local do
           {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-          {:ok, cm_ref} = ChatMessageReference.create(chat, object, user.ap_id != actor.ap_id)
+          {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
           # We add a cache of the unread value here so that it
           # doesn't change when being streamed out
           chat =
             chat
-            |> Map.put(:unread, ChatMessageReference.unread_count_for_chat(chat))
+            |> Map.put(:unread, MessageReference.unread_count_for_chat(chat))
 
           Streamer.stream(
             ["user", "user:pleroma_chat"],
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 2ae82eb2d..b11578623 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   use Pleroma.Web, :view
 
   alias Pleroma.Activity
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
 
   @parent_types ~w{Like Announce EmojiReact}
 
@@ -139,9 +139,9 @@ defp put_chat_message(response, activity, reading_user, opts) do
     object = Object.normalize(activity)
     author = User.get_cached_by_ap_id(object.data["actor"])
     chat = Pleroma.Chat.get(reading_user.id, author.ap_id)
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+    cm_ref = MessageReference.for_chat_and_object(chat, object)
     render_opts = Map.merge(opts, %{for: reading_user, chat_message_reference: cm_ref})
-    chat_message_render = ChatMessageReferenceView.render("show.json", render_opts)
+    chat_message_render = MessageReferenceView.render("show.json", render_opts)
 
     Map.put(response, :chat_message, chat_message_render)
   end
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 01d47045d..d6b3415d1 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -6,14 +6,14 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
   alias Pleroma.Web.PleromaAPI.ChatView
 
   import Ecto.Query
@@ -46,13 +46,13 @@ def delete_message(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
         message_id: message_id,
         id: chat_id
       }) do
-    with %ChatMessageReference{} = cm_ref <-
-           ChatMessageReference.get_by_id(message_id),
+    with %MessageReference{} = cm_ref <-
+           MessageReference.get_by_id(message_id),
          ^chat_id <- cm_ref.chat_id |> to_string(),
          %Chat{user_id: ^user_id} <- Chat.get_by_id(chat_id),
          {:ok, _} <- remove_or_delete(cm_ref, user) do
       conn
-      |> put_view(ChatMessageReferenceView)
+      |> put_view(MessageReferenceView)
       |> render("show.json", chat_message_reference: cm_ref)
     else
       _e ->
@@ -71,7 +71,7 @@ defp remove_or_delete(
 
   defp remove_or_delete(cm_ref, _) do
     cm_ref
-    |> ChatMessageReference.delete()
+    |> MessageReference.delete()
   end
 
   def post_chat_message(
@@ -87,9 +87,9 @@ def post_chat_message(
              media_id: params[:media_id]
            ),
          message <- Object.normalize(activity, false),
-         cm_ref <- ChatMessageReference.for_chat_and_object(chat, message) do
+         cm_ref <- MessageReference.for_chat_and_object(chat, message) do
       conn
-      |> put_view(ChatMessageReferenceView)
+      |> put_view(MessageReferenceView)
       |> render("show.json", for: user, chat_message_reference: cm_ref)
     end
   end
@@ -98,20 +98,20 @@ def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
         id: chat_id,
         message_id: message_id
       }) do
-    with %ChatMessageReference{} = cm_ref <-
-           ChatMessageReference.get_by_id(message_id),
+    with %MessageReference{} = cm_ref <-
+           MessageReference.get_by_id(message_id),
          ^chat_id <- cm_ref.chat_id |> to_string(),
          %Chat{user_id: ^user_id} <- Chat.get_by_id(chat_id),
-         {:ok, cm_ref} <- ChatMessageReference.mark_as_read(cm_ref) do
+         {:ok, cm_ref} <- MessageReference.mark_as_read(cm_ref) do
       conn
-      |> put_view(ChatMessageReferenceView)
+      |> put_view(MessageReferenceView)
       |> render("show.json", for: user, chat_message_reference: cm_ref)
     end
   end
 
   def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
-         {_n, _} <- ChatMessageReference.set_all_seen_for_chat(chat) do
+         {_n, _} <- MessageReference.set_all_seen_for_chat(chat) do
       conn
       |> put_view(ChatView)
       |> render("show.json", chat: chat)
@@ -122,11 +122,11 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = para
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       cm_refs =
         chat
-        |> ChatMessageReference.for_chat_query()
+        |> MessageReference.for_chat_query()
         |> Pagination.fetch_paginated(params |> stringify_keys())
 
       conn
-      |> put_view(ChatMessageReferenceView)
+      |> put_view(MessageReferenceView)
       |> render("index.json", for: user, chat_message_references: cm_refs)
     else
       _ ->
diff --git a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
similarity index 95%
rename from lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
rename to lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
index 592bb17f0..f2112a86e 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_message_reference_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceView do
+defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
   use Pleroma.Web, :view
 
   alias Pleroma.User
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 91d50dd1e..d4c10977f 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -6,24 +6,24 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   use Pleroma.Web, :view
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
 
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
 
-    last_message = opts[:last_message] || ChatMessageReference.last_message_for_chat(chat)
+    last_message = opts[:last_message] || MessageReference.last_message_for_chat(chat)
 
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
-      unread: Map.get(chat, :unread) || ChatMessageReference.unread_count_for_chat(chat),
+      unread: Map.get(chat, :unread) || MessageReference.unread_count_for_chat(chat),
       last_message:
         last_message &&
-          ChatMessageReferenceView.render("show.json", chat_message_reference: last_message),
+          MessageReferenceView.render("show.json", chat_message_reference: last_message),
       updated_at: Utils.to_masto_date(chat.updated_at)
     }
   end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index b22297955..d1d2c9b9c 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Streamer do
   require Logger
 
   alias Pleroma.Activity
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
@@ -187,7 +187,7 @@ defp do_stream(topic, %Notification{} = item)
     end)
   end
 
-  defp do_stream(topic, {user, %ChatMessageReference{} = cm_ref})
+  defp do_stream(topic, {user, %MessageReference{} = cm_ref})
        when topic in ["user", "user:pleroma_chat"] do
     topic = "#{topic}:#{user.id}"
 
diff --git a/test/chat_message_reference_test.exs b/test/chat/message_reference_test.exs
similarity index 82%
rename from test/chat_message_reference_test.exs
rename to test/chat/message_reference_test.exs
index 66bf493b4..aaa7c1ad4 100644
--- a/test/chat_message_reference_test.exs
+++ b/test/chat/message_reference_test.exs
@@ -2,11 +2,11 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ChatMessageReferenceTest do
+defmodule Pleroma.Chat.MessageReferenceTest do
   use Pleroma.DataCase, async: true
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
@@ -21,7 +21,7 @@ test "it returns the last message in a chat" do
 
       {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
 
-      message = ChatMessageReference.last_message_for_chat(chat)
+      message = MessageReference.last_message_for_chat(chat)
 
       assert message.object.data["content"] == "ho"
     end
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index 43ffe1337..b1afa6a2e 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -8,7 +8,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -391,7 +391,7 @@ test "it streams the created ChatMessage" do
       end
     end
 
-    test "it creates a Chat and ChatMessageReferences for the local users and bumps the unread count, except for the author" do
+    test "it creates a Chat and MessageReferences for the local users and bumps the unread count, except for the author" do
       author = insert(:user, local: true)
       recipient = insert(:user, local: true)
 
@@ -431,14 +431,14 @@ test "it creates a Chat and ChatMessageReferences for the local users and bumps
 
         chat = Chat.get(author.id, recipient.ap_id)
 
-        [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+        [cm_ref] = MessageReference.for_chat_query(chat) |> Repo.all()
 
         assert cm_ref.object.data["content"] == "hey"
         assert cm_ref.unread == false
 
         chat = Chat.get(recipient.id, author.ap_id)
 
-        [cm_ref] = ChatMessageReference.for_chat_query(chat) |> Repo.all()
+        [cm_ref] = MessageReference.for_chat_query(chat) |> Repo.all()
 
         assert cm_ref.object.data["content"] == "hey"
         assert cm_ref.unread == true
diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs
index c5691341a..b2fa5b302 100644
--- a/test/web/mastodon_api/views/notification_view_test.exs
+++ b/test/web/mastodon_api/views/notification_view_test.exs
@@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
 
   alias Pleroma.Activity
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.StatusView
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
   import Pleroma.Factory
 
   defp test_notifications_rendering(notifications, user, expected_result) do
@@ -45,15 +45,14 @@ test "ChatMessage notification" do
     object = Object.normalize(activity)
     chat = Chat.get(recipient.id, user.ap_id)
 
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+    cm_ref = MessageReference.for_chat_and_object(chat, object)
 
     expected = %{
       id: to_string(notification.id),
       pleroma: %{is_seen: false},
       type: "pleroma:chat_mention",
       account: AccountView.render("show.json", %{user: user, for: recipient}),
-      chat_message:
-        ChatMessageReferenceView.render("show.json", %{chat_message_reference: cm_ref}),
+      chat_message: MessageReferenceView.render("show.json", %{chat_message_reference: cm_ref}),
       created_at: Utils.to_masto_date(notification.inserted_at)
     }
 
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 7af6dec1c..e73e4a32e 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   use Pleroma.Web.ConnCase, async: true
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -23,7 +23,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
       {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
       object = Object.normalize(create, false)
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == true
 
@@ -34,7 +34,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
 
       assert result["unread"] == false
 
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == false
     end
@@ -50,7 +50,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
       {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
       object = Object.normalize(create, false)
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == true
 
@@ -61,7 +61,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
 
       assert result["unread"] == 0
 
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == false
     end
@@ -139,7 +139,7 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
 
       chat = Chat.get(user.id, recipient.ap_id)
 
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       # Deleting your own message removes the message and the reference
       result =
@@ -149,12 +149,12 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
         |> json_response_and_validate_schema(200)
 
       assert result["id"] == cm_ref.id
-      refute ChatMessageReference.get_by_id(cm_ref.id)
+      refute MessageReference.get_by_id(cm_ref.id)
       assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id)
 
       # Deleting other people's messages just removes the reference
       object = Object.normalize(other_message, false)
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       result =
         conn
@@ -163,7 +163,7 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
         |> json_response_and_validate_schema(200)
 
       assert result["id"] == cm_ref.id
-      refute ChatMessageReference.get_by_id(cm_ref.id)
+      refute MessageReference.get_by_id(cm_ref.id)
       assert Object.get_by_id(object.id)
     end
   end
diff --git a/test/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/web/pleroma_api/views/chat/message_reference_view_test.exs
similarity index 77%
rename from test/web/pleroma_api/views/chat_message_reference_view_test.exs
rename to test/web/pleroma_api/views/chat/message_reference_view_test.exs
index b53bd3490..e5b165255 100644
--- a/test/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/web/pleroma_api/views/chat/message_reference_view_test.exs
@@ -2,15 +2,15 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
+defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
 
   import Pleroma.Factory
 
@@ -31,9 +31,9 @@ test "it displays a chat message" do
 
     object = Object.normalize(activity)
 
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+    cm_ref = MessageReference.for_chat_and_object(chat, object)
 
-    chat_message = ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+    chat_message = MessageReferenceView.render("show.json", chat_message_reference: cm_ref)
 
     assert chat_message[:id] == cm_ref.id
     assert chat_message[:content] == "kippis :firefox:"
@@ -47,10 +47,9 @@ test "it displays a chat message" do
 
     object = Object.normalize(activity)
 
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+    cm_ref = MessageReference.for_chat_and_object(chat, object)
 
-    chat_message_two =
-      ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+    chat_message_two = MessageReferenceView.render("show.json", chat_message_reference: cm_ref)
 
     assert chat_message_two[:id] == cm_ref.id
     assert chat_message_two[:content] == "gkgkgk"
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index f77584dd1..f7af5d4e0 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -6,12 +6,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.PleromaAPI.ChatMessageReferenceView
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
   alias Pleroma.Web.PleromaAPI.ChatView
 
   import Pleroma.Factory
@@ -55,9 +55,9 @@ test "it represents a chat" do
 
     represented_chat = ChatView.render("show.json", chat: chat)
 
-    cm_ref = ChatMessageReference.for_chat_and_object(chat, chat_message)
+    cm_ref = MessageReference.for_chat_and_object(chat, chat_message)
 
     assert represented_chat[:last_message] ==
-             ChatMessageReferenceView.render("show.json", chat_message_reference: cm_ref)
+             MessageReferenceView.render("show.json", chat_message_reference: cm_ref)
   end
 end
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index 893ae5449..245f6e63f 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -8,7 +8,7 @@ defmodule Pleroma.Web.StreamerTest do
   import Pleroma.Factory
 
   alias Pleroma.Chat
-  alias Pleroma.ChatMessageReference
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.Conversation.Participation
   alias Pleroma.List
   alias Pleroma.Object
@@ -155,7 +155,7 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} d
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
       object = Object.normalize(create_activity, false)
       chat = Chat.get(user.id, other_user.ap_id)
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
       Streamer.get_topic_and_add_socket("user:pleroma_chat", user)
@@ -173,7 +173,7 @@ test "it sends chat messages to the 'user' stream", %{user: user} do
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
       object = Object.normalize(create_activity, false)
       chat = Chat.get(user.id, other_user.ap_id)
-      cm_ref = ChatMessageReference.for_chat_and_object(chat, object)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
       Streamer.get_topic_and_add_socket("user", user)

From 9fa3f0b156f92ba575b58b191685fa068a83f4d2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 13:08:45 +0200
Subject: [PATCH 160/283] Notification: Change type of `type` to an enum.

---
 lib/pleroma/notification.ex                   |  3 ++
 ..._change_type_to_enum_for_notifications.exs | 36 +++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 priv/repo/migrations/20200606105430_change_type_to_enum_for_notifications.exs

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 49e27c05a..5c8994e35 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -30,6 +30,9 @@ defmodule Pleroma.Notification do
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
+    # This is an enum type in the database. If you add a new notification type,
+    # remembert to add a migration to add it to the `notifications_type` enum
+    # as well.
     field(:type, :string)
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
diff --git a/priv/repo/migrations/20200606105430_change_type_to_enum_for_notifications.exs b/priv/repo/migrations/20200606105430_change_type_to_enum_for_notifications.exs
new file mode 100644
index 000000000..9ea34436b
--- /dev/null
+++ b/priv/repo/migrations/20200606105430_change_type_to_enum_for_notifications.exs
@@ -0,0 +1,36 @@
+defmodule Pleroma.Repo.Migrations.ChangeTypeToEnumForNotifications do
+  use Ecto.Migration
+
+  def up do
+    """
+    create type notification_type as enum (
+      'follow',
+      'follow_request',
+      'mention',
+      'move',
+      'pleroma:emoji_reaction',
+      'pleroma:chat_mention',
+      'reblog',
+      'favourite'
+    )
+    """
+    |> execute()
+
+    """
+    alter table notifications 
+    alter column type type notification_type using (type::notification_type)
+    """
+    |> execute()
+  end
+
+  def down do
+    alter table(:notifications) do
+      modify(:type, :string)
+    end
+
+    """
+    drop type notification_type
+    """
+    |> execute()
+  end
+end

From 9189b489eef29be723389e1b3642a843bc0d01bc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 15:33:02 +0200
Subject: [PATCH 161/283] Migrations: Move Notification migration code to
 helper

---
 lib/pleroma/migration_helper.ex               | 85 +++++++++++++++++++
 lib/pleroma/notification.ex                   | 37 +-------
 ...0602125218_backfill_notification_types.exs |  2 +-
 test/migration_helper_test.exs                | 56 ++++++++++++
 test/notification_test.exs                    | 42 ---------
 5 files changed, 144 insertions(+), 78 deletions(-)
 create mode 100644 lib/pleroma/migration_helper.ex
 create mode 100644 test/migration_helper_test.exs

diff --git a/lib/pleroma/migration_helper.ex b/lib/pleroma/migration_helper.ex
new file mode 100644
index 000000000..e6346aff1
--- /dev/null
+++ b/lib/pleroma/migration_helper.ex
@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MigrationHelper do
+  alias Pleroma.User
+  alias Pleroma.Object
+  alias Pleroma.Notification
+  alias Pleroma.Repo
+
+  import Ecto.Query
+
+  def fill_in_notification_types do
+    query =
+      from(n in Pleroma.Notification,
+        where: is_nil(n.type),
+        preload: :activity
+      )
+
+    query
+    |> Repo.all()
+    |> Enum.each(fn notification ->
+      type =
+        notification.activity
+        |> type_from_activity()
+
+      notification
+      |> Notification.changeset(%{type: type})
+      |> Repo.update()
+    end)
+  end
+
+  # This is copied over from Notifications to keep this stable.
+  defp type_from_activity(%{data: %{"type" => type}} = activity) do
+    case type do
+      "Follow" ->
+        accepted_function = fn activity ->
+          with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]),
+               %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do
+            Pleroma.FollowingRelationship.following?(follower, followed)
+          end
+        end
+
+        if accepted_function.(activity) do
+          "follow"
+        else
+          "follow_request"
+        end
+
+      "Announce" ->
+        "reblog"
+
+      "Like" ->
+        "favourite"
+
+      "Move" ->
+        "move"
+
+      "EmojiReact" ->
+        "pleroma:emoji_reaction"
+
+      # Compatibility with old reactions
+      "EmojiReaction" ->
+        "pleroma:emoji_reaction"
+
+      "Create" ->
+        activity
+        |> type_from_activity_object()
+
+      t ->
+        raise "No notification type for activity type #{t}"
+    end
+  end
+
+  defp type_from_activity_object(%{data: %{"type" => "Create", "object" => %{}}}), do: "mention"
+
+  defp type_from_activity_object(%{data: %{"type" => "Create"}} = activity) do
+    object = Object.get_by_ap_id(activity.data["object"])
+
+    case object && object.data["type"] do
+      "ChatMessage" -> "pleroma:chat_mention"
+      _ -> "mention"
+    end
+  end
+end
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 5c8994e35..682a26912 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -40,26 +40,6 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
-  def fill_in_notification_types do
-    query =
-      from(n in __MODULE__,
-        where: is_nil(n.type),
-        preload: :activity
-      )
-
-    query
-    |> Repo.all()
-    |> Enum.each(fn notification ->
-      type =
-        notification.activity
-        |> type_from_activity(no_cachex: true)
-
-      notification
-      |> changeset(%{type: type})
-      |> Repo.update()
-    end)
-  end
-
   def update_notification_type(user, activity) do
     with %__MODULE__{} = notification <-
            Repo.get_by(__MODULE__, user_id: user.id, activity_id: activity.id) do
@@ -371,23 +351,10 @@ defp do_create_notifications(%Activity{} = activity, options) do
     {:ok, notifications}
   end
 
-  defp type_from_activity(%{data: %{"type" => type}} = activity, opts \\ []) do
+  defp type_from_activity(%{data: %{"type" => type}} = activity) do
     case type do
       "Follow" ->
-        accepted_function =
-          if Keyword.get(opts, :no_cachex, false) do
-            # A special function to make this usable in a migration.
-            fn activity ->
-              with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]),
-                   %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do
-                Pleroma.FollowingRelationship.following?(follower, followed)
-              end
-            end
-          else
-            &Activity.follow_accepted?/1
-          end
-
-        if accepted_function.(activity) do
+        if Activity.follow_accepted?(activity) do
           "follow"
         else
           "follow_request"
diff --git a/priv/repo/migrations/20200602125218_backfill_notification_types.exs b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
index 493c0280c..58943fad0 100644
--- a/priv/repo/migrations/20200602125218_backfill_notification_types.exs
+++ b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
@@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.BackfillNotificationTypes do
   use Ecto.Migration
 
   def up do
-    Pleroma.Notification.fill_in_notification_types()
+    Pleroma.MigrationHelper.fill_in_notification_types()
   end
 
   def down do
diff --git a/test/migration_helper_test.exs b/test/migration_helper_test.exs
new file mode 100644
index 000000000..1c8173987
--- /dev/null
+++ b/test/migration_helper_test.exs
@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MigrationHelperTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.MigrationHelper
+  alias Pleroma.Notification
+  alias Pleroma.Repo
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  describe "fill_in_notification_types" do
+    test "it fills in missing notification types" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, post} = CommonAPI.post(user, %{status: "yeah, @#{other_user.nickname}"})
+      {:ok, chat} = CommonAPI.post_chat_message(user, other_user, "yo")
+      {:ok, react} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
+      {:ok, like} = CommonAPI.favorite(other_user, post.id)
+      {:ok, react_2} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
+
+      data =
+        react_2.data
+        |> Map.put("type", "EmojiReaction")
+
+      {:ok, react_2} =
+        react_2
+        |> Activity.change(%{data: data})
+        |> Repo.update()
+
+      assert {5, nil} = Repo.update_all(Notification, set: [type: nil])
+
+      MigrationHelper.fill_in_notification_types()
+
+      assert %{type: "mention"} =
+               Repo.get_by(Notification, user_id: other_user.id, activity_id: post.id)
+
+      assert %{type: "favourite"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: like.id)
+
+      assert %{type: "pleroma:emoji_reaction"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: react.id)
+
+      assert %{type: "pleroma:emoji_reaction"} =
+               Repo.get_by(Notification, user_id: user.id, activity_id: react_2.id)
+
+      assert %{type: "pleroma:chat_mention"} =
+               Repo.get_by(Notification, user_id: other_user.id, activity_id: chat.id)
+    end
+  end
+end
diff --git a/test/notification_test.exs b/test/notification_test.exs
index f2115a29e..b9bbdceca 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.NotificationTest do
   import Pleroma.Factory
   import Mock
 
-  alias Pleroma.Activity
   alias Pleroma.FollowingRelationship
   alias Pleroma.Notification
   alias Pleroma.Repo
@@ -22,47 +21,6 @@ defmodule Pleroma.NotificationTest do
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
 
-  describe "fill_in_notification_types" do
-    test "it fills in missing notification types" do
-      user = insert(:user)
-      other_user = insert(:user)
-
-      {:ok, post} = CommonAPI.post(user, %{status: "yeah, @#{other_user.nickname}"})
-      {:ok, chat} = CommonAPI.post_chat_message(user, other_user, "yo")
-      {:ok, react} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
-      {:ok, like} = CommonAPI.favorite(other_user, post.id)
-      {:ok, react_2} = CommonAPI.react_with_emoji(post.id, other_user, "☕")
-
-      data =
-        react_2.data
-        |> Map.put("type", "EmojiReaction")
-
-      {:ok, react_2} =
-        react_2
-        |> Activity.change(%{data: data})
-        |> Repo.update()
-
-      assert {5, nil} = Repo.update_all(Notification, set: [type: nil])
-
-      Notification.fill_in_notification_types()
-
-      assert %{type: "mention"} =
-               Repo.get_by(Notification, user_id: other_user.id, activity_id: post.id)
-
-      assert %{type: "favourite"} =
-               Repo.get_by(Notification, user_id: user.id, activity_id: like.id)
-
-      assert %{type: "pleroma:emoji_reaction"} =
-               Repo.get_by(Notification, user_id: user.id, activity_id: react.id)
-
-      assert %{type: "pleroma:emoji_reaction"} =
-               Repo.get_by(Notification, user_id: user.id, activity_id: react_2.id)
-
-      assert %{type: "pleroma:chat_mention"} =
-               Repo.get_by(Notification, user_id: other_user.id, activity_id: chat.id)
-    end
-  end
-
   describe "create_notifications" do
     test "creates a notification for an emoji reaction" do
       user = insert(:user)

From f77d4a302d8ee5999979136290caac556cac8873 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 15:51:08 +0200
Subject: [PATCH 162/283] Credo fixes.

---
 lib/pleroma/migration_helper.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/migration_helper.ex b/lib/pleroma/migration_helper.ex
index e6346aff1..a20d27a01 100644
--- a/lib/pleroma/migration_helper.ex
+++ b/lib/pleroma/migration_helper.ex
@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MigrationHelper do
-  alias Pleroma.User
-  alias Pleroma.Object
   alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.User
 
   import Ecto.Query
 

From e1b07402ab077899dd5b9c0023fbe1c48af259e9 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 23 Mar 2020 22:52:25 +0100
Subject: [PATCH 163/283] User: Add raw_bio, storing unformatted bio

Related: https://git.pleroma.social/pleroma/pleroma/issues/1643
---
 lib/pleroma/user.ex                           | 13 +++++++++-
 .../controllers/account_controller.ex         |  1 +
 .../web/mastodon_api/views/account_view.ex    | 13 +---------
 .../20200322174133_user_raw_bio.exs           |  9 +++++++
 .../20200328193433_populate_user_raw_bio.exs  | 25 +++++++++++++++++++
 test/support/factory.ex                       |  3 ++-
 .../update_credentials_test.exs               | 13 +++++++---
 .../mastodon_api/views/account_view_test.exs  |  3 ++-
 8 files changed, 61 insertions(+), 19 deletions(-)
 create mode 100644 priv/repo/migrations/20200322174133_user_raw_bio.exs
 create mode 100644 priv/repo/migrations/20200328193433_populate_user_raw_bio.exs

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 72ee2d58e..23ca8c9f3 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -79,6 +79,7 @@ defmodule Pleroma.User do
 
   schema "users" do
     field(:bio, :string)
+    field(:raw_bio, :string)
     field(:email, :string)
     field(:name, :string)
     field(:nickname, :string)
@@ -432,6 +433,7 @@ def update_changeset(struct, params \\ %{}) do
       params,
       [
         :bio,
+        :raw_bio,
         :name,
         :emoji,
         :avatar,
@@ -607,7 +609,16 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
 
     struct
     |> confirmation_changeset(need_confirmation: need_confirmation?)
-    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji])
+    |> cast(params, [
+      :bio,
+      :raw_bio,
+      :email,
+      :name,
+      :nickname,
+      :password,
+      :password_confirmation,
+      :emoji
+    ])
     |> validate_required([:name, :nickname, :password, :password_confirmation])
     |> validate_confirmation(:password)
     |> unique_constraint(:email)
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 5734bb854..ebfa533dd 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -165,6 +165,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
       end)
       |> Maps.put_if_present(:name, params[:display_name])
       |> Maps.put_if_present(:bio, params[:note])
+      |> Maps.put_if_present(:raw_bio, params[:note])
       |> Maps.put_if_present(:avatar, params[:avatar])
       |> Maps.put_if_present(:banner, params[:header])
       |> Maps.put_if_present(:background, params[:pleroma_background_image])
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 04c419d2f..5326b02c6 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -224,7 +224,7 @@ defp do_render("show.json", %{user: user} = opts) do
       fields: user.fields,
       bot: bot,
       source: %{
-        note: prepare_user_bio(user),
+        note: user.raw_bio || "",
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
@@ -259,17 +259,6 @@ defp do_render("show.json", %{user: user} = opts) do
     |> maybe_put_unread_notification_count(user, opts[:for])
   end
 
-  defp prepare_user_bio(%User{bio: ""}), do: ""
-
-  defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do
-    bio
-    |> String.replace(~r(<br */?>), "\n")
-    |> Pleroma.HTML.strip_tags()
-    |> HtmlEntities.decode()
-  end
-
-  defp prepare_user_bio(_), do: ""
-
   defp username_from_nickname(string) when is_binary(string) do
     hd(String.split(string, "@"))
   end
diff --git a/priv/repo/migrations/20200322174133_user_raw_bio.exs b/priv/repo/migrations/20200322174133_user_raw_bio.exs
new file mode 100644
index 000000000..ddf9be4f5
--- /dev/null
+++ b/priv/repo/migrations/20200322174133_user_raw_bio.exs
@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.UserRawBio do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add_if_not_exists(:raw_bio, :text)
+    end
+  end
+end
diff --git a/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs
new file mode 100644
index 000000000..cb35db3f5
--- /dev/null
+++ b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs
@@ -0,0 +1,25 @@
+defmodule Pleroma.Repo.Migrations.PopulateUserRawBio do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def change do
+    {:ok, _} = Application.ensure_all_started(:fast_sanitize)
+
+    User.Query.build(%{local: true})
+    |> select([u], struct(u, [:id, :ap_id, :bio]))
+    |> Repo.stream()
+    |> Enum.each(fn %{bio: bio} = user ->
+      if bio do
+        raw_bio =
+          bio
+          |> String.replace(~r(<br */?>), "\n")
+          |> Pleroma.HTML.strip_tags()
+
+        Ecto.Changeset.cast(user, %{raw_bio: raw_bio}, [:raw_bio])
+        |> Repo.update()
+      end
+    end)
+  end
+end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 6e3676aca..1a9b96180 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -42,7 +42,8 @@ def user_factory do
       user
       | ap_id: User.ap_id(user),
         follower_address: User.ap_followers(user),
-        following_address: User.ap_following(user)
+        following_address: User.ap_following(user),
+        raw_bio: user.bio
     }
   end
 
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index 7c420985d..76e6d603a 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -83,10 +83,9 @@ test "sets user settings in a generic way", %{conn: conn} do
     test "updates the user's bio", %{conn: conn} do
       user2 = insert(:user)
 
-      conn =
-        patch(conn, "/api/v1/accounts/update_credentials", %{
-          "note" => "I drink #cofe with @#{user2.nickname}\n\nsuya.."
-        })
+      raw_bio = "I drink #cofe with @#{user2.nickname}\n\nsuya.."
+
+      conn = patch(conn, "/api/v1/accounts/update_credentials", %{"note" => raw_bio})
 
       assert user_data = json_response_and_validate_schema(conn, 200)
 
@@ -94,6 +93,12 @@ test "updates the user's bio", %{conn: conn} do
                ~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe">#cofe</a> with <span class="h-card"><a class="u-url mention" data-user="#{
                  user2.id
                }" href="#{user2.ap_id}" rel="ugc">@<span>#{user2.nickname}</span></a></span><br/><br/>suya..)
+
+      assert user_data["source"]["note"] == raw_bio
+
+      user = Repo.get(User, user_data["id"])
+
+      assert user.raw_bio == raw_bio
     end
 
     test "updates the user's locking status", %{conn: conn} do
diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs
index f91333e5c..7ac70dc58 100644
--- a/test/web/mastodon_api/views/account_view_test.exs
+++ b/test/web/mastodon_api/views/account_view_test.exs
@@ -33,7 +33,8 @@ test "Represent a user account" do
         bio:
           "<script src=\"invalid-html\"></script><span>valid html</span>. a<br>b<br/>c<br >d<br />f '&<>\"",
         inserted_at: ~N[2017-08-15 15:47:06.597036],
-        emoji: %{"karjalanpiirakka" => "/file.png"}
+        emoji: %{"karjalanpiirakka" => "/file.png"},
+        raw_bio: "valid html. a\nb\nc\nd\nf '&<>\""
       })
 
     expected = %{

From f4cf4ae16ee84655bf6630cf7e98e9eef2f410cc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 16:48:02 +0200
Subject: [PATCH 164/283] ChatController: Use new oauth scope *:chats.

---
 .../web/api_spec/operations/chat_operation.ex    | 14 +++++++-------
 .../pleroma_api/controllers/chat_controller.ex   |  4 ++--
 .../controllers/chat_controller_test.exs         | 16 ++++++++--------
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 6ad325113..74c3ad0bd 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -33,7 +33,7 @@ def mark_as_read_operation do
       },
       security: [
         %{
-          "oAuth" => ["write"]
+          "oAuth" => ["write:chats"]
         }
       ]
     }
@@ -58,7 +58,7 @@ def mark_message_as_read_operation do
       },
       security: [
         %{
-          "oAuth" => ["write"]
+          "oAuth" => ["write:chats"]
         }
       ]
     }
@@ -120,7 +120,7 @@ def create_operation do
       },
       security: [
         %{
-          "oAuth" => ["write"]
+          "oAuth" => ["write:chats"]
         }
       ]
     }
@@ -137,7 +137,7 @@ def index_operation do
       },
       security: [
         %{
-          "oAuth" => ["read"]
+          "oAuth" => ["read:chats"]
         }
       ]
     }
@@ -161,7 +161,7 @@ def messages_operation do
       },
       security: [
         %{
-          "oAuth" => ["read"]
+          "oAuth" => ["read:chats"]
         }
       ]
     }
@@ -187,7 +187,7 @@ def post_chat_message_operation do
       },
       security: [
         %{
-          "oAuth" => ["write"]
+          "oAuth" => ["write:chats"]
         }
       ]
     }
@@ -212,7 +212,7 @@ def delete_message_operation do
       },
       security: [
         %{
-          "oAuth" => ["write"]
+          "oAuth" => ["write:chats"]
         }
       ]
     }
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index d6b3415d1..983550b13 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["write:statuses"]}
+    %{scopes: ["write:chats"]}
     when action in [
            :post_chat_message,
            :create,
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["read:statuses"]} when action in [:messages, :index, :show]
+    %{scopes: ["read:chats"]} when action in [:messages, :index, :show]
   )
 
   plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index e73e4a32e..2128fd9dd 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
   import Pleroma.Factory
 
   describe "POST /api/v1/pleroma/chats/:id/messages/:message_id/read" do
-    setup do: oauth_access(["write:statuses"])
+    setup do: oauth_access(["write:chats"])
 
     test "it marks one message as read", %{conn: conn, user: user} do
       other_user = insert(:user)
@@ -41,7 +41,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
   end
 
   describe "POST /api/v1/pleroma/chats/:id/read" do
-    setup do: oauth_access(["write:statuses"])
+    setup do: oauth_access(["write:chats"])
 
     test "it marks all messages in a chat as read", %{conn: conn, user: user} do
       other_user = insert(:user)
@@ -68,7 +68,7 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
   end
 
   describe "POST /api/v1/pleroma/chats/:id/messages" do
-    setup do: oauth_access(["write:statuses"])
+    setup do: oauth_access(["write:chats"])
 
     test "it posts a message to the chat", %{conn: conn, user: user} do
       other_user = insert(:user)
@@ -125,7 +125,7 @@ test "it works with an attachment", %{conn: conn, user: user} do
   end
 
   describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do
-    setup do: oauth_access(["write:statuses"])
+    setup do: oauth_access(["write:chats"])
 
     test "it deletes a message from the chat", %{conn: conn, user: user} do
       recipient = insert(:user)
@@ -169,7 +169,7 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
   end
 
   describe "GET /api/v1/pleroma/chats/:id/messages" do
-    setup do: oauth_access(["read:statuses"])
+    setup do: oauth_access(["read:chats"])
 
     test "it paginates", %{conn: conn, user: user} do
       recipient = insert(:user)
@@ -229,7 +229,7 @@ test "it returns the messages for a given chat", %{conn: conn, user: user} do
   end
 
   describe "POST /api/v1/pleroma/chats/by-account-id/:id" do
-    setup do: oauth_access(["write:statuses"])
+    setup do: oauth_access(["write:chats"])
 
     test "it creates or returns a chat", %{conn: conn} do
       other_user = insert(:user)
@@ -244,7 +244,7 @@ test "it creates or returns a chat", %{conn: conn} do
   end
 
   describe "GET /api/v1/pleroma/chats/:id" do
-    setup do: oauth_access(["read:statuses"])
+    setup do: oauth_access(["read:chats"])
 
     test "it returns a chat", %{conn: conn, user: user} do
       other_user = insert(:user)
@@ -261,7 +261,7 @@ test "it returns a chat", %{conn: conn, user: user} do
   end
 
   describe "GET /api/v1/pleroma/chats" do
-    setup do: oauth_access(["read:statuses"])
+    setup do: oauth_access(["read:chats"])
 
     test "it does not return chats with users you blocked", %{conn: conn, user: user} do
       recipient = insert(:user)

From 40fc4e974e5f60c3d61702b17029566774898e84 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 6 Jun 2020 16:59:08 +0200
Subject: [PATCH 165/283] Notfication: Add validation of notification types

---
 lib/pleroma/notification.ex | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 682a26912..3ac8737e2 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -61,9 +61,21 @@ def unread_notifications_count(%User{id: user_id}) do
     |> Repo.aggregate(:count, :id)
   end
 
+  @notification_types ~w{
+    favourite
+    follow
+    follow_request
+    mention
+    move
+    pleroma:chat_mention
+    pleroma:emoji_reaction
+    reblog
+  }
+
   def changeset(%Notification{} = notification, attrs) do
     notification
     |> cast(attrs, [:seen, :type])
+    |> validate_inclusion(:type, @notification_types)
   end
 
   @spec last_read_query(User.t()) :: Ecto.Queryable.t()

From 0365053c8dbbcae4a4883f68b7eaec263c14f656 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 09:19:00 +0200
Subject: [PATCH 166/283] AttachmentValidator: Check if the mime type is valid.

---
 .../activity_pub/object_validators/attachment_validator.ex    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index c4b502cb9..f53bb02be 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -45,11 +45,11 @@ def fix_media_type(data) do
       data
       |> Map.put_new("mediaType", data["mimeType"])
 
-    if data["mediaType"] == "" do
+    if MIME.valid?(data["mediaType"]) do
       data
-      |> Map.put("mediaType", "application/octet-stream")
     else
       data
+      |> Map.put("mediaType", "application/octet-stream")
     end
   end
 

From 1a11f0e453527070a8ab5511318045470abc95e2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 14:25:30 +0200
Subject: [PATCH 167/283] Chats: Change id to flake id.

---
 lib/pleroma/chat.ex                           |  3 +++
 lib/pleroma/chat/message_reference.ex         |  2 +-
 ...20200607112923_change_chat_id_to_flake.exs | 23 +++++++++++++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 priv/repo/migrations/20200607112923_change_chat_id_to_flake.exs

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 4fe31de94..24a86371e 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -16,6 +16,8 @@ defmodule Pleroma.Chat do
   It is a helper only, to make it easy to display a list of chats with other people, ordered by last bump. The actual messages are retrieved by querying the recipients of the ChatMessages.
   """
 
+  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+
   schema "chats" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     field(:recipient, :string)
@@ -63,6 +65,7 @@ def bump_or_create(user_id, recipient) do
     |> changeset(%{user_id: user_id, recipient: recipient})
     |> Repo.insert(
       on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
+      returning: true,
       conflict_target: [:user_id, :recipient]
     )
   end
diff --git a/lib/pleroma/chat/message_reference.ex b/lib/pleroma/chat/message_reference.ex
index 4b201db2e..7ee7508ca 100644
--- a/lib/pleroma/chat/message_reference.ex
+++ b/lib/pleroma/chat/message_reference.ex
@@ -21,7 +21,7 @@ defmodule Pleroma.Chat.MessageReference do
 
   schema "chat_message_references" do
     belongs_to(:object, Object)
-    belongs_to(:chat, Chat)
+    belongs_to(:chat, Chat, type: FlakeId.Ecto.CompatType)
 
     field(:unread, :boolean, default: true)
 
diff --git a/priv/repo/migrations/20200607112923_change_chat_id_to_flake.exs b/priv/repo/migrations/20200607112923_change_chat_id_to_flake.exs
new file mode 100644
index 000000000..f14e269ca
--- /dev/null
+++ b/priv/repo/migrations/20200607112923_change_chat_id_to_flake.exs
@@ -0,0 +1,23 @@
+defmodule Pleroma.Repo.Migrations.ChangeChatIdToFlake do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    alter table chats
+    drop constraint chats_pkey cascade,
+    alter column id drop default,
+    alter column id set data type uuid using cast( lpad( to_hex(id), 32, '0') as uuid),
+    add primary key (id)
+    """)
+
+    execute("""
+    alter table chat_message_references
+    alter column chat_id set data type uuid using cast( lpad( to_hex(chat_id), 32, '0') as uuid),
+    add constraint chat_message_references_chat_id_fkey foreign key (chat_id) references chats(id) on delete cascade
+    """)
+  end
+
+  def down do
+    :ok
+  end
+end

From 2cdaac433035d8df3890eae098b55380b9e1c9fc Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 14:52:56 +0200
Subject: [PATCH 168/283] SideEffects: Move streaming of chats to after the
 transaction.

---
 lib/pleroma/web/activity_pub/side_effects.ex  | 59 ++++++++++++-------
 .../web/pleroma_api/views/chat_view.ex        |  3 +-
 test/web/activity_pub/side_effects_test.exs   | 41 ++++---------
 test/web/common_api/common_api_test.exs       |  1 +
 test/web/pleroma_api/views/chat_view_test.exs | 15 -----
 5 files changed, 52 insertions(+), 67 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 1e9d6c2fc..1a1cc675c 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -37,7 +37,7 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
   # - Rollback if we couldn't create it
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
-    with {:ok, _object, _meta} <- handle_object_creation(meta[:object_data], meta) do
+    with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
 
       meta =
@@ -142,24 +142,24 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
       actor = User.get_cached_by_ap_id(object.data["actor"])
       recipient = User.get_cached_by_ap_id(hd(object.data["to"]))
 
-      [[actor, recipient], [recipient, actor]]
-      |> Enum.each(fn [user, other_user] ->
-        if user.local do
-          {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
-          {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
+      streamables =
+        [[actor, recipient], [recipient, actor]]
+        |> Enum.map(fn [user, other_user] ->
+          if user.local do
+            {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
+            {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
-          # We add a cache of the unread value here so that it
-          # doesn't change when being streamed out
-          chat =
-            chat
-            |> Map.put(:unread, MessageReference.unread_count_for_chat(chat))
+            {
+              ["user", "user:pleroma_chat"],
+              {user, %{cm_ref | chat: chat, object: object}}
+            }
+          end
+        end)
+        |> Enum.filter(& &1)
 
-          Streamer.stream(
-            ["user", "user:pleroma_chat"],
-            {user, %{cm_ref | chat: chat, object: object}}
-          )
-        end
-      end)
+      meta =
+        meta
+        |> add_streamables(streamables)
 
       {:ok, object, meta}
     end
@@ -208,7 +208,7 @@ def handle_undoing(
   def handle_undoing(object), do: {:error, ["don't know how to handle", object]}
 
   defp send_notifications(meta) do
-    Keyword.get(meta, :created_notifications, [])
+    Keyword.get(meta, :notifications, [])
     |> Enum.each(fn notification ->
       Streamer.stream(["user", "user:notification"], notification)
       Push.send(notification)
@@ -217,15 +217,32 @@ defp send_notifications(meta) do
     meta
   end
 
-  defp add_notifications(meta, notifications) do
-    existing = Keyword.get(meta, :created_notifications, [])
+  defp send_streamables(meta) do
+    Keyword.get(meta, :streamables, [])
+    |> Enum.each(fn {topics, items} ->
+      Streamer.stream(topics, items)
+    end)
 
     meta
-    |> Keyword.put(:created_notifications, notifications ++ existing)
+  end
+
+  defp add_streamables(meta, streamables) do
+    existing = Keyword.get(meta, :streamables, [])
+
+    meta
+    |> Keyword.put(:streamables, streamables ++ existing)
+  end
+
+  defp add_notifications(meta, notifications) do
+    existing = Keyword.get(meta, :notifications, [])
+
+    meta
+    |> Keyword.put(:notifications, notifications ++ existing)
   end
 
   def handle_after_transaction(meta) do
     meta
     |> send_notifications()
+    |> send_streamables()
   end
 end
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index d4c10977f..1c996da11 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -14,13 +14,12 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
 
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
-
     last_message = opts[:last_message] || MessageReference.last_message_for_chat(chat)
 
     %{
       id: chat.id |> to_string(),
       account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
-      unread: Map.get(chat, :unread) || MessageReference.unread_count_for_chat(chat),
+      unread: MessageReference.unread_count_for_chat(chat),
       last_message:
         last_message &&
           MessageReferenceView.render("show.json", chat_message_reference: last_message),
diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs
index b1afa6a2e..6bbbaae87 100644
--- a/test/web/activity_pub/side_effects_test.exs
+++ b/test/web/activity_pub/side_effects_test.exs
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
   import Mock
 
   describe "handle_after_transaction" do
-    test "it streams out notifications" do
+    test "it streams out notifications and streams" do
       author = insert(:user, local: true)
       recipient = insert(:user, local: true)
 
@@ -37,7 +37,7 @@ test "it streams out notifications" do
       {:ok, _create_activity, meta} =
         SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
-      assert [notification] = meta[:created_notifications]
+      assert [notification] = meta[:notifications]
 
       with_mocks([
         {
@@ -58,6 +58,7 @@ test "it streams out notifications" do
         SideEffects.handle_after_transaction(meta)
 
         assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], :_))
         assert called(Pleroma.Web.Push.send(notification))
       end
     end
@@ -362,33 +363,10 @@ test "it streams the created ChatMessage" do
 
       {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false)
 
-      with_mock Pleroma.Web.Streamer, [],
-        stream: fn _, payload ->
-          case payload do
-            {^author, cm_ref} ->
-              assert cm_ref.unread == false
+      {:ok, _create_activity, meta} =
+        SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
-            {^recipient, cm_ref} ->
-              assert cm_ref.unread == true
-
-              view =
-                Pleroma.Web.PleromaAPI.ChatView.render("show.json",
-                  last_message: cm_ref,
-                  chat: cm_ref.chat
-                )
-
-              assert view.unread == 1
-
-            _ ->
-              nil
-          end
-        end do
-        {:ok, _create_activity, _meta} =
-          SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
-
-        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], {author, :_}))
-        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], {recipient, :_}))
-      end
+      assert [_, _] = meta[:streamables]
     end
 
     test "it creates a Chat and MessageReferences for the local users and bumps the unread count, except for the author" do
@@ -422,13 +400,18 @@ test "it creates a Chat and MessageReferences for the local users and bumps the
           SideEffects.handle(create_activity, local: false, object_data: chat_message_data)
 
         # The notification gets created
-        assert [notification] = meta[:created_notifications]
+        assert [notification] = meta[:notifications]
         assert notification.activity_id == create_activity.id
 
         # But it is not sent out
         refute called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
         refute called(Pleroma.Web.Push.send(notification))
 
+        # Same for the user chat stream
+        assert [{topics, _}, _] = meta[:streamables]
+        assert topics == ["user", "user:pleroma_chat"]
+        refute called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], :_))
+
         chat = Chat.get(author.id, recipient.ap_id)
 
         [cm_ref] = MessageReference.for_chat_query(chat) |> Repo.all()
diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs
index 63b59820e..6bd26050e 100644
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@ -72,6 +72,7 @@ test "it posts a chat message without content but with an attachment" do
 
         assert called(Pleroma.Web.Push.send(notification))
         assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification))
+        assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], :_))
 
         assert activity
       end
diff --git a/test/web/pleroma_api/views/chat_view_test.exs b/test/web/pleroma_api/views/chat_view_test.exs
index f7af5d4e0..14eecb1bd 100644
--- a/test/web/pleroma_api/views/chat_view_test.exs
+++ b/test/web/pleroma_api/views/chat_view_test.exs
@@ -16,21 +16,6 @@ defmodule Pleroma.Web.PleromaAPI.ChatViewTest do
 
   import Pleroma.Factory
 
-  test "giving a chat with an 'unread' field, it uses that" do
-    user = insert(:user)
-    recipient = insert(:user)
-
-    {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
-
-    chat =
-      chat
-      |> Map.put(:unread, 5)
-
-    represented_chat = ChatView.render("show.json", chat: chat)
-
-    assert represented_chat[:unread] == 5
-  end
-
   test "it represents a chat" do
     user = insert(:user)
     recipient = insert(:user)

From 801e668a97adff4a33451dd7bb48799562ed8796 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 15:38:33 +0200
Subject: [PATCH 169/283] ChatController: Add `last_read_id` option to
 mark_as_read.

---
 lib/pleroma/chat/message_reference.ex         | 20 +++++++++++-----
 .../web/api_spec/operations/chat_operation.ex | 18 ++++++++++++++
 .../controllers/chat_controller.ex            |  3 ++-
 .../controllers/chat_controller_test.exs      | 24 +++++++++++++++++++
 4 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/chat/message_reference.ex b/lib/pleroma/chat/message_reference.ex
index 7ee7508ca..131ae0186 100644
--- a/lib/pleroma/chat/message_reference.ex
+++ b/lib/pleroma/chat/message_reference.ex
@@ -98,12 +98,20 @@ def mark_as_read(cm_ref) do
     |> Repo.update()
   end
 
-  def set_all_seen_for_chat(chat) do
-    chat
-    |> for_chat_query()
-    |> exclude(:order_by)
-    |> exclude(:preload)
-    |> where([cmr], cmr.unread == true)
+  def set_all_seen_for_chat(chat, last_read_id \\ nil) do
+    query =
+      chat
+      |> for_chat_query()
+      |> exclude(:order_by)
+      |> exclude(:preload)
+      |> where([cmr], cmr.unread == true)
+
+    if last_read_id do
+      query
+      |> where([cmr], cmr.id <= ^last_read_id)
+    else
+      query
+    end
     |> Repo.update_all(set: [unread: false])
   end
 end
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 74c3ad0bd..45fbad311 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -23,6 +23,7 @@ def mark_as_read_operation do
       summary: "Mark all messages in the chat as read",
       operationId: "ChatController.mark_as_read",
       parameters: [Operation.parameter(:id, :path, :string, "The ID of the Chat")],
+      requestBody: request_body("Parameters", mark_as_read()),
       responses: %{
         200 =>
           Operation.response(
@@ -333,4 +334,21 @@ def chat_message_create do
       }
     }
   end
+
+  def mark_as_read do
+    %Schema{
+      title: "MarkAsReadRequest",
+      description: "POST body for marking a number of chat messages as read",
+      type: :object,
+      properties: %{
+        last_read_id: %Schema{
+          type: :string,
+          description: "The content of your message. Optional."
+        }
+      },
+      example: %{
+        "last_read_id" => "abcdef12456"
+      }
+    }
+  end
 end
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 983550b13..002b75082 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -111,7 +111,8 @@ def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
 
   def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
-         {_n, _} <- MessageReference.set_all_seen_for_chat(chat) do
+         {_n, _} <-
+           MessageReference.set_all_seen_for_chat(chat, conn.body_params[:last_read_id]) do
       conn
       |> put_view(ChatView)
       |> render("show.json", chat: chat)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 2128fd9dd..63cd89c73 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -65,6 +65,30 @@ test "it marks all messages in a chat as read", %{conn: conn, user: user} do
 
       assert cm_ref.unread == false
     end
+
+    test "it given a `last_read_id` ", %{conn: conn, user: user} do
+      other_user = insert(:user)
+
+      {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
+      {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+      object = Object.normalize(create, false)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
+
+      assert cm_ref.unread == true
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/v1/pleroma/chats/#{chat.id}/read", %{"last_read_id" => cm_ref.id})
+        |> json_response_and_validate_schema(200)
+
+      assert result["unread"] == 1
+
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
+
+      assert cm_ref.unread == false
+    end
   end
 
   describe "POST /api/v1/pleroma/chats/:id/messages" do

From 680fa5fa36d8b30a9a9749edacf1a2c69fded29a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 15:41:46 +0200
Subject: [PATCH 170/283] Docs: Update docs on mark as read.

---
 docs/API/chats.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 761047336..81ff57941 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -79,6 +79,11 @@ To set the `unread` count of a chat to 0, call
 
 `POST /api/v1/pleroma/chats/:id/read`
 
+
+Parameters:
+- last_read_id: Given this id, all chat messages until this one will be marked as read. This should always be used.
+
+
 Returned data:
 
 ```json

From fe2a5d061463313f447b0557de05572fa3771728 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 20:22:08 +0200
Subject: [PATCH 171/283] ChatController: Make last_read_id mandatory.

---
 .../web/api_spec/operations/chat_operation.ex |  3 +-
 .../controllers/chat_controller.ex            |  7 +++--
 .../controllers/chat_controller_test.exs      | 28 +++----------------
 3 files changed, 11 insertions(+), 27 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 45fbad311..cf299bfc2 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -340,10 +340,11 @@ def mark_as_read do
       title: "MarkAsReadRequest",
       description: "POST body for marking a number of chat messages as read",
       type: :object,
+      required: [:last_read_id],
       properties: %{
         last_read_id: %Schema{
           type: :string,
-          description: "The content of your message. Optional."
+          description: "The content of your message."
         }
       },
       example: %{
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 002b75082..b9949236c 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -109,10 +109,13 @@ def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
     end
   end
 
-  def mark_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id}) do
+  def mark_as_read(
+        %{body_params: %{last_read_id: last_read_id}, assigns: %{user: %{id: user_id}}} = conn,
+        %{id: id}
+      ) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
          {_n, _} <-
-           MessageReference.set_all_seen_for_chat(chat, conn.body_params[:last_read_id]) do
+           MessageReference.set_all_seen_for_chat(chat, last_read_id) do
       conn
       |> put_view(ChatView)
       |> render("show.json", chat: chat)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index 63cd89c73..c2960956d 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -43,30 +43,10 @@ test "it marks one message as read", %{conn: conn, user: user} do
   describe "POST /api/v1/pleroma/chats/:id/read" do
     setup do: oauth_access(["write:chats"])
 
-    test "it marks all messages in a chat as read", %{conn: conn, user: user} do
-      other_user = insert(:user)
-
-      {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
-      {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
-      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
-      object = Object.normalize(create, false)
-      cm_ref = MessageReference.for_chat_and_object(chat, object)
-
-      assert cm_ref.unread == true
-
-      result =
-        conn
-        |> post("/api/v1/pleroma/chats/#{chat.id}/read")
-        |> json_response_and_validate_schema(200)
-
-      assert result["unread"] == 0
-
-      cm_ref = MessageReference.for_chat_and_object(chat, object)
-
-      assert cm_ref.unread == false
-    end
-
-    test "it given a `last_read_id` ", %{conn: conn, user: user} do
+    test "given a `last_read_id`, it marks everything until then as read", %{
+      conn: conn,
+      user: user
+    } do
       other_user = insert(:user)
 
       {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")

From 1a2acce7c5927cd113ebcffd0acc7a5c547bbf0e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 7 Jun 2020 20:23:17 +0200
Subject: [PATCH 172/283] Docs: Document new mandatory parameter.

---
 docs/API/chats.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 81ff57941..9eb581943 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -75,13 +75,13 @@ Returned data:
 
 ### Marking a chat as read
 
-To set the `unread` count of a chat to 0, call
+To mark a number of messages in a chat up to a certain message as read, you can use
 
 `POST /api/v1/pleroma/chats/:id/read`
 
 
 Parameters:
-- last_read_id: Given this id, all chat messages until this one will be marked as read. This should always be used.
+- last_read_id: Given this id, all chat messages until this one will be marked as read. Required.
 
 
 Returned data:

From 89b85f65297ef4b8ce92eacb27c90e8f7c874f54 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 8 Jun 2020 11:09:53 +0200
Subject: [PATCH 173/283] ChatController: Remove nonsensical pagination.

---
 docs/API/chats.md                                     |  2 +-
 .../web/pleroma_api/controllers/chat_controller.ex    |  4 ++--
 .../pleroma_api/controllers/chat_controller_test.exs  | 11 ++---------
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 9eb581943..aa6119670 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -135,7 +135,7 @@ Returned data:
 ```
 
 The recipient of messages that are sent to this chat is given by their AP ID.
-The usual pagination options are implemented.
+No pagination is implemented for now.
 
 ### Getting the messages for a Chat
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index b9949236c..e4760f53e 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -140,7 +140,7 @@ def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = para
     end
   end
 
-  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
+  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
     blocked_ap_ids = User.blocked_users_ap_ids(user)
 
     chats =
@@ -149,7 +149,7 @@ def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
         where: c.recipient not in ^blocked_ap_ids,
         order_by: [desc: c.updated_at]
       )
-      |> Pagination.fetch_paginated(params |> stringify_keys)
+      |> Repo.all()
 
     conn
     |> put_view(ChatView)
diff --git a/test/web/pleroma_api/controllers/chat_controller_test.exs b/test/web/pleroma_api/controllers/chat_controller_test.exs
index c2960956d..82e16741d 100644
--- a/test/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/web/pleroma_api/controllers/chat_controller_test.exs
@@ -289,7 +289,7 @@ test "it does not return chats with users you blocked", %{conn: conn, user: user
       assert length(result) == 0
     end
 
-    test "it paginates", %{conn: conn, user: user} do
+    test "it returns all chats", %{conn: conn, user: user} do
       Enum.each(1..30, fn _ ->
         recipient = insert(:user)
         {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
@@ -300,14 +300,7 @@ test "it paginates", %{conn: conn, user: user} do
         |> get("/api/v1/pleroma/chats")
         |> json_response_and_validate_schema(200)
 
-      assert length(result) == 20
-
-      result =
-        conn
-        |> get("/api/v1/pleroma/chats?max_id=#{List.last(result)["id"]}")
-        |> json_response_and_validate_schema(200)
-
-      assert length(result) == 10
+      assert length(result) == 30
     end
 
     test "it return a list of chats the current user is participating in, in descending order of updates",

From d44843e6774ed1c60d510a5307e0113e39569416 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 8 Jun 2020 17:56:34 +0400
Subject: [PATCH 174/283] Restrict ActivityExpirationPolicy to Notes only

---
 .../mrf/activity_expiration_policy.ex         |  6 ++++-
 .../mrf/activity_expiration_policy_test.exs   | 26 +++++++++++++++----
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index a9bdf3b69..8e47f1e02 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
   @impl true
   def filter(activity) do
     activity =
-      if activity["type"] == "Create" && local?(activity) do
+      if note?(activity) and local?(activity) do
         maybe_add_expiration(activity)
       else
         activity
@@ -25,6 +25,10 @@ defp local?(%{"id" => id}) do
     String.starts_with?(id, Pleroma.Web.Endpoint.url())
   end
 
+  defp note?(activity) do
+    match?(%{"type" => "Create", "object" => %{"type" => "Note"}}, activity)
+  end
+
   defp maybe_add_expiration(activity) do
     days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
     expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days)
diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
index 0d3bcc457..8babf49e7 100644
--- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
+++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs
@@ -10,7 +10,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
 
   test "adds `expires_at` property" do
     assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
-             ActivityExpirationPolicy.filter(%{"id" => @id, "type" => "Create"})
+             ActivityExpirationPolicy.filter(%{
+               "id" => @id,
+               "type" => "Create",
+               "object" => %{"type" => "Note"}
+             })
 
     assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
   end
@@ -22,7 +26,8 @@ test "keeps existing `expires_at` if it less than the config setting" do
              ActivityExpirationPolicy.filter(%{
                "id" => @id,
                "type" => "Create",
-               "expires_at" => expires_at
+               "expires_at" => expires_at,
+               "object" => %{"type" => "Note"}
              })
   end
 
@@ -33,7 +38,8 @@ test "overwrites existing `expires_at` if it greater than the config setting" do
              ActivityExpirationPolicy.filter(%{
                "id" => @id,
                "type" => "Create",
-               "expires_at" => too_distant_future
+               "expires_at" => too_distant_future,
+               "object" => %{"type" => "Note"}
              })
 
     assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364
@@ -43,13 +49,14 @@ test "ignores remote activities" do
     assert {:ok, activity} =
              ActivityExpirationPolicy.filter(%{
                "id" => "https://example.com/123",
-               "type" => "Create"
+               "type" => "Create",
+               "object" => %{"type" => "Note"}
              })
 
     refute Map.has_key?(activity, "expires_at")
   end
 
-  test "ignores non-Create activities" do
+  test "ignores non-Create/Note activities" do
     assert {:ok, activity} =
              ActivityExpirationPolicy.filter(%{
                "id" => "https://example.com/123",
@@ -57,5 +64,14 @@ test "ignores non-Create activities" do
              })
 
     refute Map.has_key?(activity, "expires_at")
+
+    assert {:ok, activity} =
+             ActivityExpirationPolicy.filter(%{
+               "id" => "https://example.com/123",
+               "type" => "Create",
+               "object" => %{"type" => "Cofe"}
+             })
+
+    refute Map.has_key?(activity, "expires_at")
   end
 end

From fe1cb56fdc52092a8af2895fae4c020679156674 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 8 Jun 2020 21:04:16 +0200
Subject: [PATCH 175/283] transmogrifier: MIME.valid?/1 for mediaType

No issues with the rest of the network yet but this makes sure it will work
once https://git.pleroma.social/pleroma/pleroma/-/merge_requests/2429
is merged.
---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index fda1c71df..543972ae9 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -221,9 +221,9 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm
 
         media_type =
           cond do
-            is_map(url) && is_binary(url["mediaType"]) -> url["mediaType"]
-            is_binary(data["mediaType"]) -> data["mediaType"]
-            is_binary(data["mimeType"]) -> data["mimeType"]
+            is_map(url) && MIME.valid?(url["mediaType"]) -> url["mediaType"]
+            MIME.valid?(data["mediaType"]) -> data["mediaType"]
+            MIME.valid?(data["mimeType"]) -> data["mimeType"]
             true -> nil
           end
 

From fc04a138d46c43860a2838d60fc8668112fdc1ec Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 8 Jun 2020 20:01:37 +0000
Subject: [PATCH 176/283] Apply suggestion to lib/pleroma/notification.ex

---
 lib/pleroma/notification.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 3ac8737e2..3386a1933 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -31,7 +31,7 @@ defmodule Pleroma.Notification do
   schema "notifications" do
     field(:seen, :boolean, default: false)
     # This is an enum type in the database. If you add a new notification type,
-    # remembert to add a migration to add it to the `notifications_type` enum
+    # remember to add a migration to add it to the `notifications_type` enum
     # as well.
     field(:type, :string)
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)

From e1bc37d11852684a5007a9550208944d899800ca Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 9 Jun 2020 09:20:55 +0200
Subject: [PATCH 177/283] MigrationHelper: Move notification backfilling to own
 module.

---
 .../notification_backfill.ex}                               | 2 +-
 .../20200602125218_backfill_notification_types.exs          | 2 +-
 .../notification_backfill_test.exs}                         | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)
 rename lib/pleroma/{migration_helper.ex => migration_helper/notification_backfill.ex} (97%)
 rename test/{migration_helper_test.exs => migration_helper/notification_backfill_test.exs} (91%)

diff --git a/lib/pleroma/migration_helper.ex b/lib/pleroma/migration_helper/notification_backfill.ex
similarity index 97%
rename from lib/pleroma/migration_helper.ex
rename to lib/pleroma/migration_helper/notification_backfill.ex
index a20d27a01..09647d12a 100644
--- a/lib/pleroma/migration_helper.ex
+++ b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.MigrationHelper do
+defmodule Pleroma.MigrationHelper.NotificationBackfill do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
diff --git a/priv/repo/migrations/20200602125218_backfill_notification_types.exs b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
index 58943fad0..996d721ee 100644
--- a/priv/repo/migrations/20200602125218_backfill_notification_types.exs
+++ b/priv/repo/migrations/20200602125218_backfill_notification_types.exs
@@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.BackfillNotificationTypes do
   use Ecto.Migration
 
   def up do
-    Pleroma.MigrationHelper.fill_in_notification_types()
+    Pleroma.MigrationHelper.NotificationBackfill.fill_in_notification_types()
   end
 
   def down do
diff --git a/test/migration_helper_test.exs b/test/migration_helper/notification_backfill_test.exs
similarity index 91%
rename from test/migration_helper_test.exs
rename to test/migration_helper/notification_backfill_test.exs
index 1c8173987..2a62a2b00 100644
--- a/test/migration_helper_test.exs
+++ b/test/migration_helper/notification_backfill_test.exs
@@ -2,11 +2,11 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.MigrationHelperTest do
+defmodule Pleroma.MigrationHelper.NotificationBackfillTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
-  alias Pleroma.MigrationHelper
+  alias Pleroma.MigrationHelper.NotificationBackfill
   alias Pleroma.Notification
   alias Pleroma.Repo
   alias Pleroma.Web.CommonAPI
@@ -35,7 +35,7 @@ test "it fills in missing notification types" do
 
       assert {5, nil} = Repo.update_all(Notification, set: [type: nil])
 
-      MigrationHelper.fill_in_notification_types()
+      NotificationBackfill.fill_in_notification_types()
 
       assert %{type: "mention"} =
                Repo.get_by(Notification, user_id: other_user.id, activity_id: post.id)

From 063e6b9841ec72c7e89339c54581d199fa31e675 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 9 Jun 2020 10:53:40 +0200
Subject: [PATCH 178/283] StatusController: Correctly paginate favorites.

Favorites were paginating wrongly, because the pagination headers
where using the id of the id of the `Create` activity, while the
ordering was by the id of the `Like` activity. This isn't easy to
notice in most cases, as they usually have a similar order because
people tend to favorite posts as they come in. This commit adds a
way to give different pagination ids to the pagination helper, so
we can paginate correctly in cases like this.
---
 lib/pleroma/activity.ex                       |  4 ++
 lib/pleroma/web/activity_pub/activity_pub.ex  |  5 +-
 .../api_spec/operations/status_operation.ex   |  3 +-
 lib/pleroma/web/controller_helper.ex          | 56 +++++++++++--------
 .../controllers/status_controller_test.exs    | 43 ++++++++++++--
 5 files changed, 79 insertions(+), 32 deletions(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 6213d0eb7..f800447fd 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -41,6 +41,10 @@ defmodule Pleroma.Activity do
     field(:recipients, {:array, :string}, default: [])
     field(:thread_muted?, :boolean, virtual: true)
 
+    # A field that can be used if you need to join some kind of other
+    # id to order / paginate this field by
+    field(:pagination_id, :string, virtual: true)
+
     # This is a fake relation,
     # do not use outside of with_preloaded_user_actor/with_joined_user_actor
     has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eb73c95fe..cc883ccce 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1138,12 +1138,11 @@ def fetch_favourites(user, params \\ %{}, pagination \\ :keyset) do
     |> Activity.Queries.by_type("Like")
     |> Activity.with_joined_object()
     |> Object.with_joined_activity()
-    |> select([_like, object, activity], %{activity | object: object})
+    |> select([like, object, activity], %{activity | object: object, pagination_id: like.id})
     |> order_by([like, _, _], desc_nulls_last: like.id)
     |> Pagination.fetch_paginated(
       Map.merge(params, %{skip_order: true}),
-      pagination,
-      :object_activity
+      pagination
     )
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index ca9db01e5..0b7fad793 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -333,7 +333,8 @@ def favourites_operation do
     %Operation{
       tags: ["Statuses"],
       summary: "Favourited statuses",
-      description: "Statuses the user has favourited",
+      description:
+        "Statuses the user has favourited. Please note that you have to use the link headers to paginate this. You can not build the query parameters yourself.",
       operationId: "StatusController.favourites",
       parameters: pagination_params(),
       security: [%{"oAuth" => ["read:favourites"]}],
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 5d67d75b5..5e33e0810 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -57,35 +57,45 @@ def add_link_headers(conn, activities, extra_params) do
     end
   end
 
+  defp build_pagination_fields(conn, min_id, max_id, extra_params) do
+    params =
+      conn.params
+      |> Map.drop(Map.keys(conn.path_params))
+      |> Map.merge(extra_params)
+      |> Map.drop(Pagination.page_keys() -- ["limit", "order"])
+
+    fields = %{
+      "next" => current_url(conn, Map.put(params, :max_id, max_id)),
+      "prev" => current_url(conn, Map.put(params, :min_id, min_id))
+    }
+
+    #  Generating an `id` without already present pagination keys would
+    # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
+    # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
+    #  This is because we only have ids present inside of the page, while
+    # `min_id`, `since_id` and `max_id` requires to know one outside of it.
+    if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
+      Map.put(fields, "id", current_url(conn, conn.params))
+    else
+      fields
+    end
+  end
+
   def get_pagination_fields(conn, activities, extra_params \\ %{}) do
     case List.last(activities) do
-      %{id: max_id} ->
-        params =
-          conn.params
-          |> Map.drop(Map.keys(conn.path_params))
-          |> Map.merge(extra_params)
-          |> Map.drop(Pagination.page_keys() -- ["limit", "order"])
-
-        min_id =
+      %{pagination_id: max_id} when not is_nil(max_id) ->
+        %{pagination_id: min_id} =
           activities
           |> List.first()
-          |> Map.get(:id)
 
-        fields = %{
-          "next" => current_url(conn, Map.put(params, :max_id, max_id)),
-          "prev" => current_url(conn, Map.put(params, :min_id, min_id))
-        }
+        build_pagination_fields(conn, min_id, max_id, extra_params)
 
-        #  Generating an `id` without already present pagination keys would
-        # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
-        # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
-        #  This is because we only have ids present inside of the page, while
-        # `min_id`, `since_id` and `max_id` requires to know one outside of it.
-        if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
-          Map.put(fields, "id", current_url(conn, conn.params))
-        else
-          fields
-        end
+      %{id: max_id} ->
+        %{id: min_id} =
+          activities
+          |> List.first()
+
+        build_pagination_fields(conn, min_id, max_id, extra_params)
 
       _ ->
         %{}
diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs
index 700c82e4f..648e6f2ce 100644
--- a/test/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/web/mastodon_api/controllers/status_controller_test.exs
@@ -1541,14 +1541,49 @@ test "context" do
            } = response
   end
 
+  test "favorites paginate correctly" do
+    %{user: user, conn: conn} = oauth_access(["read:favourites"])
+    other_user = insert(:user)
+    {:ok, first_post} = CommonAPI.post(other_user, %{status: "bla"})
+    {:ok, second_post} = CommonAPI.post(other_user, %{status: "bla"})
+    {:ok, third_post} = CommonAPI.post(other_user, %{status: "bla"})
+
+    {:ok, _first_favorite} = CommonAPI.favorite(user, third_post.id)
+    {:ok, _second_favorite} = CommonAPI.favorite(user, first_post.id)
+    {:ok, third_favorite} = CommonAPI.favorite(user, second_post.id)
+
+    result =
+      conn
+      |> get("/api/v1/favourites?limit=1")
+
+    assert [%{"id" => post_id}] = json_response_and_validate_schema(result, 200)
+    assert post_id == second_post.id
+
+    # Using the header for pagination works correctly
+    [next, _] = get_resp_header(result, "link") |> hd() |> String.split(", ")
+    [_, max_id] = Regex.run(~r/max_id=(.*)>;/, next)
+
+    assert max_id == third_favorite.id
+
+    result =
+      conn
+      |> get("/api/v1/favourites?max_id=#{max_id}")
+
+    assert [%{"id" => first_post_id}, %{"id" => third_post_id}] =
+             json_response_and_validate_schema(result, 200)
+
+    assert first_post_id == first_post.id
+    assert third_post_id == third_post.id
+  end
+
   test "returns the favorites of a user" do
     %{user: user, conn: conn} = oauth_access(["read:favourites"])
     other_user = insert(:user)
 
     {:ok, _} = CommonAPI.post(other_user, %{status: "bla"})
-    {:ok, activity} = CommonAPI.post(other_user, %{status: "traps are happy"})
+    {:ok, activity} = CommonAPI.post(other_user, %{status: "trees are happy"})
 
-    {:ok, _} = CommonAPI.favorite(user, activity.id)
+    {:ok, last_like} = CommonAPI.favorite(user, activity.id)
 
     first_conn = get(conn, "/api/v1/favourites")
 
@@ -1566,9 +1601,7 @@ test "returns the favorites of a user" do
 
     {:ok, _} = CommonAPI.favorite(user, second_activity.id)
 
-    last_like = status["id"]
-
-    second_conn = get(conn, "/api/v1/favourites?since_id=#{last_like}")
+    second_conn = get(conn, "/api/v1/favourites?since_id=#{last_like.id}")
 
     assert [second_status] = json_response_and_validate_schema(second_conn, 200)
     assert second_status["id"] == to_string(second_activity.id)

From 3dd1de61a78f9571a1d886411d70cd52584e084a Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 8 Jun 2020 22:08:57 +0400
Subject: [PATCH 179/283] Add `url` field to AdminAPI.AccountView

---
 .../web/admin_api/views/account_view.ex       |  3 +-
 .../controllers/admin_api_controller_test.exs | 66 ++++++++++++-------
 2 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 120159527..e1e929632 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -76,7 +76,8 @@ def render("show.json", %{user: user}) do
       "local" => user.local,
       "roles" => User.roles(user),
       "tags" => user.tags || [],
-      "confirmation_pending" => user.confirmation_pending
+      "confirmation_pending" => user.confirmation_pending,
+      "url" => user.uri || user.ap_id
     }
   end
 
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index bea810c4a..e3d3ccb8d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -337,7 +337,8 @@ test "Show", %{conn: conn} do
         "tags" => [],
         "avatar" => User.avatar_url(user) |> MediaProxy.url(),
         "display_name" => HTML.strip_tags(user.name || user.nickname),
-        "confirmation_pending" => false
+        "confirmation_pending" => false,
+        "url" => user.ap_id
       }
 
       assert expected == json_response(conn, 200)
@@ -614,7 +615,8 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
             "tags" => [],
             "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => admin.ap_id
           },
           %{
             "deactivated" => user.deactivated,
@@ -625,7 +627,8 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
             "tags" => ["foo", "bar"],
             "avatar" => User.avatar_url(user) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => user.ap_id
           }
         ]
         |> Enum.sort_by(& &1["nickname"])
@@ -697,7 +700,8 @@ test "regular search", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -722,7 +726,8 @@ test "search by domain", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -747,7 +752,8 @@ test "search by full nickname", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -772,7 +778,8 @@ test "search by display name", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -797,7 +804,8 @@ test "search by email", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -822,7 +830,8 @@ test "regular search with page size", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -842,7 +851,8 @@ test "regular search with page size", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user2.ap_id
                  }
                ]
              }
@@ -874,7 +884,8 @@ test "only local users" do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -899,7 +910,8 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
             "tags" => [],
             "avatar" => User.avatar_url(user) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => user.ap_id
           },
           %{
             "deactivated" => admin.deactivated,
@@ -910,7 +922,8 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
             "tags" => [],
             "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => admin.ap_id
           },
           %{
             "deactivated" => false,
@@ -921,7 +934,8 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
             "tags" => [],
             "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => old_admin.ap_id
           }
         ]
         |> Enum.sort_by(& &1["nickname"])
@@ -951,7 +965,8 @@ test "load only admins", %{conn: conn, admin: admin} do
             "tags" => [],
             "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => admin.ap_id
           },
           %{
             "deactivated" => false,
@@ -962,7 +977,8 @@ test "load only admins", %{conn: conn, admin: admin} do
             "tags" => [],
             "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => second_admin.ap_id
           }
         ]
         |> Enum.sort_by(& &1["nickname"])
@@ -994,7 +1010,8 @@ test "load only moderators", %{conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => moderator.ap_id
                  }
                ]
              }
@@ -1019,7 +1036,8 @@ test "load users with tags list", %{conn: conn} do
             "tags" => ["first"],
             "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => user1.ap_id
           },
           %{
             "deactivated" => false,
@@ -1030,7 +1048,8 @@ test "load users with tags list", %{conn: conn} do
             "tags" => ["second"],
             "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
             "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false
+            "confirmation_pending" => false,
+            "url" => user2.ap_id
           }
         ]
         |> Enum.sort_by(& &1["nickname"])
@@ -1069,7 +1088,8 @@ test "it works with multiple filters" do
                    "tags" => [],
                    "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => user.ap_id
                  }
                ]
              }
@@ -1093,7 +1113,8 @@ test "it omits relay user", %{admin: admin, conn: conn} do
                    "tags" => [],
                    "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
                    "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-                   "confirmation_pending" => false
+                   "confirmation_pending" => false,
+                   "url" => admin.ap_id
                  }
                ]
              }
@@ -1155,7 +1176,8 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admi
                "tags" => [],
                "avatar" => User.avatar_url(user) |> MediaProxy.url(),
                "display_name" => HTML.strip_tags(user.name || user.nickname),
-               "confirmation_pending" => false
+               "confirmation_pending" => false,
+               "url" => user.ap_id
              }
 
     log_entry = Repo.one(ModerationLog)

From c4f267b3bef90dcac21b7db2a91f86d3ba5dc7c2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 08:02:26 +0000
Subject: [PATCH 180/283] Apply suggestion to
 lib/pleroma/web/controller_helper.ex

---
 lib/pleroma/web/controller_helper.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 5e33e0810..6cb19d539 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -57,6 +57,7 @@ def add_link_headers(conn, activities, extra_params) do
     end
   end
 
+  @id_keys Pagination.page_keys() -- ["limit", "order"]
   defp build_pagination_fields(conn, min_id, max_id, extra_params) do
     params =
       conn.params

From be7c322865b2b7aa1c8c25147cc598b6362ab187 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 08:02:35 +0000
Subject: [PATCH 181/283] Apply suggestion to
 lib/pleroma/web/controller_helper.ex

---
 lib/pleroma/web/controller_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 6cb19d539..b7971e940 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -63,7 +63,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
       conn.params
       |> Map.drop(Map.keys(conn.path_params))
       |> Map.merge(extra_params)
-      |> Map.drop(Pagination.page_keys() -- ["limit", "order"])
+      |> Map.drop(@id_keys)
 
     fields = %{
       "next" => current_url(conn, Map.put(params, :max_id, max_id)),

From b4c50be9df701dc9faf0a25f776f631d2175c99f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 08:12:29 +0000
Subject: [PATCH 182/283] Apply suggestion to
 lib/pleroma/web/controller_helper.ex

---
 lib/pleroma/web/controller_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index b7971e940..ab6e6c61a 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
     # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
     #  This is because we only have ids present inside of the page, while
     # `min_id`, `since_id` and `max_id` requires to know one outside of it.
-    if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
+    if Map.take(conn.params, @id_keys) != %{} do
       Map.put(fields, "id", current_url(conn, conn.params))
     else
       fields

From 86fec45f40dfa45cc89eddc6dcc7799e89d6f461 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 11:09:45 +0200
Subject: [PATCH 183/283] ControllerHelper: Fix wrong comparison.

---
 lib/pleroma/web/controller_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index ab6e6c61a..88f2cc6f1 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
     # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
     #  This is because we only have ids present inside of the page, while
     # `min_id`, `since_id` and `max_id` requires to know one outside of it.
-    if Map.take(conn.params, @id_keys) != %{} do
+    if Map.take(conn.params, @id_keys) != [] do
       Map.put(fields, "id", current_url(conn, conn.params))
     else
       fields

From 9e411372d0b7ae286941063956305c0a2eae46a6 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 12:10:09 +0200
Subject: [PATCH 184/283] ActivityPub: Don't show announces of your own objects
 in timeline.

---
 lib/pleroma/web/activity_pub/activity_pub.ex  | 40 ++++++++++---------
 .../controllers/timeline_controller.ex        |  1 +
 test/web/activity_pub/activity_pub_test.exs   | 24 +++++++++++
 3 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eb73c95fe..4182275bc 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -31,25 +31,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   require Logger
   require Pleroma.Constants
 
-  # For Announce activities, we filter the recipients based on following status for any actors
-  # that match actual users.  See issue #164 for more information about why this is necessary.
-  defp get_recipients(%{"type" => "Announce"} = data) do
-    to = Map.get(data, "to", [])
-    cc = Map.get(data, "cc", [])
-    bcc = Map.get(data, "bcc", [])
-    actor = User.get_cached_by_ap_id(data["actor"])
-
-    recipients =
-      Enum.filter(Enum.concat([to, cc, bcc]), fn recipient ->
-        case User.get_cached_by_ap_id(recipient) do
-          nil -> true
-          user -> User.following?(user, actor)
-        end
-      end)
-
-    {recipients, to, cc}
-  end
-
   defp get_recipients(%{"type" => "Create"} = data) do
     to = Map.get(data, "to", [])
     cc = Map.get(data, "cc", [])
@@ -702,6 +683,26 @@ defp user_activities_recipients(%{reading_user: reading_user}) do
     end
   end
 
+  defp restrict_announce_object_actor(_query, %{announce_filtering_user: _, skip_preload: true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
+  defp restrict_announce_object_actor(query, %{announce_filtering_user: %{ap_id: actor}}) do
+    from(
+      [activity, object] in query,
+      where:
+        fragment(
+          "?->>'type' != ? or ?->>'actor' != ?",
+          activity.data,
+          "Announce",
+          object.data,
+          ^actor
+        )
+    )
+  end
+
+  defp restrict_announce_object_actor(query, _), do: query
+
   defp restrict_since(query, %{since_id: ""}), do: query
 
   defp restrict_since(query, %{since_id: since_id}) do
@@ -1113,6 +1114,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_pinned(opts)
     |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
     |> restrict_instance(opts)
+    |> restrict_announce_object_actor(opts)
     |> Activity.restrict_deactivated_users()
     |> exclude_poll_votes(opts)
     |> exclude_invisible_actors(opts)
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
index 9270ca267..4bdd46d7e 100644
--- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -48,6 +48,7 @@ def home(%{assigns: %{user: user}} = conn, params) do
       |> Map.put(:blocking_user, user)
       |> Map.put(:muting_user, user)
       |> Map.put(:reply_filtering_user, user)
+      |> Map.put(:announce_filtering_user, user)
       |> Map.put(:user, user)
 
     activities =
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 2f65dfc8e..e17cc4ab1 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1643,6 +1643,30 @@ test "home timeline with reply_visibility `self`", %{
 
       assert Enum.all?(visible_ids, &(&1 in activities_ids))
     end
+
+    test "filtering out announces where the user is the actor of the announced message" do
+      user = insert(:user)
+      other_user = insert(:user)
+      third_user = insert(:user)
+      User.follow(user, other_user)
+
+      {:ok, post} = CommonAPI.post(user, %{status: "yo"})
+      {:ok, other_post} = CommonAPI.post(third_user, %{status: "yo"})
+      {:ok, _announce} = CommonAPI.repeat(post.id, other_user)
+      {:ok, _announce} = CommonAPI.repeat(post.id, third_user)
+      {:ok, announce} = CommonAPI.repeat(other_post.id, other_user)
+
+      params = %{
+        type: ["Announce"],
+        announce_filtering_user: user
+      }
+
+      [result] =
+        [user.ap_id | User.following(user)]
+        |> ActivityPub.fetch_activities(params)
+
+      assert result.id == announce.id
+    end
   end
 
   describe "replies filtering with private messages" do

From 600e2ea07396489325e06dee3e8432288e0e13c2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 12:15:56 +0200
Subject: [PATCH 185/283] ActivityPubTest: Make test easier to understand.

---
 test/web/activity_pub/activity_pub_test.exs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index e17cc4ab1..6cd3b8d1b 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -1656,6 +1656,16 @@ test "filtering out announces where the user is the actor of the announced messa
       {:ok, _announce} = CommonAPI.repeat(post.id, third_user)
       {:ok, announce} = CommonAPI.repeat(other_post.id, other_user)
 
+      params = %{
+        type: ["Announce"]
+      }
+
+      results =
+        [user.ap_id | User.following(user)]
+        |> ActivityPub.fetch_activities(params)
+
+      assert length(results) == 3
+
       params = %{
         type: ["Announce"],
         announce_filtering_user: user

From 570123ae21382c7e78b99442e3c025b0e66b8f6d Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Sun, 7 Jun 2020 18:21:11 +0200
Subject: [PATCH 186/283] Add test

---
 test/web/activity_pub/activity_pub_test.exs | 35 ++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 6cd3b8d1b..72d3f3dfa 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -574,7 +574,7 @@ test "doesn't return transitive interactions concerning blocked users" do
     refute Enum.member?(activities, activity_four)
   end
 
-  test "doesn't return announce activities concerning blocked users" do
+  test "doesn't return announce activities with blocked users in 'to'" do
     blocker = insert(:user)
     blockee = insert(:user)
     friend = insert(:user)
@@ -596,6 +596,39 @@ test "doesn't return announce activities concerning blocked users" do
     refute Enum.member?(activities, activity_three.id)
   end
 
+  test "doesn't return announce activities with blocked users in 'cc'" do
+    blocker = insert(:user)
+    blockee = insert(:user)
+    friend = insert(:user)
+
+    {:ok, _user_relationship} = User.block(blocker, blockee)
+
+    {:ok, activity_one} = CommonAPI.post(friend, %{status: "hey!"})
+
+    {:ok, activity_two} = CommonAPI.post(blockee, %{status: "hey! @#{friend.nickname}"})
+
+    assert object = Pleroma.Object.normalize(activity_two)
+
+    data = %{
+      "actor" => friend.ap_id,
+      "object" => object.data["id"],
+      "context" => object.data["context"],
+      "type" => "Announce",
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "cc" => [blockee.ap_id]
+    }
+
+    assert {:ok, activity_three} = ActivityPub.insert(data)
+
+    activities =
+      ActivityPub.fetch_activities([], %{"blocking_user" => blocker})
+      |> Enum.map(fn act -> act.id end)
+
+    assert Enum.member?(activities, activity_one.id)
+    refute Enum.member?(activities, activity_two.id)
+    refute Enum.member?(activities, activity_three.id)
+  end
+
   test "doesn't return activities from blocked domains" do
     domain = "dogwhistle.zone"
     domain_user = insert(:user, %{ap_id: "https://#{domain}/@pundit"})

From 5d87405b51efe9f99fea669090a5914db22ca9ed Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 16:55:30 +0200
Subject: [PATCH 187/283] ActivityPubTest: Update test for atomized parameters.

---
 test/web/activity_pub/activity_pub_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 72d3f3dfa..b239b812f 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -621,7 +621,7 @@ test "doesn't return announce activities with blocked users in 'cc'" do
     assert {:ok, activity_three} = ActivityPub.insert(data)
 
     activities =
-      ActivityPub.fetch_activities([], %{"blocking_user" => blocker})
+      ActivityPub.fetch_activities([], %{blocking_user: blocker})
       |> Enum.map(fn act -> act.id end)
 
     assert Enum.member?(activities, activity_one.id)

From 99afc7f4e423997079aaee1287b9ffb28a851d8b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 10 Jun 2020 20:09:16 +0300
Subject: [PATCH 188/283] HTTP security plug: add media proxy base url host to
 csp

---
 lib/pleroma/plugs/http_security_plug.ex | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 6a339b32c..620408d0f 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -113,6 +113,10 @@ defp get_proxy_and_attachment_sources do
         add_source(acc, host)
       end)
 
+    media_proxy_base_url =
+      if Config.get([Pleroma.Upload, :base_url]),
+        do: URI.parse(Config.get([:media_proxy, :base_url])).host
+
     upload_base_url =
       if Config.get([Pleroma.Upload, :base_url]),
         do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host
@@ -122,6 +126,7 @@ defp get_proxy_and_attachment_sources do
         do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host
 
     []
+    |> add_source(media_proxy_base_url)
     |> add_source(upload_base_url)
     |> add_source(s3_endpoint)
     |> add_source(media_proxy_whitelist)

From 7c47f791a803aa5cee2f2f6931b8445d2c0551e5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 10 Jun 2020 13:02:08 -0500
Subject: [PATCH 189/283] Add command to reload emoji packs from cli for OTP
 users

Not useful for source releases as we don't have a way to automate connecting to the running instance.
---
 docs/administration/CLI_tasks/emoji.md | 8 ++++++++
 lib/mix/tasks/pleroma/emoji.ex         | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/docs/administration/CLI_tasks/emoji.md b/docs/administration/CLI_tasks/emoji.md
index 3d524a52b..ddcb7e62c 100644
--- a/docs/administration/CLI_tasks/emoji.md
+++ b/docs/administration/CLI_tasks/emoji.md
@@ -44,3 +44,11 @@ Currently, only .zip archives are recognized as remote pack files and packs are
   The manifest entry will either be written to a newly created `pack_name.json` file (pack name is asked in questions) or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
 
   The file list will be written to the file specified previously, *replacing* that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).
+
+## Reload emoji packs
+
+```sh tab="OTP"
+./bin/pleroma_ctl emoji reload
+```
+
+This command only works with OTP releases.
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 29a5fa99c..f4eaeac98 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -237,6 +237,12 @@ def run(["gen-pack" | args]) do
     end
   end
 
+  def run(["reload"]) do
+    start_pleroma()
+    Pleroma.Emoji.reload()
+    IO.puts("Emoji packs have been reloaded.")
+  end
+
   defp fetch_and_decode(from) do
     with {:ok, json} <- fetch(from) do
       Jason.decode!(json)

From 5e44e9d69871f2e5805a8dddcfce43ae713eb52d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 10 Jun 2020 18:56:46 +0000
Subject: [PATCH 190/283] Apply suggestion to
 lib/pleroma/web/controller_helper.ex

---
 lib/pleroma/web/controller_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 88f2cc6f1..a5eb3e9e0 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -76,7 +76,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
     #  This is because we only have ids present inside of the page, while
     # `min_id`, `since_id` and `max_id` requires to know one outside of it.
     if Map.take(conn.params, @id_keys) != [] do
-      Map.put(fields, "id", current_url(conn, conn.params))
+      Map.put(fields, "id", current_url(conn))
     else
       fields
     end

From b28cec4271c52d55f6e6cf8a1bcdb41efec3ef03 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 11 Jun 2020 16:05:14 +0300
Subject: [PATCH 191/283] [#1794] Fixes URI query handling for hashtags
 extraction in search.

---
 .../mastodon_api/controllers/search_controller.ex  | 14 ++++++++++++++
 .../controllers/search_controller_test.exs         |  9 +++++++++
 2 files changed, 23 insertions(+)

diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 8840fc19c..46bcf4228 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -124,6 +124,7 @@ defp resource_search(:v1, "hashtags", query, _options) do
   defp prepare_tags(query, add_joined_tag \\ true) do
     tags =
       query
+      |> preprocess_uri_query()
       |> String.split(~r/[^#\w]+/u, trim: true)
       |> Enum.uniq_by(&String.downcase/1)
 
@@ -147,6 +148,19 @@ defp prepare_tags(query, add_joined_tag \\ true) do
     end
   end
 
+  # If `query` is a URI, returns last component of its path, otherwise returns `query`
+  defp preprocess_uri_query(query) do
+    if query =~ ~r/https?:\/\// do
+      query
+      |> URI.parse()
+      |> Map.get(:path)
+      |> String.split("/")
+      |> Enum.at(-1)
+    else
+      query
+    end
+  end
+
   defp joined_tag(tags) do
     tags
     |> Enum.map(fn tag -> String.capitalize(tag) end)
diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs
index 84d46895e..0e025adca 100644
--- a/test/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/web/mastodon_api/controllers/search_controller_test.exs
@@ -111,6 +111,15 @@ test "constructs hashtags from search query", %{conn: conn} do
                %{"name" => "prone", "url" => "#{Web.base_url()}/tag/prone"},
                %{"name" => "AccidentProne", "url" => "#{Web.base_url()}/tag/AccidentProne"}
              ]
+
+      results =
+        conn
+        |> get("/api/v2/search?#{URI.encode_query(%{q: "https://shpposter.club/users/shpuld"})}")
+        |> json_response_and_validate_schema(200)
+
+      assert results["hashtags"] == [
+               %{"name" => "shpuld", "url" => "#{Web.base_url()}/tag/shpuld"}
+             ]
     end
 
     test "excludes a blocked users from search results", %{conn: conn} do

From 1f35acce54ea6924a54b4fc387be3346a6f5551e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 11 Jun 2020 17:57:31 +0400
Subject: [PATCH 192/283] Merge OGP parser with TwitterCard

---
 CHANGELOG.md                                  |   1 +
 config/config.exs                             |   1 -
 config/description.exs                        |   2 -
 lib/pleroma/web/rich_media/parser.ex          |   4 +-
 .../rich_media/parsers/meta_tags_parser.ex    |  23 ++--
 .../web/rich_media/parsers/oembed_parser.ex   |   4 +-
 lib/pleroma/web/rich_media/parsers/ogp.ex     |   3 +-
 .../web/rich_media/parsers/twitter_card.ex    |  15 +-
 .../rich_media/parsers/twitter_card_test.exs  | 130 ++++++++++--------
 9 files changed, 90 insertions(+), 93 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cf2210f5..575eb67b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- OGP rich media parser merged with TwitterCard
 <details>
   <summary>API Changes</summary>
 - **Breaking:** Emoji API: changed methods and renamed routes.
diff --git a/config/config.exs b/config/config.exs
index 9508ae077..cafa40820 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -385,7 +385,6 @@
   ignore_tld: ["local", "localdomain", "lan"],
   parsers: [
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
-    Pleroma.Web.RichMedia.Parsers.OGP,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
   ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
diff --git a/config/description.exs b/config/description.exs
index 807c945e0..b993959d7 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -2091,9 +2091,7 @@
         description:
           "List of Rich Media parsers. Module names are shortened (removed leading `Pleroma.Web.RichMedia.Parsers.` part), but on adding custom module you need to use full name.",
         suggestions: [
-          Pleroma.Web.RichMedia.Parsers.MetaTagsParser,
           Pleroma.Web.RichMedia.Parsers.OEmbed,
-          Pleroma.Web.RichMedia.Parsers.OGP,
           Pleroma.Web.RichMedia.Parsers.TwitterCard
         ]
       },
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 40980def8..78e9048f3 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -105,8 +105,8 @@ defp parse_html(html), do: Floki.parse_document!(html)
   defp maybe_parse(html) do
     Enum.reduce_while(parsers(), %{}, fn parser, acc ->
       case parser.parse(html, acc) do
-        {:ok, data} -> {:halt, data}
-        {:error, _msg} -> {:cont, acc}
+        data when data != %{} -> {:halt, data}
+        _ -> {:cont, acc}
       end
     end)
   end
diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
index ae0f36702..c09b96eae 100644
--- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
@@ -3,22 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
-  def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do
-    meta_data =
-      html
-      |> get_elements(key_name, prefix)
-      |> Enum.reduce(data, fn el, acc ->
-        attributes = normalize_attributes(el, prefix, key_name, value_name)
+  def parse(data, html, prefix, key_name, value_name \\ "content") do
+    html
+    |> get_elements(key_name, prefix)
+    |> Enum.reduce(data, fn el, acc ->
+      attributes = normalize_attributes(el, prefix, key_name, value_name)
 
-        Map.merge(acc, attributes)
-      end)
-      |> maybe_put_title(html)
-
-    if Enum.empty?(meta_data) do
-      {:error, error_message}
-    else
-      {:ok, meta_data}
-    end
+      Map.merge(acc, attributes)
+    end)
+    |> maybe_put_title(html)
   end
 
   defp get_elements(html, key_name, prefix) do
diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
index 8f32bf91b..5d87a90e9 100644
--- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
@@ -7,9 +7,9 @@ def parse(html, _data) do
     with elements = [_ | _] <- get_discovery_data(html),
          {:ok, oembed_url} <- get_oembed_url(elements),
          {:ok, oembed_data} <- get_oembed_data(oembed_url) do
-      {:ok, oembed_data}
+      oembed_data
     else
-      _e -> {:error, "No OEmbed data found"}
+      _e -> %{}
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/parsers/ogp.ex b/lib/pleroma/web/rich_media/parsers/ogp.ex
index 3e9012588..5eebe42f7 100644
--- a/lib/pleroma/web/rich_media/parsers/ogp.ex
+++ b/lib/pleroma/web/rich_media/parsers/ogp.ex
@@ -5,10 +5,9 @@
 defmodule Pleroma.Web.RichMedia.Parsers.OGP do
   def parse(html, data) do
     Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse(
-      html,
       data,
+      html,
       "og",
-      "No OGP metadata found",
       "property"
     )
   end
diff --git a/lib/pleroma/web/rich_media/parsers/twitter_card.ex b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
index 09d4b526e..4a04865d2 100644
--- a/lib/pleroma/web/rich_media/parsers/twitter_card.ex
+++ b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
@@ -5,18 +5,11 @@
 defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
   alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
 
-  @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
+  @spec parse(list(), map()) :: map()
   def parse(html, data) do
     data
-    |> parse_name_attrs(html)
-    |> parse_property_attrs(html)
-  end
-
-  defp parse_name_attrs(data, html) do
-    MetaTagsParser.parse(html, data, "twitter", %{}, "name")
-  end
-
-  defp parse_property_attrs({_, data}, html) do
-    MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property")
+    |> MetaTagsParser.parse(html, "og", "property")
+    |> MetaTagsParser.parse(html, "twitter", "name")
+    |> MetaTagsParser.parse(html, "twitter", "property")
   end
 end
diff --git a/test/web/rich_media/parsers/twitter_card_test.exs b/test/web/rich_media/parsers/twitter_card_test.exs
index 87c767c15..3ccf26651 100644
--- a/test/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/web/rich_media/parsers/twitter_card_test.exs
@@ -7,8 +7,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
   alias Pleroma.Web.RichMedia.Parsers.TwitterCard
 
   test "returns error when html not contains twitter card" do
-    assert TwitterCard.parse([{"html", [], [{"head", [], []}, {"body", [], []}]}], %{}) ==
-             {:error, "No twitter card metadata found"}
+    assert TwitterCard.parse([{"html", [], [{"head", [], []}, {"body", [], []}]}], %{}) == %{}
   end
 
   test "parses twitter card with only name attributes" do
@@ -17,15 +16,21 @@ test "parses twitter card with only name attributes" do
       |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
-             {:ok,
-              %{
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622",
-                site: nil,
-                title:
-                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times"
-              }}
+             %{
+               "app:id:googleplay": "com.nytimes.android",
+               "app:name:googleplay": "NYTimes",
+               "app:url:googleplay": "nytimes://reader/id/100000006583622",
+               site: nil,
+               description:
+                 "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+               image:
+                 "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+               type: "article",
+               url:
+                 "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+               title:
+                 "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database."
+             }
   end
 
   test "parses twitter card with only property attributes" do
@@ -34,19 +39,19 @@ test "parses twitter card with only property attributes" do
       |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
-             {:ok,
-              %{
-                card: "summary_large_image",
-                description:
-                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-                image:
-                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-                "image:alt": "",
-                title:
-                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-                url:
-                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
-              }}
+             %{
+               card: "summary_large_image",
+               description:
+                 "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+               image:
+                 "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+               "image:alt": "",
+               title:
+                 "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+               url:
+                 "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+               type: "article"
+             }
   end
 
   test "parses twitter card with name & property attributes" do
@@ -55,23 +60,23 @@ test "parses twitter card with name & property attributes" do
       |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
-             {:ok,
-              %{
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622",
-                card: "summary_large_image",
-                description:
-                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-                image:
-                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-                "image:alt": "",
-                site: nil,
-                title:
-                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-                url:
-                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
-              }}
+             %{
+               "app:id:googleplay": "com.nytimes.android",
+               "app:name:googleplay": "NYTimes",
+               "app:url:googleplay": "nytimes://reader/id/100000006583622",
+               card: "summary_large_image",
+               description:
+                 "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+               image:
+                 "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+               "image:alt": "",
+               site: nil,
+               title:
+                 "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+               url:
+                 "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+               type: "article"
+             }
   end
 
   test "respect only first title tag on the page" do
@@ -84,14 +89,17 @@ test "respect only first title tag on the page" do
       File.read!("test/fixtures/margaret-corbin-grave-west-point.html") |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
-             {:ok,
-              %{
-                site: "@atlasobscura",
-                title:
-                  "The Missing Grave of Margaret Corbin, Revolutionary War Veteran - Atlas Obscura",
-                card: "summary_large_image",
-                image: image_path
-              }}
+             %{
+               site: "@atlasobscura",
+               title: "The Missing Grave of Margaret Corbin, Revolutionary War Veteran",
+               card: "summary_large_image",
+               image: image_path,
+               description:
+                 "She's the only woman veteran honored with a monument at West Point. But where was she buried?",
+               site_name: "Atlas Obscura",
+               type: "article",
+               url: "http://www.atlasobscura.com/articles/margaret-corbin-grave-west-point"
+             }
   end
 
   test "takes first founded title in html head if there is html markup error" do
@@ -100,14 +108,20 @@ test "takes first founded title in html head if there is html markup error" do
       |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
-             {:ok,
-              %{
-                site: nil,
-                title:
-                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622"
-              }}
+             %{
+               site: nil,
+               title:
+                 "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+               "app:id:googleplay": "com.nytimes.android",
+               "app:name:googleplay": "NYTimes",
+               "app:url:googleplay": "nytimes://reader/id/100000006583622",
+               description:
+                 "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+               image:
+                 "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+               type: "article",
+               url:
+                 "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+             }
   end
 end

From 7f7a1a467677471e0e1ec688e4eca9ba759d976a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 11 Jun 2020 11:05:22 -0500
Subject: [PATCH 193/283] Check for media proxy base_url, not Upload base_url

---
 lib/pleroma/plugs/http_security_plug.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 620408d0f..1420a9611 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -114,7 +114,7 @@ defp get_proxy_and_attachment_sources do
       end)
 
     media_proxy_base_url =
-      if Config.get([Pleroma.Upload, :base_url]),
+      if Config.get([:media_proxy, :base_url]),
         do: URI.parse(Config.get([:media_proxy, :base_url])).host
 
     upload_base_url =

From 2419776e192316cefbdbe607306c9b92ec558319 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 11 Jun 2020 23:11:46 +0400
Subject: [PATCH 194/283] Deprecate Pleroma.Web.RichMedia.Parsers.OGP

---
 lib/pleroma/web/rich_media/parsers/ogp.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/rich_media/parsers/ogp.ex b/lib/pleroma/web/rich_media/parsers/ogp.ex
index 5eebe42f7..363815f81 100644
--- a/lib/pleroma/web/rich_media/parsers/ogp.ex
+++ b/lib/pleroma/web/rich_media/parsers/ogp.ex
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parsers.OGP do
+  @deprecated "OGP parser is deprecated. Use TwitterCard instead."
   def parse(html, data) do
     Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse(
       data,

From 40970f6bb94760d19cc1d3201405df5bb32f5083 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 11 Jun 2020 22:54:39 +0200
Subject: [PATCH 195/283] New mix task: pleroma.user reset_mfa <nickname>

---
 docs/administration/CLI_tasks/user.md | 10 +++++++++
 lib/mix/tasks/pleroma/user.ex         | 12 +++++++++++
 test/tasks/user_test.exs              | 30 +++++++++++++++++++++++++++
 3 files changed, 52 insertions(+)

diff --git a/docs/administration/CLI_tasks/user.md b/docs/administration/CLI_tasks/user.md
index afeb8d52f..1e6f4a8b4 100644
--- a/docs/administration/CLI_tasks/user.md
+++ b/docs/administration/CLI_tasks/user.md
@@ -135,6 +135,16 @@ mix pleroma.user reset_password <nickname>
 ```
 
 
+## Disable Multi Factor Authentication (MFA/2FA) for a user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user reset_mfa <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user reset_mfa <nickname>
+```
+
+
 ## Set the value of the given user's settings
 ```sh tab="OTP"
  ./bin/pleroma_ctl user set <nickname> [option ...]
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 3635c02bc..bca7e87bf 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -144,6 +144,18 @@ def run(["reset_password", nickname]) do
     end
   end
 
+  def run(["reset_mfa", nickname]) do
+    start_pleroma()
+
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
+         {:ok, _token} <- Pleroma.MFA.disable(user) do
+      shell_info("Multi-Factor Authentication disabled for #{user.nickname}")
+    else
+      _ ->
+        shell_error("No local user #{nickname}")
+    end
+  end
+
   def run(["deactivate", nickname]) do
     start_pleroma()
 
diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs
index b55aa1cdb..9220d23fc 100644
--- a/test/tasks/user_test.exs
+++ b/test/tasks/user_test.exs
@@ -4,6 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.UserTest do
   alias Pleroma.Activity
+  alias Pleroma.MFA
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.Tests.ObanHelpers
@@ -278,6 +279,35 @@ test "no user to reset password" do
     end
   end
 
+  describe "running reset_mfa" do
+    test "disables MFA" do
+      user =
+        insert(:user,
+          multi_factor_authentication_settings: %MFA.Settings{
+            enabled: true,
+            totp: %MFA.Settings.TOTP{secret: "xx", confirmed: true}
+          }
+        )
+
+      Mix.Tasks.Pleroma.User.run(["reset_mfa", user.nickname])
+
+      assert_received {:mix_shell, :info, [message]}
+      assert message == "Multi-Factor Authentication disabled for #{user.nickname}"
+
+      assert %{enabled: false, totp: false} ==
+               user.nickname
+               |> User.get_cached_by_nickname()
+               |> MFA.mfa_settings()
+    end
+
+    test "no user to reset MFA" do
+      Mix.Tasks.Pleroma.User.run(["reset_password", "nonexistent"])
+
+      assert_received {:mix_shell, :error, [message]}
+      assert message =~ "No local user"
+    end
+  end
+
   describe "running invite" do
     test "invite token is generated" do
       assert capture_io(fn ->

From 122328b93a708e396b5c0cd1930a4b759e7b7db6 Mon Sep 17 00:00:00 2001
From: normandy <normandy@firemail.cc>
Date: Fri, 12 Jun 2020 01:41:09 +0000
Subject: [PATCH 196/283] Update pleroma.nginx to support TLSv1.3

Based on SSL config from https://ssl-config.mozilla.org/
---
 installation/pleroma.nginx | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
index 688be3e71..d301ca615 100644
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@@ -37,18 +37,17 @@ server {
 
     listen 443 ssl http2;
     listen [::]:443 ssl http2;
-    ssl_session_timeout 5m;
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+    ssl_session_tickets off;
 
     ssl_trusted_certificate   /etc/letsencrypt/live/example.tld/chain.pem;
     ssl_certificate           /etc/letsencrypt/live/example.tld/fullchain.pem;
     ssl_certificate_key       /etc/letsencrypt/live/example.tld/privkey.pem;
 
-    # Add TLSv1.0 to support older devices
-    ssl_protocols TLSv1.2;
-    # Uncomment line below if you want to support older devices (Before Android 4.4.2, IE 8, etc.)
-    # ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
+    ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
-    ssl_prefer_server_ciphers on;
+    ssl_prefer_server_ciphers off;
     # In case of an old server with an OpenSSL version of 1.0.2 or below,
     # leave only prime256v1 or comment out the following line.
     ssl_ecdh_curve X25519:prime256v1:secp384r1:secp521r1;

From 21880970660906d8072dc501e6a8b25fb4a4b0c7 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 12 Jun 2020 14:25:41 +0300
Subject: [PATCH 197/283] [#1794] Fixes URI query handling for hashtags
 extraction in search.

---
 .../controllers/search_controller.ex          |  1 +
 .../controllers/search_controller_test.exs    | 29 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 46bcf4228..3be0ca095 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -152,6 +152,7 @@ defp prepare_tags(query, add_joined_tag \\ true) do
   defp preprocess_uri_query(query) do
     if query =~ ~r/https?:\/\// do
       query
+      |> String.trim_trailing("/")
       |> URI.parse()
       |> Map.get(:path)
       |> String.split("/")
diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs
index 0e025adca..c605957b1 100644
--- a/test/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/web/mastodon_api/controllers/search_controller_test.exs
@@ -120,6 +120,35 @@ test "constructs hashtags from search query", %{conn: conn} do
       assert results["hashtags"] == [
                %{"name" => "shpuld", "url" => "#{Web.base_url()}/tag/shpuld"}
              ]
+
+      results =
+        conn
+        |> get(
+          "/api/v2/search?#{
+            URI.encode_query(%{
+              q:
+                "https://www.washingtonpost.com/sports/2020/06/10/" <>
+                  "nascar-ban-display-confederate-flag-all-events-properties/"
+            })
+          }"
+        )
+        |> json_response_and_validate_schema(200)
+
+      assert results["hashtags"] == [
+               %{"name" => "nascar", "url" => "#{Web.base_url()}/tag/nascar"},
+               %{"name" => "ban", "url" => "#{Web.base_url()}/tag/ban"},
+               %{"name" => "display", "url" => "#{Web.base_url()}/tag/display"},
+               %{"name" => "confederate", "url" => "#{Web.base_url()}/tag/confederate"},
+               %{"name" => "flag", "url" => "#{Web.base_url()}/tag/flag"},
+               %{"name" => "all", "url" => "#{Web.base_url()}/tag/all"},
+               %{"name" => "events", "url" => "#{Web.base_url()}/tag/events"},
+               %{"name" => "properties", "url" => "#{Web.base_url()}/tag/properties"},
+               %{
+                 "name" => "NascarBanDisplayConfederateFlagAllEventsProperties",
+                 "url" =>
+                   "#{Web.base_url()}/tag/NascarBanDisplayConfederateFlagAllEventsProperties"
+               }
+             ]
     end
 
     test "excludes a blocked users from search results", %{conn: conn} do

From f9dcf15ecb684b4b802d731a216448c76913d462 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 12 Jun 2020 14:49:54 +0300
Subject: [PATCH 198/283] added admin api for MediaProxy cache invalidation

---
 .../media_proxy_cache_controller.ex           |  38 ++++++
 .../admin_api/views/media_proxy_cache_view.ex |  11 ++
 .../admin/media_proxy_cache_operation.ex      | 109 ++++++++++++++++++
 lib/pleroma/web/router.ex                     |   4 +
 .../media_proxy_cache_controller_test.exs     |  66 +++++++++++
 5 files changed, 228 insertions(+)
 create mode 100644 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
 create mode 100644 lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
 create mode 100644 lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
 create mode 100644 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs

diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
new file mode 100644
index 000000000..7b28f7c72
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.ApiSpec.Admin, as: Spec
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
+
+  def index(%{assigns: %{user: _}} = conn, _) do
+    render(conn, "index.json", urls: [])
+  end
+
+  def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
+    render(conn, "index.json", urls: urls)
+  end
+
+  def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do
+    render(conn, "index.json", urls: urls)
+  end
+end
diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
new file mode 100644
index 000000000..c97400beb
--- /dev/null
+++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{urls: urls}) do
+    %{urls: urls}
+  end
+end
diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
new file mode 100644
index 000000000..0358cfbad
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
@@ -0,0 +1,109 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "MediaProxyCache"],
+      summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
+      operationId: "AdminAPI.MediaProxyCacheController.index",
+      security: [%{"oAuth" => ["read:media_proxy_caches"]}],
+      parameters: [
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of statuses to return"
+        )
+      ],
+      responses: %{
+        200 => success_response()
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "MediaProxyCache"],
+      summary: "Remove a banned MediaProxy URL from Cachex",
+      operationId: "AdminAPI.MediaProxyCacheController.delete",
+      security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            required: [:urls],
+            properties: %{
+              urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        200 => success_response(),
+        400 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  def purge_operation do
+    %Operation{
+      tags: ["Admin", "MediaProxyCache"],
+      summary: "Purge and optionally ban a MediaProxy URL",
+      operationId: "AdminAPI.MediaProxyCacheController.purge",
+      security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            required: [:urls],
+            properties: %{
+              urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
+              ban: %Schema{type: :boolean, default: true}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        200 => success_response(),
+        400 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp success_response do
+    Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
+      type: :object,
+      properties: %{
+        urls: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :string,
+            format: :uri,
+            description: "MediaProxy URLs"
+          }
+        }
+      }
+    })
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 57570b672..eda74a171 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do
     post("/oauth_app", OAuthAppController, :create)
     patch("/oauth_app/:id", OAuthAppController, :update)
     delete("/oauth_app/:id", OAuthAppController, :delete)
+
+    get("/media_proxy_caches", MediaProxyCacheController, :index)
+    post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
+    post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
   end
 
   scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
new file mode 100644
index 000000000..1b1d6bc36
--- /dev/null
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -0,0 +1,66 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
+  use Pleroma.Web.ConnCase
+
+  import Pleroma.Factory
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "GET /api/pleroma/admin/media_proxy_caches" do
+    test "shows banned MediaProxy URLs", %{conn: conn} do
+      response =
+        conn
+        |> get("/api/pleroma/admin/media_proxy_caches")
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == []
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do
+    test "deleted MediaProxy URLs from banned", %{conn: conn} do
+      response =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
+          urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"]
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == [
+               "http://example.com/media/a688346.jpg",
+               "http://example.com/media/fb1f4d.jpg"
+             ]
+    end
+  end
+
+  describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do
+    test "perform invalidates cache of MediaProxy", %{conn: conn} do
+      response =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
+          urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"]
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == [
+               "http://example.com/media/a688346.jpg",
+               "http://example.com/media/fb1f4d.jpg"
+             ]
+    end
+  end
+end

From c2048f75cd09696e30b443423cae4ba6ef3e593b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 12 Jun 2020 08:42:23 -0500
Subject: [PATCH 199/283] Add changelog entry for emoji pack reload command

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cf2210f5..b19cae8b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add support for filtering replies in public and home timelines
 - Admin API: endpoints for create/update/delete OAuth Apps.
 - Admin API: endpoint for status view.
+- OTP: Add command to reload emoji packs
 </details>
 
 ### Fixed

From e505e59d9c43db286ccf7fe70da2fa974ae3d700 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 12 Jun 2020 08:51:11 -0500
Subject: [PATCH 200/283] Document new mix task feature to reset mfa

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1cf2210f5..c23beec9e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `filename_display_max_length` option to set filename truncate limit, if filename display enabled (0 = no limit).
 - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
 - Mix task to create trusted OAuth App.
+- Mix task to reset MFA for user accounts
 - Notifications: Added `follow_request` notification type.
 - Added `:reject_deletes` group to SimplePolicy
 - MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances

From 4655407451c8dd05b6024f607e598359047efce2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 12 Jun 2020 14:03:33 +0000
Subject: [PATCH 201/283] Apply suggestion to config/description.exs

---
 config/description.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/description.exs b/config/description.exs
index 086a28ace..add1601e2 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1476,7 +1476,7 @@
     key: :mrf_activity_expiration,
     label: "MRF Activity Expiration Policy",
     type: :group,
-    description: "Adds expiration to all local Create activities",
+    description: "Adds expiration to all local Create Note activities",
     children: [
       %{
         key: :days,

From 09d31d24de568aac06fe203beeb8bb2a9de8f602 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 12 Jun 2020 18:37:32 +0400
Subject: [PATCH 202/283] Return an empty map from
 Pleroma.Web.RichMedia.Parsers.OGP.parse/2

---
 lib/pleroma/web/rich_media/parsers/ogp.ex | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/rich_media/parsers/ogp.ex b/lib/pleroma/web/rich_media/parsers/ogp.ex
index 363815f81..b3b3b059c 100644
--- a/lib/pleroma/web/rich_media/parsers/ogp.ex
+++ b/lib/pleroma/web/rich_media/parsers/ogp.ex
@@ -4,12 +4,7 @@
 
 defmodule Pleroma.Web.RichMedia.Parsers.OGP do
   @deprecated "OGP parser is deprecated. Use TwitterCard instead."
-  def parse(html, data) do
-    Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse(
-      data,
-      html,
-      "og",
-      "property"
-    )
+  def parse(_html, _data) do
+    %{}
   end
 end

From 520367d6fd8a268e0bc8c145a46aca46a62e8b66 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 9 Jun 2020 21:49:24 +0400
Subject: [PATCH 203/283] Fix atom leak in Rich Media Parser

---
 .../web/mastodon_api/views/status_view.ex     | 14 ++--
 lib/pleroma/web/rich_media/helpers.ex         |  6 +-
 lib/pleroma/web/rich_media/parser.ex          | 12 +--
 .../rich_media/parsers/meta_tags_parser.ex    |  8 +-
 .../web/rich_media/parsers/oembed_parser.ex   | 18 ++---
 test/web/rich_media/parser_test.exs           | 75 ++++++++++---------
 .../rich_media/parsers/twitter_card_test.exs  | 60 +++++++--------
 7 files changed, 91 insertions(+), 102 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 8e3715093..2c49bedb3 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -377,8 +377,8 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
     page_url_data = URI.parse(page_url)
 
     page_url_data =
-      if rich_media[:url] != nil do
-        URI.merge(page_url_data, URI.parse(rich_media[:url]))
+      if is_binary(rich_media["url"]) do
+        URI.merge(page_url_data, URI.parse(rich_media["url"]))
       else
         page_url_data
       end
@@ -386,11 +386,9 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
     page_url = page_url_data |> to_string
 
     image_url =
-      if rich_media[:image] != nil do
-        URI.merge(page_url_data, URI.parse(rich_media[:image]))
+      if is_binary(rich_media["image"]) do
+        URI.merge(page_url_data, URI.parse(rich_media["image"]))
         |> to_string
-      else
-        nil
       end
 
     %{
@@ -399,8 +397,8 @@ def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
       provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
       url: page_url,
       image: image_url |> MediaProxy.url(),
-      title: rich_media[:title] || "",
-      description: rich_media[:description] || "",
+      title: rich_media["title"] || "",
+      description: rich_media["description"] || "",
       pleroma: %{
         opengraph: rich_media
       }
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 9d3d7f978..1729141e9 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
-  @spec validate_page_url(any()) :: :ok | :error
+  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
 
@@ -18,8 +18,8 @@ defp validate_page_url(page_url) when is_binary(page_url) do
     |> parse_uri(page_url)
   end
 
-  defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
-       when scheme == "https" and not is_nil(authority) do
+  defp validate_page_url(%URI{host: host, scheme: "https", authority: authority})
+       when is_binary(authority) do
     cond do
       host in Config.get([:rich_media, :ignore_hosts], []) ->
         :error
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 40980def8..d9b5068b1 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -91,7 +91,7 @@ defp parse_url(url) do
       html
       |> parse_html()
       |> maybe_parse()
-      |> Map.put(:url, url)
+      |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
     rescue
@@ -111,8 +111,8 @@ defp maybe_parse(html) do
     end)
   end
 
-  defp check_parsed_data(%{title: title} = data)
-       when is_binary(title) and byte_size(title) > 0 do
+  defp check_parsed_data(%{"title" => title} = data)
+       when is_binary(title) and title != "" do
     {:ok, data}
   end
 
@@ -123,11 +123,7 @@ defp check_parsed_data(data) do
   defp clean_parsed_data(data) do
     data
     |> Enum.reject(fn {key, val} ->
-      with {:ok, _} <- Jason.encode(%{key => val}) do
-        false
-      else
-        _ -> true
-      end
+      not match?({:ok, _}, Jason.encode(%{key => val}))
     end)
     |> Map.new()
   end
diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
index ae0f36702..2762b5902 100644
--- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
@@ -29,19 +29,19 @@ defp normalize_attributes(html_node, prefix, key_name, value_name) do
     {_tag, attributes, _children} = html_node
 
     data =
-      Enum.into(attributes, %{}, fn {name, value} ->
+      Map.new(attributes, fn {name, value} ->
         {name, String.trim_leading(value, "#{prefix}:")}
       end)
 
-    %{String.to_atom(data[key_name]) => data[value_name]}
+    %{data[key_name] => data[value_name]}
   end
 
-  defp maybe_put_title(%{title: _} = meta, _), do: meta
+  defp maybe_put_title(%{"title" => _} = meta, _), do: meta
 
   defp maybe_put_title(meta, html) when meta != %{} do
     case get_page_title(html) do
       "" -> meta
-      title -> Map.put_new(meta, :title, title)
+      title -> Map.put_new(meta, "title", title)
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
index 8f32bf91b..db8ccf15d 100644
--- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   def parse(html, _data) do
     with elements = [_ | _] <- get_discovery_data(html),
-         {:ok, oembed_url} <- get_oembed_url(elements),
+         oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
          {:ok, oembed_data} <- get_oembed_data(oembed_url) do
       {:ok, oembed_data}
     else
@@ -17,19 +17,13 @@ defp get_discovery_data(html) do
     html |> Floki.find("link[type='application/json+oembed']")
   end
 
-  defp get_oembed_url(nodes) do
-    {"link", attributes, _children} = nodes |> hd()
-
-    {:ok, Enum.into(attributes, %{})["href"]}
+  defp get_oembed_url([{"link", attributes, _children} | _]) do
+    Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
   end
 
   defp get_oembed_data(url) do
-    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
-
-    {:ok, data} = Jason.decode(json)
-
-    data = data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end)
-
-    {:ok, data}
+    with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do
+      Jason.decode(json)
+    end
   end
 end
diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs
index e54a13bc8..420a612c6 100644
--- a/test/web/rich_media/parser_test.exs
+++ b/test/web/rich_media/parser_test.exs
@@ -60,19 +60,19 @@ test "returns error when no metadata present" do
   test "doesn't just add a title" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/non-ogp") ==
              {:error,
-              "Found metadata was invalid or incomplete: %{url: \"http://example.com/non-ogp\"}"}
+              "Found metadata was invalid or incomplete: %{\"url\" => \"http://example.com/non-ogp\"}"}
   end
 
   test "parses ogp" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp") ==
              {:ok,
               %{
-                image: "http://ia.media-imdb.com/images/rock.jpg",
-                title: "The Rock",
-                description:
+                "image" => "http://ia.media-imdb.com/images/rock.jpg",
+                "title" => "The Rock",
+                "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
-                type: "video.movie",
-                url: "http://example.com/ogp"
+                "type" => "video.movie",
+                "url" => "http://example.com/ogp"
               }}
   end
 
@@ -80,12 +80,12 @@ test "falls back to <title> when ogp:title is missing" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp-missing-title") ==
              {:ok,
               %{
-                image: "http://ia.media-imdb.com/images/rock.jpg",
-                title: "The Rock (1996)",
-                description:
+                "image" => "http://ia.media-imdb.com/images/rock.jpg",
+                "title" => "The Rock (1996)",
+                "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
-                type: "video.movie",
-                url: "http://example.com/ogp-missing-title"
+                "type" => "video.movie",
+                "url" => "http://example.com/ogp-missing-title"
               }}
   end
 
@@ -93,12 +93,12 @@ test "parses twitter card" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/twitter-card") ==
              {:ok,
               %{
-                card: "summary",
-                site: "@flickr",
-                image: "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
-                title: "Small Island Developing States Photo Submission",
-                description: "View the album on Flickr.",
-                url: "http://example.com/twitter-card"
+                "card" => "summary",
+                "site" => "@flickr",
+                "image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
+                "title" => "Small Island Developing States Photo Submission",
+                "description" => "View the album on Flickr.",
+                "url" => "http://example.com/twitter-card"
               }}
   end
 
@@ -106,27 +106,28 @@ test "parses OEmbed" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/oembed") ==
              {:ok,
               %{
-                author_name: "‮‭‬bees‬",
-                author_url: "https://www.flickr.com/photos/bees/",
-                cache_age: 3600,
-                flickr_type: "photo",
-                height: "768",
-                html:
+                "author_name" => "‮‭‬bees‬",
+                "author_url" => "https://www.flickr.com/photos/bees/",
+                "cache_age" => 3600,
+                "flickr_type" => "photo",
+                "height" => "768",
+                "html" =>
                   "<a data-flickr-embed=\"true\" href=\"https://www.flickr.com/photos/bees/2362225867/\" title=\"Bacon Lollys by ‮‭‬bees‬, on Flickr\"><img src=\"https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_b.jpg\" width=\"1024\" height=\"768\" alt=\"Bacon Lollys\"></a><script async src=\"https://embedr.flickr.com/assets/client-code.js\" charset=\"utf-8\"></script>",
-                license: "All Rights Reserved",
-                license_id: 0,
-                provider_name: "Flickr",
-                provider_url: "https://www.flickr.com/",
-                thumbnail_height: 150,
-                thumbnail_url: "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_q.jpg",
-                thumbnail_width: 150,
-                title: "Bacon Lollys",
-                type: "photo",
-                url: "http://example.com/oembed",
-                version: "1.0",
-                web_page: "https://www.flickr.com/photos/bees/2362225867/",
-                web_page_short_url: "https://flic.kr/p/4AK2sc",
-                width: "1024"
+                "license" => "All Rights Reserved",
+                "license_id" => 0,
+                "provider_name" => "Flickr",
+                "provider_url" => "https://www.flickr.com/",
+                "thumbnail_height" => 150,
+                "thumbnail_url" =>
+                  "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_q.jpg",
+                "thumbnail_width" => 150,
+                "title" => "Bacon Lollys",
+                "type" => "photo",
+                "url" => "http://example.com/oembed",
+                "version" => "1.0",
+                "web_page" => "https://www.flickr.com/photos/bees/2362225867/",
+                "web_page_short_url" => "https://flic.kr/p/4AK2sc",
+                "width" => "1024"
               }}
   end
 
diff --git a/test/web/rich_media/parsers/twitter_card_test.exs b/test/web/rich_media/parsers/twitter_card_test.exs
index 87c767c15..847623535 100644
--- a/test/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/web/rich_media/parsers/twitter_card_test.exs
@@ -19,11 +19,11 @@ test "parses twitter card with only name attributes" do
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
               %{
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622",
-                site: nil,
-                title:
+                "app:id:googleplay" => "com.nytimes.android",
+                "app:name:googleplay" => "NYTimes",
+                "app:url:googleplay" => "nytimes://reader/id/100000006583622",
+                "site" => nil,
+                "title" =>
                   "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times"
               }}
   end
@@ -36,15 +36,15 @@ test "parses twitter card with only property attributes" do
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
               %{
-                card: "summary_large_image",
-                description:
+                "card" => "summary_large_image",
+                "description" =>
                   "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-                image:
+                "image" =>
                   "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-                "image:alt": "",
-                title:
+                "image:alt" => "",
+                "title" =>
                   "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-                url:
+                "url" =>
                   "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
               }}
   end
@@ -57,19 +57,19 @@ test "parses twitter card with name & property attributes" do
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
               %{
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622",
-                card: "summary_large_image",
-                description:
+                "app:id:googleplay" => "com.nytimes.android",
+                "app:name:googleplay" => "NYTimes",
+                "app:url:googleplay" => "nytimes://reader/id/100000006583622",
+                "card" => "summary_large_image",
+                "description" =>
                   "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-                image:
+                "image" =>
                   "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-                "image:alt": "",
-                site: nil,
-                title:
+                "image:alt" => "",
+                "site" => nil,
+                "title" =>
                   "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-                url:
+                "url" =>
                   "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
               }}
   end
@@ -86,11 +86,11 @@ test "respect only first title tag on the page" do
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
               %{
-                site: "@atlasobscura",
-                title:
+                "site" => "@atlasobscura",
+                "title" =>
                   "The Missing Grave of Margaret Corbin, Revolutionary War Veteran - Atlas Obscura",
-                card: "summary_large_image",
-                image: image_path
+                "card" => "summary_large_image",
+                "image" => image_path
               }}
   end
 
@@ -102,12 +102,12 @@ test "takes first founded title in html head if there is html markup error" do
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
               %{
-                site: nil,
-                title:
+                "site" => nil,
+                "title" =>
                   "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
-                "app:id:googleplay": "com.nytimes.android",
-                "app:name:googleplay": "NYTimes",
-                "app:url:googleplay": "nytimes://reader/id/100000006583622"
+                "app:id:googleplay" => "com.nytimes.android",
+                "app:name:googleplay" => "NYTimes",
+                "app:url:googleplay" => "nytimes://reader/id/100000006583622"
               }}
   end
 end

From cb7be6eef252216d7ba5d5f72c8005d66b04986c Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 10 Jun 2020 17:34:23 +0200
Subject: [PATCH 204/283] Remove use of atoms in MRF.UserAllowListPolicy

---
 config/description.exs                        |  6 ++---
 docs/configuration/cheatsheet.md              |  5 ++--
 lib/pleroma/config/deprecation_warnings.ex    | 25 ++++++++++++++++++-
 .../mrf/user_allow_list_policy.ex             |  2 +-
 .../mrf/user_allowlist_policy_test.exs        |  6 ++---
 5 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index add1601e2..2f1eaf5f2 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1623,14 +1623,12 @@
   # %{
   #   group: :pleroma,
   #   key: :mrf_user_allowlist,
-  #   type: :group,
+  #   type: :map,
   #   description:
   #     "The keys in this section are the domain names that the policy should apply to." <>
   #       " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
-  #   children: [
-  #     ["example.org": ["https://example.org/users/admin"]],
   #     suggestions: [
-  #       ["example.org": ["https://example.org/users/admin"]]
+  #       %{"example.org" => ["https://example.org/users/admin"]}
   #     ]
   #   ]
   # },
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 456762151..fad67fc4d 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -138,8 +138,9 @@ their ActivityPub ID.
 An example:
 
 ```elixir
-config :pleroma, :mrf_user_allowlist,
-  "example.org": ["https://example.org/users/admin"]
+config :pleroma, :mrf_user_allowlist, %{
+  "example.org" => ["https://example.org/users/admin"]
+}
 ```
 
 #### :mrf_object_age
diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index c39a8984b..b68ded01f 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -4,9 +4,10 @@
 
 defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
+  alias Pleroma.Config
 
   def check_hellthread_threshold do
-    if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
+    if Config.get([:mrf_hellthread, :threshold]) do
       Logger.warn("""
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
@@ -14,7 +15,29 @@ def check_hellthread_threshold do
     end
   end
 
+  def mrf_user_allowlist do
+    config = Config.get(:mrf_user_allowlist)
+
+    if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
+      rewritten =
+        Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
+          Map.put(acc, to_string(k), v)
+        end)
+
+      Config.put(:mrf_user_allowlist, rewritten)
+
+      Logger.error("""
+      !!!DEPRECATION WARNING!!!
+      As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
+      Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
+
+      config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
+      """)
+    end
+  end
+
   def warn do
     check_hellthread_threshold()
+    mrf_user_allowlist()
   end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index a927a4ed8..651aed70f 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -24,7 +24,7 @@ def filter(%{"actor" => actor} = object) do
 
     allow_list =
       Config.get(
-        [:mrf_user_allowlist, String.to_atom(actor_info.host)],
+        [:mrf_user_allowlist, actor_info.host],
         []
       )
 
diff --git a/test/web/activity_pub/mrf/user_allowlist_policy_test.exs b/test/web/activity_pub/mrf/user_allowlist_policy_test.exs
index 724bae058..ba1b69658 100644
--- a/test/web/activity_pub/mrf/user_allowlist_policy_test.exs
+++ b/test/web/activity_pub/mrf/user_allowlist_policy_test.exs
@@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicyTest do
 
   alias Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy
 
-  setup do: clear_config([:mrf_user_allowlist, :localhost])
+  setup do: clear_config(:mrf_user_allowlist)
 
   test "pass filter if allow list is empty" do
     actor = insert(:user)
@@ -17,14 +17,14 @@ test "pass filter if allow list is empty" do
 
   test "pass filter if allow list isn't empty and user in allow list" do
     actor = insert(:user)
-    Pleroma.Config.put([:mrf_user_allowlist, :localhost], [actor.ap_id, "test-ap-id"])
+    Pleroma.Config.put([:mrf_user_allowlist], %{"localhost" => [actor.ap_id, "test-ap-id"]})
     message = %{"actor" => actor.ap_id}
     assert UserAllowListPolicy.filter(message) == {:ok, message}
   end
 
   test "rejected if allow list isn't empty and user not in allow list" do
     actor = insert(:user)
-    Pleroma.Config.put([:mrf_user_allowlist, :localhost], ["test-ap-id"])
+    Pleroma.Config.put([:mrf_user_allowlist], %{"localhost" => ["test-ap-id"]})
     message = %{"actor" => actor.ap_id}
     assert UserAllowListPolicy.filter(message) == {:reject, nil}
   end

From 4b865bba107b0db1de886cefd14227454cbece1e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 13 Jun 2020 10:37:15 +0000
Subject: [PATCH 205/283] Apply suggestion to
 lib/pleroma/web/controller_helper.ex

---
 lib/pleroma/web/controller_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index a5eb3e9e0..d5e9c33f5 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -75,7 +75,7 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
     # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
     #  This is because we only have ids present inside of the page, while
     # `min_id`, `since_id` and `max_id` requires to know one outside of it.
-    if Map.take(conn.params, @id_keys) != [] do
+    if Map.take(conn.params, @id_keys) != %{} do
       Map.put(fields, "id", current_url(conn))
     else
       fields

From 1d625c29a09cf7c0fb415d5606a91315902efaad Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 13 Jun 2020 13:12:43 +0200
Subject: [PATCH 206/283] ControllerHelper: Always return id field.

---
 lib/pleroma/web/controller_helper.ex | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index d5e9c33f5..69946fb81 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -65,21 +65,11 @@ defp build_pagination_fields(conn, min_id, max_id, extra_params) do
       |> Map.merge(extra_params)
       |> Map.drop(@id_keys)
 
-    fields = %{
+    %{
       "next" => current_url(conn, Map.put(params, :max_id, max_id)),
-      "prev" => current_url(conn, Map.put(params, :min_id, min_id))
+      "prev" => current_url(conn, Map.put(params, :min_id, min_id)),
+      "id" => current_url(conn)
     }
-
-    #  Generating an `id` without already present pagination keys would
-    # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
-    # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
-    #  This is because we only have ids present inside of the page, while
-    # `min_id`, `since_id` and `max_id` requires to know one outside of it.
-    if Map.take(conn.params, @id_keys) != %{} do
-      Map.put(fields, "id", current_url(conn))
-    else
-      fields
-    end
   end
 
   def get_pagination_fields(conn, activities, extra_params \\ %{}) do

From b15cfc3d365dcfa5f99159fe06e29de6f8aceb4f Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Mon, 18 May 2020 18:46:04 +0300
Subject: [PATCH 207/283] Mastodon API: ensure the notification endpoint
 doesn't return less than the requested amount of records unless it's the last
 page

---
 CHANGELOG.md                                  |  1 +
 lib/pleroma/notification.ex                   | 19 +++++-
 lib/pleroma/user.ex                           |  8 +++
 .../mastodon_api/views/notification_view.ex   | 68 +++++++++----------
 ...ete_notifications_from_invisible_users.exs | 18 +++++
 test/notification_test.exs                    |  8 +++
 .../notification_controller_test.exs          | 27 ++++++++
 .../views/notification_view_test.exs          |  4 +-
 8 files changed, 112 insertions(+), 41 deletions(-)
 create mode 100644 priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9361fa260..b3f2dd10f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Filtering of push notifications on activities from blocked domains
 - Resolving Peertube accounts with Webfinger
 - `blob:` urls not being allowed by connect-src CSP
+- Mastodon API: fix `GET /api/v1/notifications` not returning the full result set
 
 ## [Unreleased (patch)]
 
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 3386a1933..9ee9606be 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -166,8 +166,16 @@ defp exclude_visibility(query, %{exclude_visibilities: visibility})
       query
       |> join(:left, [n, a], mutated_activity in Pleroma.Activity,
         on:
-          fragment("?->>'context'", a.data) ==
-            fragment("?->>'context'", mutated_activity.data) and
+          fragment(
+            "COALESCE((?->'object')->>'id', ?->>'object')",
+            a.data,
+            a.data
+          ) ==
+            fragment(
+              "COALESCE((?->'object')->>'id', ?->>'object')",
+              mutated_activity.data,
+              mutated_activity.data
+            ) and
             fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
             fragment("?->>'type'", mutated_activity.data) == "Create",
         as: :mutated_activity
@@ -541,6 +549,7 @@ def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
   def skip?(%Activity{} = activity, %User{} = user) do
     [
       :self,
+      :invisible,
       :followers,
       :follows,
       :non_followers,
@@ -557,6 +566,12 @@ def skip?(:self, %Activity{} = activity, %User{} = user) do
     activity.data["actor"] == user.ap_id
   end
 
+  def skip?(:invisible, %Activity{} = activity, _) do
+    actor = activity.data["actor"]
+    user = User.get_cached_by_ap_id(actor)
+    User.invisible?(user)
+  end
+
   def skip?(
         :followers,
         %Activity{} = activity,
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index c5c74d132..52ac9052b 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1488,6 +1488,7 @@ def perform(:delete, %User{} = user) do
     end)
 
     delete_user_activities(user)
+    delete_notifications_from_user_activities(user)
 
     delete_outgoing_pending_follow_requests(user)
 
@@ -1576,6 +1577,13 @@ def follow_import(%User{} = follower, followed_identifiers)
     })
   end
 
+  def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
+    Notification
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> where([n, a], fragment("? = ?", a.actor, ^ap_id))
+    |> Repo.delete_all()
+  end
+
   def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index b11578623..3865be280 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -46,6 +46,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op
             activities
             |> Enum.filter(&(&1.data["type"] == "Move"))
             |> Enum.map(&User.get_cached_by_ap_id(&1.data["target"]))
+            |> Enum.filter(& &1)
 
           actors =
             activities
@@ -84,50 +85,45 @@ def render(
     # Note: :relationships contain user mutes (needed for :muted flag in :status)
     status_render_opts = %{relationships: opts[:relationships]}
 
-    with %{id: _} = account <-
-           AccountView.render(
-             "show.json",
-             %{user: actor, for: reading_user}
-           ) do
-      response = %{
-        id: to_string(notification.id),
-        type: notification.type,
-        created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
-        account: account,
-        pleroma: %{
-          is_seen: notification.seen
-        }
+    account =
+      AccountView.render(
+        "show.json",
+        %{user: actor, for: reading_user}
+      )
+
+    response = %{
+      id: to_string(notification.id),
+      type: notification.type,
+      created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
+      account: account,
+      pleroma: %{
+        is_seen: notification.seen
       }
+    }
 
-      case notification.type do
-        "mention" ->
-          put_status(response, activity, reading_user, status_render_opts)
+    case notification.type do
+      "mention" ->
+        put_status(response, activity, reading_user, status_render_opts)
 
-        "favourite" ->
-          put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
+      "favourite" ->
+        put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
 
-        "reblog" ->
-          put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
+      "reblog" ->
+        put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
 
-        "move" ->
-          put_target(response, activity, reading_user, %{})
+      "move" ->
+        put_target(response, activity, reading_user, %{})
 
-        "pleroma:emoji_reaction" ->
-          response
-          |> put_status(parent_activity_fn.(), reading_user, status_render_opts)
-          |> put_emoji(activity)
+      "pleroma:emoji_reaction" ->
+        response
+        |> put_status(parent_activity_fn.(), reading_user, status_render_opts)
+        |> put_emoji(activity)
 
-        "pleroma:chat_mention" ->
-          put_chat_message(response, activity, reading_user, status_render_opts)
+      "pleroma:chat_mention" ->
+        put_chat_message(response, activity, reading_user, status_render_opts)
 
-        type when type in ["follow", "follow_request"] ->
-          response
-
-        _ ->
-          nil
-      end
-    else
-      _ -> nil
+      type when type in ["follow", "follow_request"] ->
+        response
     end
   end
 
diff --git a/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs b/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs
new file mode 100644
index 000000000..9e95a8111
--- /dev/null
+++ b/priv/repo/migrations/20200527163635_delete_notifications_from_invisible_users.exs
@@ -0,0 +1,18 @@
+defmodule Pleroma.Repo.Migrations.DeleteNotificationsFromInvisibleUsers do
+  use Ecto.Migration
+
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    Pleroma.Notification
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> where(
+      [n, a],
+      fragment("? in (SELECT ap_id FROM users WHERE invisible = true)", a.actor)
+    )
+    |> Repo.delete_all()
+  end
+
+  def down, do: :ok
+end
diff --git a/test/notification_test.exs b/test/notification_test.exs
index b9bbdceca..526f43fab 100644
--- a/test/notification_test.exs
+++ b/test/notification_test.exs
@@ -306,6 +306,14 @@ test "it doesn't create subscription notifications if the recipient cannot see t
 
       assert {:ok, []} == Notification.create_notifications(status)
     end
+
+    test "it disables notifications from people who are invisible" do
+      author = insert(:user, invisible: true)
+      user = insert(:user)
+
+      {:ok, status} = CommonAPI.post(author, %{status: "hey @#{user.nickname}"})
+      refute Notification.create_notification(status, user)
+    end
   end
 
   describe "follow / follow_request notifications" do
diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs
index 698c99711..70ef0e8b5 100644
--- a/test/web/mastodon_api/controllers/notification_controller_test.exs
+++ b/test/web/mastodon_api/controllers/notification_controller_test.exs
@@ -313,6 +313,33 @@ test "filters notifications for Announce activities" do
       assert public_activity.id in activity_ids
       refute unlisted_activity.id in activity_ids
     end
+
+    test "doesn't return less than the requested amount of records when the user's reply is liked" do
+      user = insert(:user)
+      %{user: other_user, conn: conn} = oauth_access(["read:notifications"])
+
+      {:ok, mention} =
+        CommonAPI.post(user, %{status: "@#{other_user.nickname}", visibility: "public"})
+
+      {:ok, activity} = CommonAPI.post(user, %{status: ".", visibility: "public"})
+
+      {:ok, reply} =
+        CommonAPI.post(other_user, %{
+          status: ".",
+          visibility: "public",
+          in_reply_to_status_id: activity.id
+        })
+
+      {:ok, _favorite} = CommonAPI.favorite(user, reply.id)
+
+      activity_ids =
+        conn
+        |> get("/api/v1/notifications?exclude_visibilities[]=direct&limit=2")
+        |> json_response_and_validate_schema(200)
+        |> Enum.map(& &1["status"]["id"])
+
+      assert [reply.id, mention.id] == activity_ids
+    end
   end
 
   test "filters notifications using exclude_types" do
diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs
index b2fa5b302..9c399b2df 100644
--- a/test/web/mastodon_api/views/notification_view_test.exs
+++ b/test/web/mastodon_api/views/notification_view_test.exs
@@ -139,9 +139,7 @@ test "Follow notification" do
     test_notifications_rendering([notification], followed, [expected])
 
     User.perform(:delete, follower)
-    notification = Notification |> Repo.one() |> Repo.preload(:activity)
-
-    test_notifications_rendering([notification], followed, [])
+    refute Repo.one(Notification)
   end
 
   @tag capture_log: true

From 2e8a236cef28c0b754aecb04a5c60c3b7655c5a6 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Sun, 14 Jun 2020 21:02:57 +0300
Subject: [PATCH 208/283] fix invalidates media url's

---
 config/config.exs                             |   7 +
 config/description.exs                        |  64 +++++++++
 docs/configuration/cheatsheet.md              |   6 +-
 installation/nginx-cache-purge.sh.example     |   4 +-
 lib/pleroma/application.ex                    |   3 +-
 lib/pleroma/plugs/uploaded_media.ex           |  16 ++-
 lib/pleroma/web/media_proxy/invalidation.ex   |  29 ++--
 .../web/media_proxy/invalidations/http.ex     |   8 +-
 .../web/media_proxy/invalidations/script.ex   |  36 ++---
 lib/pleroma/web/media_proxy/media_proxy.ex    |  35 ++++-
 .../web/media_proxy/media_proxy_controller.ex |   3 +-
 .../workers/attachments_cleanup_worker.ex     | 133 ++++++++++--------
 test/web/media_proxy/invalidation_test.exs    |  65 +++++++++
 .../media_proxy/invalidations/http_test.exs   |  13 +-
 .../media_proxy/invalidations/script_test.exs |  21 ++-
 .../media_proxy_controller_test.exs           |  17 +++
 16 files changed, 346 insertions(+), 114 deletions(-)
 create mode 100644 test/web/media_proxy/invalidation_test.exs

diff --git a/config/config.exs b/config/config.exs
index 9508ae077..e299fb8dd 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -406,6 +406,13 @@
   ],
   whitelist: []
 
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
+  method: :purge,
+  headers: [],
+  options: []
+
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
+
 config :pleroma, :chat, enabled: true
 
 config :phoenix, :format_encoders, json: Jason
diff --git a/config/description.exs b/config/description.exs
index 807c945e0..857293794 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1637,6 +1637,31 @@
           "The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.",
         suggestions: ["https://example.com"]
       },
+      %{
+        key: :invalidation,
+        type: :keyword,
+        descpiption: "",
+        suggestions: [
+          enabled: true,
+          provider: Pleroma.Web.MediaProxy.Invalidation.Script
+        ],
+        children: [
+          %{
+            key: :enabled,
+            type: :boolean,
+            description: "Enables invalidate media cache"
+          },
+          %{
+            key: :provider,
+            type: :module,
+            description: "Module which will be used to cache purge.",
+            suggestions: [
+              Pleroma.Web.MediaProxy.Invalidation.Script,
+              Pleroma.Web.MediaProxy.Invalidation.Http
+            ]
+          }
+        ]
+      },
       %{
         key: :proxy_opts,
         type: :keyword,
@@ -1709,6 +1734,45 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Web.MediaProxy.Invalidation.Http,
+    type: :group,
+    description: "HTTP invalidate settings",
+    children: [
+      %{
+        key: :method,
+        type: :atom,
+        description: "HTTP method of request. Default: :purge"
+      },
+      %{
+        key: :headers,
+        type: {:list, :tuple},
+        description: "HTTP headers of request.",
+        suggestions: [{"x-refresh", 1}]
+      },
+      %{
+        key: :options,
+        type: :keyword,
+        description: "Request options.",
+        suggestions: [params: %{ts: "xxx"}]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.Web.MediaProxy.Invalidation.Script,
+    type: :group,
+    description: "Script invalidate settings",
+    children: [
+      %{
+        key: :script_path,
+        type: :string,
+        description: "Path to shell script. Which will run purge cache.",
+        suggestions: ["./installation/nginx-cache-purge.sh.example"]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :gopher,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 505acb293..20bd0ed85 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -262,7 +262,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
 
 #### Pleroma.Web.MediaProxy.Invalidation.Script
 
-This strategy allow perform external bash script to purge cache.
+This strategy allow perform external shell script to purge cache.
 Urls of attachments pass to script as arguments.
 
 * `script_path`: path to external script.
@@ -278,8 +278,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
 This strategy allow perform custom http request to purge cache.
 
 * `method`: http method. default is `purge`
-* `headers`: http headers. default is empty
-* `options`: request options. default is empty
+* `headers`: http headers.
+* `options`: request options.
 
 Example:
 ```elixir
diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example
index b2915321c..5f6cbb128 100755
--- a/installation/nginx-cache-purge.sh.example
+++ b/installation/nginx-cache-purge.sh.example
@@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache"
 ## $3 - (optional) the number of parallel processes to run for grep.
 get_cache_files() {
     local max_parallel=${3-16}
-    find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
+    find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u
 }
 
 ## Removes an item from the given cache zone.
@@ -37,4 +37,4 @@ purge() {
 
 }
 
-purge $1
+purge $@
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9d3d92b38..adebebc7a 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -148,7 +148,8 @@ defp cachex_children do
       build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
-      build_cachex("failed_proxy_url", limit: 2500)
+      build_cachex("failed_proxy_url", limit: 2500),
+      build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
     ]
   end
 
diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index 94147e0c4..2f3fde002 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
   import Pleroma.Web.Gettext
   require Logger
 
+  alias Pleroma.Web.MediaProxy
+
   @behaviour Plug
   # no slashes
   @path "media"
@@ -35,8 +37,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
         %{query_params: %{"name" => name}} = conn ->
           name = String.replace(name, "\"", "\\\"")
 
-          conn
-          |> put_resp_header("content-disposition", "filename=\"#{name}\"")
+          put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
 
         conn ->
           conn
@@ -47,7 +48,8 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
 
     with uploader <- Keyword.fetch!(config, :uploader),
          proxy_remote = Keyword.get(config, :proxy_remote, false),
-         {:ok, get_method} <- uploader.get_file(file) do
+         {:ok, get_method} <- uploader.get_file(file),
+         false <- media_is_deleted(conn, get_method) do
       get_media(conn, get_method, proxy_remote, opts)
     else
       _ ->
@@ -59,6 +61,14 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
 
   def call(conn, _opts), do: conn
 
+  defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do
+    MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path)
+  end
+
+  defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url)
+
+  defp media_is_deleted(_, _), do: false
+
   defp get_media(conn, {:static_dir, directory}, _, opts) do
     static_opts =
       Map.get(opts, :static_plug_opts)
diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index c037ff13e..83ff8589c 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -5,22 +5,33 @@
 defmodule Pleroma.Web.MediaProxy.Invalidation do
   @moduledoc false
 
-  @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
+  @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
 
   alias Pleroma.Config
+  alias Pleroma.Web.MediaProxy
 
-  @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
+  @spec enabled?() :: boolean()
+  def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
+
+  @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
   def purge(urls) do
-    [:media_proxy, :invalidation, :enabled]
-    |> Config.get()
-    |> do_purge(urls)
+    prepared_urls = prepare_urls(urls)
+
+    if enabled?() do
+      do_purge(prepared_urls)
+    else
+      {:ok, prepared_urls}
+    end
   end
 
-  defp do_purge(true, urls) do
+  defp do_purge(urls) do
     provider = Config.get([:media_proxy, :invalidation, :provider])
-    options = Config.get(provider)
-    provider.purge(urls, options)
+    provider.purge(urls, Config.get(provider))
   end
 
-  defp do_purge(_, _), do: :ok
+  def prepare_urls(urls) do
+    urls
+    |> List.wrap()
+    |> Enum.map(&MediaProxy.url(&1))
+  end
 end
diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index 07248df6e..3694b56e8 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -10,9 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
 
   @impl Pleroma.Web.MediaProxy.Invalidation
   def purge(urls, opts) do
-    method = Map.get(opts, :method, :purge)
-    headers = Map.get(opts, :headers, [])
-    options = Map.get(opts, :options, [])
+    method = Keyword.get(opts, :method, :purge)
+    headers = Keyword.get(opts, :headers, [])
+    options = Keyword.get(opts, :options, [])
 
     Logger.debug("Running cache purge: #{inspect(urls)}")
 
@@ -22,7 +22,7 @@ def purge(urls, opts) do
       end
     end)
 
-    {:ok, "success"}
+    {:ok, urls}
   end
 
   defp do_purge(method, url, headers, options) do
diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index 6be782132..d41d647bb 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
   require Logger
 
   @impl Pleroma.Web.MediaProxy.Invalidation
-  def purge(urls, %{script_path: script_path} = _options) do
+  def purge(urls, opts) do
     args =
       urls
       |> List.wrap()
       |> Enum.uniq()
       |> Enum.join(" ")
 
-    path = Path.expand(script_path)
-
-    Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
-
-    case do_purge(path, [args]) do
-      {result, exit_status} when exit_status > 0 ->
-        Logger.error("Error while cache purge: #{inspect(result)}")
-        {:error, inspect(result)}
-
-      _ ->
-        {:ok, "success"}
-    end
+    opts
+    |> Keyword.get(:script_path, nil)
+    |> do_purge([args])
+    |> handle_result(urls)
   end
 
-  def purge(_, _), do: {:error, "not found script path"}
-
-  defp do_purge(path, args) do
+  defp do_purge(script_path, args) when is_binary(script_path) do
+    path = Path.expand(script_path)
+    Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
     System.cmd(path, args)
   rescue
-    error -> {inspect(error), 1}
+    error -> error
+  end
+
+  defp do_purge(_, _), do: {:error, "not found script path"}
+
+  defp handle_result({_result, 0}, urls), do: {:ok, urls}
+  defp handle_result({:error, error}, urls), do: handle_result(error, urls)
+
+  defp handle_result(error, _) do
+    Logger.error("Error while cache purge: #{inspect(error)}")
+    {:error, inspect(error)}
   end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index b2b524524..59ca217ab 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do
   alias Pleroma.Config
   alias Pleroma.Upload
   alias Pleroma.Web
+  alias Pleroma.Web.MediaProxy.Invalidation
 
   @base64_opts [padding: false]
 
+  @spec in_deleted_urls(String.t()) :: boolean()
+  def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1)
+
+  def remove_from_deleted_urls(urls) when is_list(urls) do
+    Cachex.execute!(:deleted_urls_cache, fn cache ->
+      Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
+    end)
+  end
+
+  def remove_from_deleted_urls(url) when is_binary(url) do
+    Cachex.del(:deleted_urls_cache, url(url))
+  end
+
+  def put_in_deleted_urls(urls) when is_list(urls) do
+    Cachex.execute!(:deleted_urls_cache, fn cache ->
+      Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
+    end)
+  end
+
+  def put_in_deleted_urls(url) when is_binary(url) do
+    Cachex.put(:deleted_urls_cache, url(url), true)
+  end
+
   def url(url) when is_nil(url) or url == "", do: nil
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if disabled?() or local?(url) or whitelisted?(url) do
+    if disabled?() or not is_url_proxiable?(url) do
       url
     else
       encode_url(url)
     end
   end
 
+  @spec is_url_proxiable?(String.t()) :: boolean()
+  def is_url_proxiable?(url) do
+    if local?(url) or whitelisted?(url) do
+      false
+    else
+      true
+    end
+  end
+
   defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
 
   defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 4657a4383..ff0158d83 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -14,10 +14,11 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
     with config <- Pleroma.Config.get([:media_proxy], []),
          true <- Keyword.get(config, :enabled, false),
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
+         {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)},
          :ok <- filename_matches(params, conn.request_path, url) do
       ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
     else
-      false ->
+      error when error in [false, {:in_deleted_urls, true}] ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
index 49352db2a..4ad19c0fc 100644
--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
@@ -23,8 +23,25 @@ def perform(
         Enum.map(attachment["url"], & &1["href"])
       end)
 
-    names = Enum.map(attachments, & &1["name"])
+    # find all objects for copies of the attachments, name and actor doesn't matter here
+    hrefs
+    |> fetch_objects
+    |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
+    |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
+      with 1 <- count do
+        {ids ++ [id], hrefs ++ [href]}
+      else
+        _ -> {ids ++ [id], hrefs}
+      end
+    end)
+    |> do_clean
 
+    {:ok, :success}
+  end
+
+  def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+
+  defp do_clean({object_ids, attachment_urls}) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
     prefix =
@@ -39,68 +56,60 @@ def perform(
         "/"
       )
 
-    # find all objects for copies of the attachments, name and actor doesn't matter here
-    object_ids_and_hrefs =
-      from(o in Object,
-        where:
-          fragment(
-            "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
-            o.data,
-            o.data,
-            ^hrefs
-          )
-      )
-      # The query above can be time consumptive on large instances until we
-      # refactor how uploads are stored
-      |> Repo.all(timeout: :infinity)
-      # we should delete 1 object for any given attachment, but don't delete
-      # files if there are more than 1 object for it
-      |> Enum.reduce(%{}, fn %{
-                               id: id,
-                               data: %{
-                                 "url" => [%{"href" => href}],
-                                 "actor" => obj_actor,
-                                 "name" => name
-                               }
-                             },
-                             acc ->
-        Map.update(acc, href, %{id: id, count: 1}, fn val ->
-          case obj_actor == actor and name in names do
-            true ->
-              # set id of the actor's object that will be deleted
-              %{val | id: id, count: val.count + 1}
+    Enum.each(attachment_urls, fn href ->
+      href
+      |> String.trim_leading("#{base_url}/#{prefix}")
+      |> uploader.delete_file()
+    end)
 
-            false ->
-              # another actor's object, just increase count to not delete file
-              %{val | count: val.count + 1}
-          end
-        end)
-      end)
-      |> Enum.map(fn {href, %{id: id, count: count}} ->
-        # only delete files that have single instance
-        with 1 <- count do
-          href
-          |> String.trim_leading("#{base_url}/#{prefix}")
-          |> uploader.delete_file()
-
-          {id, href}
-        else
-          _ -> {id, nil}
-        end
-      end)
-
-    object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
-
-    from(o in Object, where: o.id in ^object_ids)
-    |> Repo.delete_all()
-
-    object_ids_and_hrefs
-    |> Enum.filter(fn {_, href} -> not is_nil(href) end)
-    |> Enum.map(&elem(&1, 1))
-    |> Pleroma.Web.MediaProxy.Invalidation.purge()
-
-    {:ok, :success}
+    delete_objects(object_ids)
   end
 
-  def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+  defp delete_objects([_ | _] = object_ids) do
+    Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
+  end
+
+  defp delete_objects(_), do: :ok
+
+  # we should delete 1 object for any given attachment, but don't delete
+  # files if there are more than 1 object for it
+  def prepare_objects(objects, actor, names) do
+    objects
+    |> Enum.reduce(%{}, fn %{
+                             id: id,
+                             data: %{
+                               "url" => [%{"href" => href}],
+                               "actor" => obj_actor,
+                               "name" => name
+                             }
+                           },
+                           acc ->
+      Map.update(acc, href, %{id: id, count: 1}, fn val ->
+        case obj_actor == actor and name in names do
+          true ->
+            # set id of the actor's object that will be deleted
+            %{val | id: id, count: val.count + 1}
+
+          false ->
+            # another actor's object, just increase count to not delete file
+            %{val | count: val.count + 1}
+        end
+      end)
+    end)
+  end
+
+  def fetch_objects(hrefs) do
+    from(o in Object,
+      where:
+        fragment(
+          "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
+          o.data,
+          o.data,
+          ^hrefs
+        )
+    )
+    # The query above can be time consumptive on large instances until we
+    # refactor how uploads are stored
+    |> Repo.all(timeout: :infinity)
+  end
 end
diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
new file mode 100644
index 000000000..3a9fa8c88
--- /dev/null
+++ b/test/web/media_proxy/invalidation_test.exs
@@ -0,0 +1,65 @@
+defmodule Pleroma.Web.MediaProxy.InvalidationTest do
+  use ExUnit.Case
+  use Pleroma.Tests.Helpers
+
+  alias Pleroma.Config
+  alias Pleroma.Web.MediaProxy.Invalidation
+
+  import ExUnit.CaptureLog
+  import Mock
+  import Tesla.Mock
+
+  setup do: clear_config([:media_proxy])
+
+  setup do
+    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    :ok
+  end
+
+  describe "Invalidation.Http" do
+    test "perform request to clear cache" do
+      Config.put([:media_proxy, :enabled], false)
+      Config.put([:media_proxy, :invalidation, :enabled], true)
+      Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
+
+      Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
+      image_url = "http://example.com/media/example.jpg"
+      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+
+      mock(fn
+        %{
+          method: :purge,
+          url: "http://example.com/media/example.jpg",
+          headers: [{"x-refresh", 1}]
+        } ->
+          %Tesla.Env{status: 200}
+      end)
+
+      assert capture_log(fn ->
+               assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+               assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+             end) =~ "Running cache purge: [\"#{image_url}\"]"
+    end
+  end
+
+  describe "Invalidation.Script" do
+    test "run script to clear cache" do
+      Config.put([:media_proxy, :enabled], false)
+      Config.put([:media_proxy, :invalidation, :enabled], true)
+      Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
+      Config.put([Invalidation.Script], script_path: "purge-nginx")
+
+      image_url = "http://example.com/media/example.jpg"
+      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+
+      with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
+        assert capture_log(fn ->
+                 assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+                 assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+                 assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               end) =~ "Running cache purge: [\"#{image_url}\"]"
+      end
+    end
+  end
+end
diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs
index 8a3b4141c..09e7ca0fb 100644
--- a/test/web/media_proxy/invalidations/http_test.exs
+++ b/test/web/media_proxy/invalidations/http_test.exs
@@ -5,6 +5,11 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
   import ExUnit.CaptureLog
   import Tesla.Mock
 
+  setup do
+    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    :ok
+  end
+
   test "logs hasn't error message when request is valid" do
     mock(fn
       %{method: :purge, url: "http://example.com/media/example.jpg"} ->
@@ -14,8 +19,8 @@ test "logs hasn't error message when request is valid" do
     refute capture_log(fn ->
              assert Invalidation.Http.purge(
                       ["http://example.com/media/example.jpg"],
-                      %{}
-                    ) == {:ok, "success"}
+                      []
+                    ) == {:ok, ["http://example.com/media/example.jpg"]}
            end) =~ "Error while cache purge"
   end
 
@@ -28,8 +33,8 @@ test "it write error message in logs when request invalid" do
     assert capture_log(fn ->
              assert Invalidation.Http.purge(
                       ["http://example.com/media/example1.jpg"],
-                      %{}
-                    ) == {:ok, "success"}
+                      []
+                    ) == {:ok, ["http://example.com/media/example1.jpg"]}
            end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
   end
 end
diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs
index 1358963ab..c69cec07a 100644
--- a/test/web/media_proxy/invalidations/script_test.exs
+++ b/test/web/media_proxy/invalidations/script_test.exs
@@ -4,17 +4,24 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
 
   import ExUnit.CaptureLog
 
+  setup do
+    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    :ok
+  end
+
   test "it logger error when script not found" do
     assert capture_log(fn ->
              assert Invalidation.Script.purge(
                       ["http://example.com/media/example.jpg"],
-                      %{script_path: "./example"}
-                    ) == {:error, "\"%ErlangError{original: :enoent}\""}
-           end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\""
+                      script_path: "./example"
+                    ) == {:error, "%ErlangError{original: :enoent}"}
+           end) =~ "Error while cache purge: %ErlangError{original: :enoent}"
 
-    assert Invalidation.Script.purge(
-             ["http://example.com/media/example.jpg"],
-             %{}
-           ) == {:error, "not found script path"}
+    capture_log(fn ->
+      assert Invalidation.Script.purge(
+               ["http://example.com/media/example.jpg"],
+               []
+             ) == {:error, "\"not found script path\""}
+    end)
   end
 end
diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs
index da79d38a5..2b6b25221 100644
--- a/test/web/media_proxy/media_proxy_controller_test.exs
+++ b/test/web/media_proxy/media_proxy_controller_test.exs
@@ -10,6 +10,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
   setup do: clear_config(:media_proxy)
   setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
 
+  setup do
+    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    :ok
+  end
+
   test "it returns 404 when MediaProxy disabled", %{conn: conn} do
     Config.put([:media_proxy, :enabled], false)
 
@@ -66,4 +71,16 @@ test "it performs ReverseProxy.call when signature valid", %{conn: conn} do
       assert %Plug.Conn{status: :success} = get(conn, url)
     end
   end
+
+  test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do
+    Config.put([:media_proxy, :enabled], true)
+    Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+    url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
+    Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png")
+
+    with_mock Pleroma.ReverseProxy,
+      call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+      assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
+    end
+  end
 end

From b7df7436c813bfcb4f27ac64c85ebc1507153601 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 15 Jun 2020 12:27:13 +0200
Subject: [PATCH 209/283] Conversations: Return last dm for conversation, not
 last message.

---
 lib/pleroma/conversation/participation.ex             | 11 +++++++----
 lib/pleroma/web/activity_pub/activity_pub.ex          |  7 ++++---
 .../web/mastodon_api/views/conversation_view.ex       | 11 +++++++----
 .../web/mastodon_api/views/conversation_view_test.exs | 11 ++++++++++-
 4 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index ce7bd2396..8bc3e85d6 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -162,10 +162,13 @@ def for_user_with_last_activity_id(user, params \\ %{}) do
     for_user(user, params)
     |> Enum.map(fn participation ->
       activity_id =
-        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
-          user: user,
-          blocking_user: user
-        })
+        ActivityPub.fetch_latest_direct_activity_id_for_context(
+          participation.conversation.ap_id,
+          %{
+            user: user,
+            blocking_user: user
+          }
+        )
 
       %{
         participation
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c9dc6135c..3e4f3ad30 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -210,7 +210,7 @@ def stream_out_participations(%Object{data: %{"context" => context}}, user) do
       conversation = Repo.preload(conversation, :participations)
 
       last_activity_id =
-        fetch_latest_activity_id_for_context(conversation.ap_id, %{
+        fetch_latest_direct_activity_id_for_context(conversation.ap_id, %{
           user: user,
           blocking_user: user
         })
@@ -517,11 +517,12 @@ def fetch_activities_for_context(context, opts \\ %{}) do
     |> Repo.all()
   end
 
-  @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) ::
+  @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) ::
           FlakeId.Ecto.CompatType.t() | nil
-  def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
+  def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do
     context
     |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts))
+    |> restrict_visibility(%{visibility: "direct"})
     |> limit(1)
     |> select([a], a.id)
     |> Repo.one()
diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index fbe618377..06f0c1728 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -23,10 +23,13 @@ def render("participation.json", %{participation: participation, for: user}) do
 
     last_activity_id =
       with nil <- participation.last_activity_id do
-        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
-          user: user,
-          blocking_user: user
-        })
+        ActivityPub.fetch_latest_direct_activity_id_for_context(
+          participation.conversation.ap_id,
+          %{
+            user: user,
+            blocking_user: user
+          }
+        )
       end
 
     activity = Activity.get_by_id_with_object(last_activity_id)
diff --git a/test/web/mastodon_api/views/conversation_view_test.exs b/test/web/mastodon_api/views/conversation_view_test.exs
index 6f84366f8..2e8203c9b 100644
--- a/test/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/web/mastodon_api/views/conversation_view_test.exs
@@ -15,8 +15,17 @@ test "represents a Mastodon Conversation entity" do
     user = insert(:user)
     other_user = insert(:user)
 
+    {:ok, parent} = CommonAPI.post(user, %{status: "parent"})
+
     {:ok, activity} =
-      CommonAPI.post(user, %{status: "hey @#{other_user.nickname}", visibility: "direct"})
+      CommonAPI.post(user, %{
+        status: "hey @#{other_user.nickname}",
+        visibility: "direct",
+        in_reply_to_id: parent.id
+      })
+
+    {:ok, _reply_activity} =
+      CommonAPI.post(user, %{status: "hu", visibility: "public", in_reply_to_id: parent.id})
 
     [participation] = Participation.for_user_with_last_activity_id(user)
 

From 1092b3650068169ece0ac95cd88ec0e4da30036b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 15 Jun 2020 12:30:11 +0200
Subject: [PATCH 210/283] Changelog: Add info about conversation view changes.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b3f2dd10f..c546f1f04 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- In Conversations, return only direct messages as `last_status`
 - MFR policy to set global expiration for all local Create activities
 <details>
   <summary>API Changes</summary>

From 62b8c31b7a84dadb2a46861fe0f2dd1dbf9d40f0 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 15 Jun 2020 14:55:00 +0300
Subject: [PATCH 211/283] added tests

---
 .../media_proxy_cache_controller.ex           |  31 ++++-
 .../web/media_proxy/invalidations/script.ex   |   2 +-
 .../media_proxy_cache_controller_test.exs     | 116 +++++++++++++++---
 3 files changed, 127 insertions(+), 22 deletions(-)

diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
index 7b28f7c72..e3fa0ac28 100644
--- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
 
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.ApiSpec.Admin, as: Spec
+  alias Pleroma.Web.MediaProxy
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 
@@ -24,15 +25,39 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
 
   defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
 
-  def index(%{assigns: %{user: _}} = conn, _) do
-    render(conn, "index.json", urls: [])
+  def index(%{assigns: %{user: _}} = conn, params) do
+    cursor =
+      :deleted_urls_cache
+      |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
+      |> :qlc.cursor()
+
+    urls =
+      case params.page do
+        1 ->
+          :qlc.next_answers(cursor, params.page_size)
+
+        _ ->
+          :qlc.next_answers(cursor, (params.page - 1) * params.page_size)
+          :qlc.next_answers(cursor, params.page_size)
+      end
+
+    :qlc.delete_cursor(cursor)
+
+    render(conn, "index.json", urls: urls)
   end
 
   def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
+    MediaProxy.remove_from_deleted_urls(urls)
     render(conn, "index.json", urls: urls)
   end
 
-  def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do
+  def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
+    MediaProxy.Invalidation.purge(urls)
+
+    if ban do
+      MediaProxy.put_in_deleted_urls(urls)
+    end
+
     render(conn, "index.json", urls: urls)
   end
 end
diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index d41d647bb..0217b119d 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
   require Logger
 
   @impl Pleroma.Web.MediaProxy.Invalidation
-  def purge(urls, opts) do
+  def purge(urls, opts \\ %{}) do
     args =
       urls
       |> List.wrap()
diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
index 1b1d6bc36..76a96f46f 100644
--- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -6,6 +6,16 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
   use Pleroma.Web.ConnCase
 
   import Pleroma.Factory
+  import Mock
+
+  alias Pleroma.Web.MediaProxy
+
+  setup do: clear_config([:media_proxy])
+
+  setup do
+    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    :ok
+  end
 
   setup do
     admin = insert(:user, is_admin: true)
@@ -16,51 +26,121 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
       |> assign(:user, admin)
       |> assign(:token, token)
 
+    Config.put([:media_proxy, :enabled], true)
+    Config.put([:media_proxy, :invalidation, :enabled], true)
+    Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
+
     {:ok, %{admin: admin, token: token, conn: conn}}
   end
 
   describe "GET /api/pleroma/admin/media_proxy_caches" do
     test "shows banned MediaProxy URLs", %{conn: conn} do
+      MediaProxy.put_in_deleted_urls([
+        "http://localhost:4001/media/a688346.jpg",
+        "http://localhost:4001/media/fb1f4d.jpg"
+      ])
+
+      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg")
+      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg")
+      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg")
+
       response =
         conn
-        |> get("/api/pleroma/admin/media_proxy_caches")
+        |> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
         |> json_response_and_validate_schema(200)
 
-      assert response["urls"] == []
+      assert response["urls"] == [
+               "http://localhost:4001/media/fb1f4d.jpg",
+               "http://localhost:4001/media/a688346.jpg"
+             ]
+
+      response =
+        conn
+        |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == [
+               "http://localhost:4001/media/gb1f44.jpg",
+               "http://localhost:4001/media/tb13f47.jpg"
+             ]
+
+      response =
+        conn
+        |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
     end
   end
 
   describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do
     test "deleted MediaProxy URLs from banned", %{conn: conn} do
+      MediaProxy.put_in_deleted_urls([
+        "http://localhost:4001/media/a688346.jpg",
+        "http://localhost:4001/media/fb1f4d.jpg"
+      ])
+
       response =
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
-          urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"]
+          urls: ["http://localhost:4001/media/a688346.jpg"]
         })
         |> json_response_and_validate_schema(200)
 
-      assert response["urls"] == [
-               "http://example.com/media/a688346.jpg",
-               "http://example.com/media/fb1f4d.jpg"
-             ]
+      assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
+      refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg")
+      assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg")
     end
   end
 
   describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do
     test "perform invalidates cache of MediaProxy", %{conn: conn} do
-      response =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
-          urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"]
-        })
-        |> json_response_and_validate_schema(200)
+      urls = [
+        "http://example.com/media/a688346.jpg",
+        "http://example.com/media/fb1f4d.jpg"
+      ]
 
-      assert response["urls"] == [
-               "http://example.com/media/a688346.jpg",
-               "http://example.com/media/fb1f4d.jpg"
-             ]
+      with_mocks [
+        {MediaProxy.Invalidation.Script, [],
+         [
+           purge: fn _, _ -> {"ok", 0} end
+         ]}
+      ] do
+        response =
+          conn
+          |> put_req_header("content-type", "application/json")
+          |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
+          |> json_response_and_validate_schema(200)
+
+        assert response["urls"] == urls
+
+        refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
+        refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+      end
+    end
+
+    test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
+      urls = [
+        "http://example.com/media/a688346.jpg",
+        "http://example.com/media/fb1f4d.jpg"
+      ]
+
+      with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
+        response =
+          conn
+          |> put_req_header("content-type", "application/json")
+          |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
+            urls: urls,
+            ban: true
+          })
+          |> json_response_and_validate_schema(200)
+
+        assert response["urls"] == urls
+
+        assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
+        assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+      end
     end
   end
 end

From bd63089a633099233d4fc19faece2796253a7ee0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 15 Jun 2020 16:20:05 +0400
Subject: [PATCH 212/283] Fix tests

---
 .../rich_media/parsers/twitter_card_test.exs  | 88 +++++++++----------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/test/web/rich_media/parsers/twitter_card_test.exs b/test/web/rich_media/parsers/twitter_card_test.exs
index 3ccf26651..219f005a2 100644
--- a/test/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/web/rich_media/parsers/twitter_card_test.exs
@@ -17,18 +17,18 @@ test "parses twitter card with only name attributes" do
 
     assert TwitterCard.parse(html, %{}) ==
              %{
-               "app:id:googleplay": "com.nytimes.android",
-               "app:name:googleplay": "NYTimes",
-               "app:url:googleplay": "nytimes://reader/id/100000006583622",
-               site: nil,
-               description:
+               "app:id:googleplay" => "com.nytimes.android",
+               "app:name:googleplay" => "NYTimes",
+               "app:url:googleplay" => "nytimes://reader/id/100000006583622",
+               "site" => nil,
+               "description" =>
                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-               image:
+               "image" =>
                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
-               type: "article",
-               url:
+               "type" => "article",
+               "url" =>
                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
-               title:
+               "title" =>
                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database."
              }
   end
@@ -40,17 +40,17 @@ test "parses twitter card with only property attributes" do
 
     assert TwitterCard.parse(html, %{}) ==
              %{
-               card: "summary_large_image",
-               description:
+               "card" => "summary_large_image",
+               "description" =>
                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-               image:
+               "image" =>
                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-               "image:alt": "",
-               title:
+               "image:alt" => "",
+               "title" =>
                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-               url:
+               "url" =>
                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
-               type: "article"
+               "type" => "article"
              }
   end
 
@@ -61,21 +61,21 @@ test "parses twitter card with name & property attributes" do
 
     assert TwitterCard.parse(html, %{}) ==
              %{
-               "app:id:googleplay": "com.nytimes.android",
-               "app:name:googleplay": "NYTimes",
-               "app:url:googleplay": "nytimes://reader/id/100000006583622",
-               card: "summary_large_image",
-               description:
+               "app:id:googleplay" => "com.nytimes.android",
+               "app:name:googleplay" => "NYTimes",
+               "app:url:googleplay" => "nytimes://reader/id/100000006583622",
+               "card" => "summary_large_image",
+               "description" =>
                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-               image:
+               "image" =>
                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
-               "image:alt": "",
-               site: nil,
-               title:
+               "image:alt" => "",
+               "site" => nil,
+               "title" =>
                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-               url:
+               "url" =>
                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
-               type: "article"
+               "type" => "article"
              }
   end
 
@@ -90,15 +90,15 @@ test "respect only first title tag on the page" do
 
     assert TwitterCard.parse(html, %{}) ==
              %{
-               site: "@atlasobscura",
-               title: "The Missing Grave of Margaret Corbin, Revolutionary War Veteran",
-               card: "summary_large_image",
-               image: image_path,
-               description:
+               "site" => "@atlasobscura",
+               "title" => "The Missing Grave of Margaret Corbin, Revolutionary War Veteran",
+               "card" => "summary_large_image",
+               "image" => image_path,
+               "description" =>
                  "She's the only woman veteran honored with a monument at West Point. But where was she buried?",
-               site_name: "Atlas Obscura",
-               type: "article",
-               url: "http://www.atlasobscura.com/articles/margaret-corbin-grave-west-point"
+               "site_name" => "Atlas Obscura",
+               "type" => "article",
+               "url" => "http://www.atlasobscura.com/articles/margaret-corbin-grave-west-point"
              }
   end
 
@@ -109,18 +109,18 @@ test "takes first founded title in html head if there is html markup error" do
 
     assert TwitterCard.parse(html, %{}) ==
              %{
-               site: nil,
-               title:
+               "site" => nil,
+               "title" =>
                  "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
-               "app:id:googleplay": "com.nytimes.android",
-               "app:name:googleplay": "NYTimes",
-               "app:url:googleplay": "nytimes://reader/id/100000006583622",
-               description:
+               "app:id:googleplay" => "com.nytimes.android",
+               "app:name:googleplay" => "NYTimes",
+               "app:url:googleplay" => "nytimes://reader/id/100000006583622",
+               "description" =>
                  "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
-               image:
+               "image" =>
                  "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
-               type: "article",
-               url:
+               "type" => "article",
+               "url" =>
                  "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
              }
   end

From efdfc85c2d8e5118c1aa18e4f04026ec90cd11d2 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 15 Jun 2020 15:24:00 +0300
Subject: [PATCH 213/283] update docs

---
 docs/API/admin_api.md | 64 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 92816baf9..6659b605d 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1224,4 +1224,66 @@ Loads json generated from `config/descriptions.exs`.
 - Response:
   - On success: `204`, empty response
   - On failure:
-    - 400 Bad Request `"Invalid parameters"` when `status` is missing
\ No newline at end of file
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing
+
+## `GET /api/pleroma/admin/media_proxy_caches`
+
+### Get a list of all banned MediaProxy URLs in Cachex
+
+- Authentication: required
+- Params:
+- *optional* `page`: **integer** page number
+- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+
+- Response:
+
+``` json
+{
+  "urls": [
+    "http://example.com/media/a688346.jpg",
+    "http://example.com/media/fb1f4d.jpg"
+  ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/delete`
+
+### Remove a banned MediaProxy URL from Cachex
+
+- Authentication: required
+- Params:
+  - `urls`
+
+- Response:
+
+``` json
+{
+  "urls": [
+    "http://example.com/media/a688346.jpg",
+    "http://example.com/media/fb1f4d.jpg"
+  ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/purge`
+
+### Purge a MediaProxy URL
+
+- Authentication: required
+- Params:
+  - `urls`
+  - `ban`
+
+- Response:
+
+``` json
+{
+  "urls": [
+    "http://example.com/media/a688346.jpg",
+    "http://example.com/media/fb1f4d.jpg"
+  ]
+}
+
+```

From e1ee8bc1da17a356c88b535db7a9228fccc5251f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 15 Jun 2020 14:29:34 +0200
Subject: [PATCH 214/283] User: update_follower_count refactor.

---
 lib/pleroma/user.ex | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 52ac9052b..39a9e13e8 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -747,7 +747,6 @@ def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do
 
         follower
         |> update_following_count()
-        |> set_cache()
     end
   end
 
@@ -776,7 +775,6 @@ defp do_unfollow(%User{} = follower, %User{} = followed) do
         {:ok, follower} =
           follower
           |> update_following_count()
-          |> set_cache()
 
         {:ok, follower, followed}
 
@@ -1128,35 +1126,25 @@ defp follow_information_changeset(user, params) do
     ])
   end
 
+  @spec update_follower_count(User.t()) :: {:ok, User.t()}
   def update_follower_count(%User{} = user) do
     if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do
-      follower_count_query =
-        User.Query.build(%{followers: user, deactivated: false})
-        |> select([u], %{count: count(u.id)})
+      follower_count = FollowingRelationship.follower_count(user)
 
-      User
-      |> where(id: ^user.id)
-      |> join(:inner, [u], s in subquery(follower_count_query))
-      |> update([u, s],
-        set: [follower_count: s.count]
-      )
-      |> select([u], u)
-      |> Repo.update_all([])
-      |> case do
-        {1, [user]} -> set_cache(user)
-        _ -> {:error, user}
-      end
+      user
+      |> follow_information_changeset(%{follower_count: follower_count})
+      |> update_and_set_cache
     else
       {:ok, maybe_fetch_follow_information(user)}
     end
   end
 
-  @spec update_following_count(User.t()) :: User.t()
+  @spec update_following_count(User.t()) :: {:ok, User.t()}
   def update_following_count(%User{local: false} = user) do
     if Pleroma.Config.get([:instance, :external_user_synchronization]) do
-      maybe_fetch_follow_information(user)
+      {:ok, maybe_fetch_follow_information(user)}
     else
-      user
+      {:ok, user}
     end
   end
 
@@ -1165,7 +1153,7 @@ def update_following_count(%User{local: true} = user) do
 
     user
     |> follow_information_changeset(%{following_count: following_count})
-    |> Repo.update!()
+    |> update_and_set_cache()
   end
 
   def set_unread_conversation_count(%User{local: true} = user) do

From b02311079961c5193af1c144516a3caeee72b582 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 15 Jun 2020 20:47:02 +0300
Subject: [PATCH 215/283] fixed a visibility of functions

---
 .../workers/attachments_cleanup_worker.ex     | 31 +++++++++----------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
index 4ad19c0fc..8deeabda0 100644
--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
@@ -18,22 +18,11 @@ def perform(
         },
         _job
       ) do
-    hrefs =
-      Enum.flat_map(attachments, fn attachment ->
-        Enum.map(attachment["url"], & &1["href"])
-      end)
-
-    # find all objects for copies of the attachments, name and actor doesn't matter here
-    hrefs
+    attachments
+    |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
     |> fetch_objects
     |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
-    |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
-      with 1 <- count do
-        {ids ++ [id], hrefs ++ [href]}
-      else
-        _ -> {ids ++ [id], hrefs}
-      end
-    end)
+    |> filter_objects
     |> do_clean
 
     {:ok, :success}
@@ -73,7 +62,17 @@ defp delete_objects(_), do: :ok
 
   # we should delete 1 object for any given attachment, but don't delete
   # files if there are more than 1 object for it
-  def prepare_objects(objects, actor, names) do
+  defp filter_objects(objects) do
+    Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
+      with 1 <- count do
+        {ids ++ [id], hrefs ++ [href]}
+      else
+        _ -> {ids ++ [id], hrefs}
+      end
+    end)
+  end
+
+  defp prepare_objects(objects, actor, names) do
     objects
     |> Enum.reduce(%{}, fn %{
                              id: id,
@@ -98,7 +97,7 @@ def prepare_objects(objects, actor, names) do
     end)
   end
 
-  def fetch_objects(hrefs) do
+  defp fetch_objects(hrefs) do
     from(o in Object,
       where:
         fragment(

From 1eb6cedaadee4e1ab3e0885b4e03a8dd17ba08ea Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 16 Jun 2020 13:08:27 +0200
Subject: [PATCH 216/283] ActivityPub: When restricting to media posts, only
 show 'Creates'.

---
 lib/pleroma/web/activity_pub/activity_pub.ex           |  3 ++-
 .../controllers/account_controller_test.exs            | 10 +++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c9dc6135c..efb8b81db 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -833,7 +833,8 @@ defp restrict_media(_query, %{only_media: _val, skip_preload: true}) do
 
   defp restrict_media(query, %{only_media: true}) do
     from(
-      [_activity, object] in query,
+      [activity, object] in query,
+      where: fragment("(?)->>'type' = ?", activity.data, "Create"),
       where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
     )
   end
diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs
index 1ce97378d..2343a9d2d 100644
--- a/test/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller_test.exs
@@ -350,9 +350,10 @@ test "unimplemented pinned statuses feature", %{conn: conn} do
       assert json_response_and_validate_schema(conn, 200) == []
     end
 
-    test "gets an users media", %{conn: conn} do
+    test "gets an users media, excludes reblogs", %{conn: conn} do
       note = insert(:note_activity)
       user = User.get_cached_by_ap_id(note.data["actor"])
+      other_user = insert(:user)
 
       file = %Plug.Upload{
         content_type: "image/jpg",
@@ -364,6 +365,13 @@ test "gets an users media", %{conn: conn} do
 
       {:ok, %{id: image_post_id}} = CommonAPI.post(user, %{status: "cofe", media_ids: [media_id]})
 
+      {:ok, %{id: media_id}} = ActivityPub.upload(file, actor: other_user.ap_id)
+
+      {:ok, %{id: other_image_post_id}} =
+        CommonAPI.post(other_user, %{status: "cofe2", media_ids: [media_id]})
+
+      {:ok, _announce} = CommonAPI.repeat(other_image_post_id, user)
+
       conn = get(conn, "/api/v1/accounts/#{user.id}/statuses?only_media=true")
 
       assert [%{"id" => ^image_post_id}] = json_response_and_validate_schema(conn, 200)

From 4733f6a3371504ebb3eeb447d7c20d56c10b43bf Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 16 Jun 2020 13:09:28 +0200
Subject: [PATCH 217/283] Changelog: Add info about `only_media` changes.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2629bf84..eee442817 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- Using the `only_media` filter on timelines will now exclude reblog media
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard
 <details>

From 9a4fde97661595630ea840917ef83b4786f2e2d3 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sun, 31 May 2020 10:46:02 +0300
Subject: [PATCH 218/283] Mogrify args as custom tuples

---
 lib/mix/tasks/pleroma/config.ex               |  10 +-
 lib/pleroma/config/config_db.ex               | 274 ++++----
 lib/pleroma/config/transfer_task.ex           |  12 +-
 lib/pleroma/config/type/atom.ex               |  22 +
 lib/pleroma/config/type/binary_value.ex       |  23 +
 .../controllers/config_controller.ex          |  34 +-
 .../web/admin_api/views/config_view.ex        |  21 +-
 test/config/config_db_test.exs                | 587 +++++++-----------
 test/config/transfer_task_test.exs            |  94 +--
 test/support/factory.ex                       |  17 +-
 test/tasks/config_test.exs                    |  41 +-
 test/upload/filter/mogrify_test.exs           |   8 +-
 .../controllers/config_controller_test.exs    | 256 +++++---
 13 files changed, 620 insertions(+), 779 deletions(-)
 create mode 100644 lib/pleroma/config/type/atom.ex
 create mode 100644 lib/pleroma/config/type/binary_value.ex

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 5c9ef6904..f1b3a8766 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -72,8 +72,7 @@ defp create(group, settings) do
     group
     |> Pleroma.Config.Loader.filter_group(settings)
     |> Enum.each(fn {key, value} ->
-      key = inspect(key)
-      {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value})
+      {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
 
       shell_info("Settings for key #{key} migrated.")
     end)
@@ -131,12 +130,9 @@ defp write_and_delete(config, file, delete?) do
   end
 
   defp write(config, file) do
-    value =
-      config.value
-      |> ConfigDB.from_binary()
-      |> inspect(limit: :infinity)
+    value = inspect(config.value, limit: :infinity)
 
-    IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n")
+    IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n")
 
     config
   end
diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 2b43d4c36..39b37c42e 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do
   use Ecto.Schema
 
   import Ecto.Changeset
-  import Ecto.Query
+  import Ecto.Query, only: [select: 3]
   import Pleroma.Web.Gettext
 
   alias __MODULE__
@@ -14,16 +14,6 @@ defmodule Pleroma.ConfigDB do
 
   @type t :: %__MODULE__{}
 
-  @full_key_update [
-    {:pleroma, :ecto_repos},
-    {:quack, :meta},
-    {:mime, :types},
-    {:cors_plug, [:max_age, :methods, :expose, :headers]},
-    {:auto_linker, :opts},
-    {:swarm, :node_blacklist},
-    {:logger, :backends}
-  ]
-
   @full_subkey_update [
     {:pleroma, :assets, :mascots},
     {:pleroma, :emoji, :groups},
@@ -32,14 +22,10 @@ defmodule Pleroma.ConfigDB do
     {:pleroma, :mrf_keyword, :replace}
   ]
 
-  @regex ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
-
-  @delimiters ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
-
   schema "config" do
-    field(:key, :string)
-    field(:group, :string)
-    field(:value, :binary)
+    field(:key, Pleroma.Config.Type.Atom)
+    field(:group, Pleroma.Config.Type.Atom)
+    field(:value, Pleroma.Config.Type.BinaryValue)
     field(:db, {:array, :string}, virtual: true, default: [])
 
     timestamps()
@@ -51,10 +37,6 @@ def get_all_as_keyword do
     |> select([c], {c.group, c.key, c.value})
     |> Repo.all()
     |> Enum.reduce([], fn {group, key, value}, acc ->
-      group = ConfigDB.from_string(group)
-      key = ConfigDB.from_string(key)
-      value = from_binary(value)
-
       Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
     end)
   end
@@ -64,50 +46,41 @@ def get_by_params(params), do: Repo.get_by(ConfigDB, params)
 
   @spec changeset(ConfigDB.t(), map()) :: Changeset.t()
   def changeset(config, params \\ %{}) do
-    params = Map.put(params, :value, transform(params[:value]))
-
     config
     |> cast(params, [:key, :group, :value])
     |> validate_required([:key, :group, :value])
     |> unique_constraint(:key, name: :config_group_key_index)
   end
 
-  @spec create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
-  def create(params) do
+  defp create(params) do
     %ConfigDB{}
     |> changeset(params)
     |> Repo.insert()
   end
 
-  @spec update(ConfigDB.t(), map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
-  def update(%ConfigDB{} = config, %{value: value}) do
+  defp update(%ConfigDB{} = config, %{value: value}) do
     config
     |> changeset(%{value: value})
     |> Repo.update()
   end
 
-  @spec get_db_keys(ConfigDB.t()) :: [String.t()]
-  def get_db_keys(%ConfigDB{} = config) do
-    config.value
-    |> ConfigDB.from_binary()
-    |> get_db_keys(config.key)
-  end
-
   @spec get_db_keys(keyword(), any()) :: [String.t()]
   def get_db_keys(value, key) do
-    if Keyword.keyword?(value) do
-      value |> Keyword.keys() |> Enum.map(&convert(&1))
-    else
-      [convert(key)]
-    end
+    keys =
+      if Keyword.keyword?(value) do
+        Keyword.keys(value)
+      else
+        [key]
+      end
+
+    Enum.map(keys, &to_json_types(&1))
   end
 
   @spec merge_group(atom(), atom(), keyword(), keyword()) :: keyword()
   def merge_group(group, key, old_value, new_value) do
-    new_keys = to_map_set(new_value)
+    new_keys = to_mapset(new_value)
 
-    intersect_keys =
-      old_value |> to_map_set() |> MapSet.intersection(new_keys) |> MapSet.to_list()
+    intersect_keys = old_value |> to_mapset() |> MapSet.intersection(new_keys) |> MapSet.to_list()
 
     merged_value = ConfigDB.merge(old_value, new_value)
 
@@ -120,12 +93,10 @@ def merge_group(group, key, old_value, new_value) do
         []
     end)
     |> List.flatten()
-    |> Enum.reduce(merged_value, fn subkey, acc ->
-      Keyword.put(acc, subkey, new_value[subkey])
-    end)
+    |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1]))
   end
 
-  defp to_map_set(keyword) do
+  defp to_mapset(keyword) do
     keyword
     |> Keyword.keys()
     |> MapSet.new()
@@ -159,43 +130,39 @@ defp deep_merge(_key, value1, value2) do
 
   @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
   def update_or_create(params) do
+    params = Map.put(params, :value, to_elixir_types(params[:value]))
     search_opts = Map.take(params, [:group, :key])
 
     with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
-         {:partial_update, true, config} <-
-           {:partial_update, can_be_partially_updated?(config), config},
-         old_value <- from_binary(config.value),
-         transformed_value <- do_transform(params[:value]),
-         {:can_be_merged, true, config} <- {:can_be_merged, is_list(transformed_value), config},
-         new_value <-
-           merge_group(
-             ConfigDB.from_string(config.group),
-             ConfigDB.from_string(config.key),
-             old_value,
-             transformed_value
-           ) do
-      ConfigDB.update(config, %{value: new_value})
+         {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config},
+         {_, true, config} <- {:can_be_merged, is_list(params[:value]), config} do
+      new_value = merge_group(config.group, config.key, config.value, params[:value])
+      update(config, %{value: new_value})
     else
       {reason, false, config} when reason in [:partial_update, :can_be_merged] ->
-        ConfigDB.update(config, params)
+        update(config, params)
 
       nil ->
-        ConfigDB.create(params)
+        create(params)
     end
   end
 
   defp can_be_partially_updated?(%ConfigDB{} = config), do: not only_full_update?(config)
 
-  defp only_full_update?(%ConfigDB{} = config) do
-    config_group = ConfigDB.from_string(config.group)
-    config_key = ConfigDB.from_string(config.key)
+  defp only_full_update?(%ConfigDB{group: group, key: key}) do
+    full_key_update = [
+      {:pleroma, :ecto_repos},
+      {:quack, :meta},
+      {:mime, :types},
+      {:cors_plug, [:max_age, :methods, :expose, :headers]},
+      {:auto_linker, :opts},
+      {:swarm, :node_blacklist},
+      {:logger, :backends}
+    ]
 
-    Enum.any?(@full_key_update, fn
-      {group, key} when is_list(key) ->
-        config_group == group and config_key in key
-
-      {group, key} ->
-        config_group == group and config_key == key
+    Enum.any?(full_key_update, fn
+      {s_group, s_key} ->
+        group == s_group and ((is_list(s_key) and key in s_key) or key == s_key)
     end)
   end
 
@@ -205,11 +172,10 @@ def delete(params) do
 
     with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
          {config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]},
-         old_value <- from_binary(config.value),
-         keys <- Enum.map(sub_keys, &do_transform_string(&1)),
-         {:partial_remove, config, new_value} when new_value != [] <-
-           {:partial_remove, config, Keyword.drop(old_value, keys)} do
-      ConfigDB.update(config, %{value: new_value})
+         keys <- Enum.map(sub_keys, &string_to_elixir_types(&1)),
+         {_, config, new_value} when new_value != [] <-
+           {:partial_remove, config, Keyword.drop(config.value, keys)} do
+      update(config, %{value: new_value})
     else
       {:partial_remove, config, []} ->
         Repo.delete(config)
@@ -225,37 +191,32 @@ def delete(params) do
     end
   end
 
-  @spec from_binary(binary()) :: term()
-  def from_binary(binary), do: :erlang.binary_to_term(binary)
-
-  @spec from_binary_with_convert(binary()) :: any()
-  def from_binary_with_convert(binary) do
-    binary
-    |> from_binary()
-    |> do_convert()
+  @spec to_json_types(term()) :: map() | list() | boolean() | String.t()
+  def to_json_types(entity) when is_list(entity) do
+    Enum.map(entity, &to_json_types/1)
   end
 
-  @spec from_string(String.t()) :: atom() | no_return()
-  def from_string(string), do: do_transform_string(string)
+  def to_json_types(%Regex{} = entity), do: inspect(entity)
 
-  @spec convert(any()) :: any()
-  def convert(entity), do: do_convert(entity)
-
-  defp do_convert(entity) when is_list(entity) do
-    for v <- entity, into: [], do: do_convert(v)
+  def to_json_types(entity) when is_map(entity) do
+    Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end)
   end
 
-  defp do_convert(%Regex{} = entity), do: inspect(entity)
+  def to_json_types({:args, args}) when is_list(args) do
+    arguments =
+      Enum.map(args, fn
+        arg when is_tuple(arg) -> inspect(arg)
+        arg -> to_json_types(arg)
+      end)
 
-  defp do_convert(entity) when is_map(entity) do
-    for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)}
+    %{"tuple" => [":args", arguments]}
   end
 
-  defp do_convert({:proxy_url, {type, :localhost, port}}) do
-    %{"tuple" => [":proxy_url", %{"tuple" => [do_convert(type), "localhost", port]}]}
+  def to_json_types({:proxy_url, {type, :localhost, port}}) do
+    %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]}
   end
 
-  defp do_convert({:proxy_url, {type, host, port}}) when is_tuple(host) do
+  def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do
     ip =
       host
       |> :inet_parse.ntoa()
@@ -264,66 +225,64 @@ defp do_convert({:proxy_url, {type, host, port}}) when is_tuple(host) do
     %{
       "tuple" => [
         ":proxy_url",
-        %{"tuple" => [do_convert(type), ip, port]}
+        %{"tuple" => [to_json_types(type), ip, port]}
       ]
     }
   end
 
-  defp do_convert({:proxy_url, {type, host, port}}) do
+  def to_json_types({:proxy_url, {type, host, port}}) do
     %{
       "tuple" => [
         ":proxy_url",
-        %{"tuple" => [do_convert(type), to_string(host), port]}
+        %{"tuple" => [to_json_types(type), to_string(host), port]}
       ]
     }
   end
 
-  defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
+  def to_json_types({:partial_chain, entity}),
+    do: %{"tuple" => [":partial_chain", inspect(entity)]}
 
-  defp do_convert(entity) when is_tuple(entity) do
+  def to_json_types(entity) when is_tuple(entity) do
     value =
       entity
       |> Tuple.to_list()
-      |> do_convert()
+      |> to_json_types()
 
     %{"tuple" => value}
   end
 
-  defp do_convert(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
+  def to_json_types(entity) when is_binary(entity), do: entity
+
+  def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do
     entity
   end
 
-  defp do_convert(entity)
-       when is_atom(entity) and entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
+  def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do
     ":#{entity}"
   end
 
-  defp do_convert(entity) when is_atom(entity), do: inspect(entity)
+  def to_json_types(entity) when is_atom(entity), do: inspect(entity)
 
-  defp do_convert(entity) when is_binary(entity), do: entity
+  @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term()
+  def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do
+    arguments =
+      Enum.map(args, fn arg ->
+        if String.contains?(arg, ["{", "}"]) do
+          {elem, []} = Code.eval_string(arg)
+          elem
+        else
+          to_elixir_types(arg)
+        end
+      end)
 
-  @spec transform(any()) :: binary() | no_return()
-  def transform(entity) when is_binary(entity) or is_map(entity) or is_list(entity) do
-    entity
-    |> do_transform()
-    |> to_binary()
+    {:args, arguments}
   end
 
-  def transform(entity), do: to_binary(entity)
-
-  @spec transform_with_out_binary(any()) :: any()
-  def transform_with_out_binary(entity), do: do_transform(entity)
-
-  @spec to_binary(any()) :: binary()
-  def to_binary(entity), do: :erlang.term_to_binary(entity)
-
-  defp do_transform(%Regex{} = entity), do: entity
-
-  defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
-    {:proxy_url, {do_transform_string(type), parse_host(host), port}}
+  def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do
+    {:proxy_url, {string_to_elixir_types(type), parse_host(host), port}}
   end
 
-  defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
+  def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do
     {partial_chain, []} =
       entity
       |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
@@ -332,25 +291,51 @@ defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
     {:partial_chain, partial_chain}
   end
 
-  defp do_transform(%{"tuple" => entity}) do
-    Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
+  def to_elixir_types(%{"tuple" => entity}) do
+    Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1)))
   end
 
-  defp do_transform(entity) when is_map(entity) do
-    for {k, v} <- entity, into: %{}, do: {do_transform(k), do_transform(v)}
+  def to_elixir_types(entity) when is_map(entity) do
+    Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end)
   end
 
-  defp do_transform(entity) when is_list(entity) do
-    for v <- entity, into: [], do: do_transform(v)
+  def to_elixir_types(entity) when is_list(entity) do
+    Enum.map(entity, &to_elixir_types/1)
   end
 
-  defp do_transform(entity) when is_binary(entity) do
+  def to_elixir_types(entity) when is_binary(entity) do
     entity
     |> String.trim()
-    |> do_transform_string()
+    |> string_to_elixir_types()
   end
 
-  defp do_transform(entity), do: entity
+  def to_elixir_types(entity), do: entity
+
+  @spec string_to_elixir_types(String.t()) ::
+          atom() | Regex.t() | module() | String.t() | no_return()
+  def string_to_elixir_types("~r" <> _pattern = regex) do
+    pattern =
+      ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u
+
+    delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}]
+
+    with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
+           Regex.named_captures(pattern, regex),
+         {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter),
+         {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
+      result
+    end
+  end
+
+  def string_to_elixir_types(":" <> atom), do: String.to_atom(atom)
+
+  def string_to_elixir_types(value) do
+    if is_module_name?(value) do
+      String.to_existing_atom("Elixir." <> value)
+    else
+      value
+    end
+  end
 
   defp parse_host("localhost"), do: :localhost
 
@@ -387,25 +372,6 @@ defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
     end
   end
 
-  defp do_transform_string("~r" <> _pattern = regex) do
-    with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <-
-           Regex.named_captures(@regex, regex),
-         {:ok, {leading, closing}} <- find_valid_delimiter(@delimiters, pattern, regex_delimiter),
-         {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do
-      result
-    end
-  end
-
-  defp do_transform_string(":" <> atom), do: String.to_atom(atom)
-
-  defp do_transform_string(value) do
-    if is_module_name?(value) do
-      String.to_existing_atom("Elixir." <> value)
-    else
-      value
-    end
-  end
-
   @spec is_module_name?(String.t()) :: boolean()
   def is_module_name?(string) do
     Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex
index c02b70e96..eb86b8ff4 100644
--- a/lib/pleroma/config/transfer_task.ex
+++ b/lib/pleroma/config/transfer_task.ex
@@ -28,10 +28,6 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, Pleroma.Captcha, [:seconds_valid]},
     {:pleroma, Pleroma.Upload, [:proxy_remote]},
     {:pleroma, :instance, [:upload_limit]},
-    {:pleroma, :email_notifications, [:digest]},
-    {:pleroma, :oauth2, [:clean_expired_tokens]},
-    {:pleroma, Pleroma.ActivityExpiration, [:enabled]},
-    {:pleroma, Pleroma.ScheduledActivity, [:enabled]},
     {:pleroma, :gopher, [:enabled]}
   ]
 
@@ -48,7 +44,7 @@ def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
 
       {logger, other} =
         (Repo.all(ConfigDB) ++ deleted_settings)
-        |> Enum.map(&transform_and_merge/1)
+        |> Enum.map(&merge_with_default/1)
         |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
 
       logger
@@ -92,11 +88,7 @@ defp maybe_set_pleroma_last(apps) do
     end
   end
 
-  defp transform_and_merge(%{group: group, key: key, value: value} = setting) do
-    group = ConfigDB.from_string(group)
-    key = ConfigDB.from_string(key)
-    value = ConfigDB.from_binary(value)
-
+  defp merge_with_default(%{group: group, key: key, value: value} = setting) do
     default = Config.Holder.default_config(group, key)
 
     merged =
diff --git a/lib/pleroma/config/type/atom.ex b/lib/pleroma/config/type/atom.ex
new file mode 100644
index 000000000..387869284
--- /dev/null
+++ b/lib/pleroma/config/type/atom.ex
@@ -0,0 +1,22 @@
+defmodule Pleroma.Config.Type.Atom do
+  use Ecto.Type
+
+  def type, do: :atom
+
+  def cast(key) when is_atom(key) do
+    {:ok, key}
+  end
+
+  def cast(key) when is_binary(key) do
+    {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+  end
+
+  def cast(_), do: :error
+
+  def load(key) do
+    {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+  end
+
+  def dump(key) when is_atom(key), do: {:ok, inspect(key)}
+  def dump(_), do: :error
+end
diff --git a/lib/pleroma/config/type/binary_value.ex b/lib/pleroma/config/type/binary_value.ex
new file mode 100644
index 000000000..17c5524a3
--- /dev/null
+++ b/lib/pleroma/config/type/binary_value.ex
@@ -0,0 +1,23 @@
+defmodule Pleroma.Config.Type.BinaryValue do
+  use Ecto.Type
+
+  def type, do: :term
+
+  def cast(value) when is_binary(value) do
+    if String.valid?(value) do
+      {:ok, value}
+    else
+      {:ok, :erlang.binary_to_term(value)}
+    end
+  end
+
+  def cast(value), do: {:ok, value}
+
+  def load(value) when is_binary(value) do
+    {:ok, :erlang.binary_to_term(value)}
+  end
+
+  def dump(value) do
+    {:ok, :erlang.term_to_binary(value)}
+  end
+end
diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex
index d6e2019bc..7f60470cb 100644
--- a/lib/pleroma/web/admin_api/controllers/config_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex
@@ -33,7 +33,11 @@ def descriptions(conn, _params) do
   def show(conn, %{only_db: true}) do
     with :ok <- configurable_from_database() do
       configs = Pleroma.Repo.all(ConfigDB)
-      render(conn, "index.json", %{configs: configs})
+
+      render(conn, "index.json", %{
+        configs: configs,
+        need_reboot: Restarter.Pleroma.need_reboot?()
+      })
     end
   end
 
@@ -61,17 +65,20 @@ def show(conn, _params) do
                 value
               end
 
-            %{
-              group: ConfigDB.convert(group),
-              key: ConfigDB.convert(key),
-              value: ConfigDB.convert(merged_value)
+            %ConfigDB{
+              group: group,
+              key: key,
+              value: merged_value
             }
             |> Pleroma.Maps.put_if_present(:db, db)
           end)
         end)
         |> List.flatten()
 
-      json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()})
+      render(conn, "index.json", %{
+        configs: merged,
+        need_reboot: Restarter.Pleroma.need_reboot?()
+      })
     end
   end
 
@@ -91,24 +98,17 @@ def update(%{body_params: %{configs: configs}} = conn, _) do
 
       {deleted, updated} =
         results
-        |> Enum.map(fn {:ok, config} ->
-          Map.put(config, :db, ConfigDB.get_db_keys(config))
-        end)
-        |> Enum.split_with(fn config ->
-          Ecto.get_meta(config, :state) == :deleted
+        |> Enum.map(fn {:ok, %{key: key, value: value} = config} ->
+          Map.put(config, :db, ConfigDB.get_db_keys(value, key))
         end)
+        |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted))
 
       Config.TransferTask.load_and_update_env(deleted, false)
 
       if not Restarter.Pleroma.need_reboot?() do
         changed_reboot_settings? =
           (updated ++ deleted)
-          |> Enum.any?(fn config ->
-            group = ConfigDB.from_string(config.group)
-            key = ConfigDB.from_string(config.key)
-            value = ConfigDB.from_binary(config.value)
-            Config.TransferTask.pleroma_need_restart?(group, key, value)
-          end)
+          |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value))
 
         if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
       end
diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex
index 587ef760e..d2d8b5907 100644
--- a/lib/pleroma/web/admin_api/views/config_view.ex
+++ b/lib/pleroma/web/admin_api/views/config_view.ex
@@ -5,23 +5,20 @@
 defmodule Pleroma.Web.AdminAPI.ConfigView do
   use Pleroma.Web, :view
 
-  def render("index.json", %{configs: configs} = params) do
-    map = %{
-      configs: render_many(configs, __MODULE__, "show.json", as: :config)
-    }
+  alias Pleroma.ConfigDB
 
-    if params[:need_reboot] do
-      Map.put(map, :need_reboot, true)
-    else
-      map
-    end
+  def render("index.json", %{configs: configs} = params) do
+    %{
+      configs: render_many(configs, __MODULE__, "show.json", as: :config),
+      need_reboot: params[:need_reboot]
+    }
   end
 
   def render("show.json", %{config: config}) do
     map = %{
-      key: config.key,
-      group: config.group,
-      value: Pleroma.ConfigDB.from_binary_with_convert(config.value)
+      key: ConfigDB.to_json_types(config.key),
+      group: ConfigDB.to_json_types(config.group),
+      value: ConfigDB.to_json_types(config.value)
     }
 
     if config.db != [] do
diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs
index 336de7359..a04575c6f 100644
--- a/test/config/config_db_test.exs
+++ b/test/config/config_db_test.exs
@@ -7,40 +7,28 @@ defmodule Pleroma.ConfigDBTest do
   import Pleroma.Factory
   alias Pleroma.ConfigDB
 
-  test "get_by_key/1" do
+  test "get_by_params/1" do
     config = insert(:config)
     insert(:config)
 
     assert config == ConfigDB.get_by_params(%{group: config.group, key: config.key})
   end
 
-  test "create/1" do
-    {:ok, config} = ConfigDB.create(%{group: ":pleroma", key: ":some_key", value: "some_value"})
-    assert config == ConfigDB.get_by_params(%{group: ":pleroma", key: ":some_key"})
-  end
-
-  test "update/1" do
-    config = insert(:config)
-    {:ok, updated} = ConfigDB.update(config, %{value: "some_value"})
-    loaded = ConfigDB.get_by_params(%{group: config.group, key: config.key})
-    assert loaded == updated
-  end
-
   test "get_all_as_keyword/0" do
     saved = insert(:config)
-    insert(:config, group: ":quack", key: ":level", value: ConfigDB.to_binary(:info))
-    insert(:config, group: ":quack", key: ":meta", value: ConfigDB.to_binary([:none]))
+    insert(:config, group: ":quack", key: ":level", value: :info)
+    insert(:config, group: ":quack", key: ":meta", value: [:none])
 
     insert(:config,
       group: ":quack",
       key: ":webhook_url",
-      value: ConfigDB.to_binary("https://hooks.slack.com/services/KEY/some_val")
+      value: "https://hooks.slack.com/services/KEY/some_val"
     )
 
     config = ConfigDB.get_all_as_keyword()
 
     assert config[:pleroma] == [
-             {ConfigDB.from_string(saved.key), ConfigDB.from_binary(saved.value)}
+             {saved.key, saved.value}
            ]
 
     assert config[:quack][:level] == :info
@@ -51,11 +39,11 @@ test "get_all_as_keyword/0" do
   describe "update_or_create/1" do
     test "common" do
       config = insert(:config)
-      key2 = "another_key"
+      key2 = :another_key
 
       params = [
-        %{group: "pleroma", key: key2, value: "another_value"},
-        %{group: config.group, key: config.key, value: "new_value"}
+        %{group: :pleroma, key: key2, value: "another_value"},
+        %{group: :pleroma, key: config.key, value: "new_value"}
       ]
 
       assert Repo.all(ConfigDB) |> length() == 1
@@ -65,16 +53,16 @@ test "common" do
       assert Repo.all(ConfigDB) |> length() == 2
 
       config1 = ConfigDB.get_by_params(%{group: config.group, key: config.key})
-      config2 = ConfigDB.get_by_params(%{group: "pleroma", key: key2})
+      config2 = ConfigDB.get_by_params(%{group: :pleroma, key: key2})
 
-      assert config1.value == ConfigDB.transform("new_value")
-      assert config2.value == ConfigDB.transform("another_value")
+      assert config1.value == "new_value"
+      assert config2.value == "another_value"
     end
 
     test "partial update" do
-      config = insert(:config, value: ConfigDB.to_binary(key1: "val1", key2: :val2))
+      config = insert(:config, value: [key1: "val1", key2: :val2])
 
-      {:ok, _config} =
+      {:ok, config} =
         ConfigDB.update_or_create(%{
           group: config.group,
           key: config.key,
@@ -83,15 +71,14 @@ test "partial update" do
 
       updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
 
-      value = ConfigDB.from_binary(updated.value)
-      assert length(value) == 3
-      assert value[:key1] == :val1
-      assert value[:key2] == :val2
-      assert value[:key3] == :val3
+      assert config.value == updated.value
+      assert updated.value[:key1] == :val1
+      assert updated.value[:key2] == :val2
+      assert updated.value[:key3] == :val3
     end
 
     test "deep merge" do
-      config = insert(:config, value: ConfigDB.to_binary(key1: "val1", key2: [k1: :v1, k2: "v2"]))
+      config = insert(:config, value: [key1: "val1", key2: [k1: :v1, k2: "v2"]])
 
       {:ok, config} =
         ConfigDB.update_or_create(%{
@@ -103,18 +90,15 @@ test "deep merge" do
       updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
 
       assert config.value == updated.value
-
-      value = ConfigDB.from_binary(updated.value)
-      assert value[:key1] == :val1
-      assert value[:key2] == [k1: :v1, k2: :v2, k3: :v3]
-      assert value[:key3] == :val3
+      assert updated.value[:key1] == :val1
+      assert updated.value[:key2] == [k1: :v1, k2: :v2, k3: :v3]
+      assert updated.value[:key3] == :val3
     end
 
     test "only full update for some keys" do
-      config1 = insert(:config, key: ":ecto_repos", value: ConfigDB.to_binary(repo: Pleroma.Repo))
+      config1 = insert(:config, key: :ecto_repos, value: [repo: Pleroma.Repo])
 
-      config2 =
-        insert(:config, group: ":cors_plug", key: ":max_age", value: ConfigDB.to_binary(18))
+      config2 = insert(:config, group: :cors_plug, key: :max_age, value: 18)
 
       {:ok, _config} =
         ConfigDB.update_or_create(%{
@@ -133,8 +117,8 @@ test "only full update for some keys" do
       updated1 = ConfigDB.get_by_params(%{group: config1.group, key: config1.key})
       updated2 = ConfigDB.get_by_params(%{group: config2.group, key: config2.key})
 
-      assert ConfigDB.from_binary(updated1.value) == [another_repo: [Pleroma.Repo]]
-      assert ConfigDB.from_binary(updated2.value) == 777
+      assert updated1.value == [another_repo: [Pleroma.Repo]]
+      assert updated2.value == 777
     end
 
     test "full update if value is not keyword" do
@@ -142,7 +126,7 @@ test "full update if value is not keyword" do
         insert(:config,
           group: ":tesla",
           key: ":adapter",
-          value: ConfigDB.to_binary(Tesla.Adapter.Hackney)
+          value: Tesla.Adapter.Hackney
         )
 
       {:ok, _config} =
@@ -154,20 +138,20 @@ test "full update if value is not keyword" do
 
       updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
 
-      assert ConfigDB.from_binary(updated.value) == Tesla.Adapter.Httpc
+      assert updated.value == Tesla.Adapter.Httpc
     end
 
     test "only full update for some subkeys" do
       config1 =
         insert(:config,
           key: ":emoji",
-          value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1])
+          value: [groups: [a: 1, b: 2], key: [a: 1]]
         )
 
       config2 =
         insert(:config,
           key: ":assets",
-          value: ConfigDB.to_binary(mascots: [a: 1, b: 2], key: [a: 1])
+          value: [mascots: [a: 1, b: 2], key: [a: 1]]
         )
 
       {:ok, _config} =
@@ -187,8 +171,8 @@ test "only full update for some subkeys" do
       updated1 = ConfigDB.get_by_params(%{group: config1.group, key: config1.key})
       updated2 = ConfigDB.get_by_params(%{group: config2.group, key: config2.key})
 
-      assert ConfigDB.from_binary(updated1.value) == [groups: [c: 3, d: 4], key: [a: 1, b: 2]]
-      assert ConfigDB.from_binary(updated2.value) == [mascots: [c: 3, d: 4], key: [a: 1, b: 2]]
+      assert updated1.value == [groups: [c: 3, d: 4], key: [a: 1, b: 2]]
+      assert updated2.value == [mascots: [c: 3, d: 4], key: [a: 1, b: 2]]
     end
   end
 
@@ -206,14 +190,14 @@ test "full delete" do
     end
 
     test "partial subkeys delete" do
-      config = insert(:config, value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1]))
+      config = insert(:config, value: [groups: [a: 1, b: 2], key: [a: 1]])
 
       {:ok, deleted} =
         ConfigDB.delete(%{group: config.group, key: config.key, subkeys: [":groups"]})
 
       assert Ecto.get_meta(deleted, :state) == :loaded
 
-      assert deleted.value == ConfigDB.to_binary(key: [a: 1])
+      assert deleted.value == [key: [a: 1]]
 
       updated = ConfigDB.get_by_params(%{group: config.group, key: config.key})
 
@@ -221,7 +205,7 @@ test "partial subkeys delete" do
     end
 
     test "full delete if remaining value after subkeys deletion is empty list" do
-      config = insert(:config, value: ConfigDB.to_binary(groups: [a: 1, b: 2]))
+      config = insert(:config, value: [groups: [a: 1, b: 2]])
 
       {:ok, deleted} =
         ConfigDB.delete(%{group: config.group, key: config.key, subkeys: [":groups"]})
@@ -232,234 +216,159 @@ test "full delete if remaining value after subkeys deletion is empty list" do
     end
   end
 
-  describe "transform/1" do
+  describe "to_elixir_types/1" do
     test "string" do
-      binary = ConfigDB.transform("value as string")
-      assert binary == :erlang.term_to_binary("value as string")
-      assert ConfigDB.from_binary(binary) == "value as string"
+      assert ConfigDB.to_elixir_types("value as string") == "value as string"
     end
 
     test "boolean" do
-      binary = ConfigDB.transform(false)
-      assert binary == :erlang.term_to_binary(false)
-      assert ConfigDB.from_binary(binary) == false
+      assert ConfigDB.to_elixir_types(false) == false
     end
 
     test "nil" do
-      binary = ConfigDB.transform(nil)
-      assert binary == :erlang.term_to_binary(nil)
-      assert ConfigDB.from_binary(binary) == nil
+      assert ConfigDB.to_elixir_types(nil) == nil
     end
 
     test "integer" do
-      binary = ConfigDB.transform(150)
-      assert binary == :erlang.term_to_binary(150)
-      assert ConfigDB.from_binary(binary) == 150
+      assert ConfigDB.to_elixir_types(150) == 150
     end
 
     test "atom" do
-      binary = ConfigDB.transform(":atom")
-      assert binary == :erlang.term_to_binary(:atom)
-      assert ConfigDB.from_binary(binary) == :atom
+      assert ConfigDB.to_elixir_types(":atom") == :atom
     end
 
     test "ssl options" do
-      binary = ConfigDB.transform([":tlsv1", ":tlsv1.1", ":tlsv1.2"])
-      assert binary == :erlang.term_to_binary([:tlsv1, :"tlsv1.1", :"tlsv1.2"])
-      assert ConfigDB.from_binary(binary) == [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
+      assert ConfigDB.to_elixir_types([":tlsv1", ":tlsv1.1", ":tlsv1.2"]) == [
+               :tlsv1,
+               :"tlsv1.1",
+               :"tlsv1.2"
+             ]
     end
 
     test "pleroma module" do
-      binary = ConfigDB.transform("Pleroma.Bookmark")
-      assert binary == :erlang.term_to_binary(Pleroma.Bookmark)
-      assert ConfigDB.from_binary(binary) == Pleroma.Bookmark
+      assert ConfigDB.to_elixir_types("Pleroma.Bookmark") == Pleroma.Bookmark
     end
 
     test "pleroma string" do
-      binary = ConfigDB.transform("Pleroma")
-      assert binary == :erlang.term_to_binary("Pleroma")
-      assert ConfigDB.from_binary(binary) == "Pleroma"
+      assert ConfigDB.to_elixir_types("Pleroma") == "Pleroma"
     end
 
     test "phoenix module" do
-      binary = ConfigDB.transform("Phoenix.Socket.V1.JSONSerializer")
-      assert binary == :erlang.term_to_binary(Phoenix.Socket.V1.JSONSerializer)
-      assert ConfigDB.from_binary(binary) == Phoenix.Socket.V1.JSONSerializer
+      assert ConfigDB.to_elixir_types("Phoenix.Socket.V1.JSONSerializer") ==
+               Phoenix.Socket.V1.JSONSerializer
     end
 
     test "tesla module" do
-      binary = ConfigDB.transform("Tesla.Adapter.Hackney")
-      assert binary == :erlang.term_to_binary(Tesla.Adapter.Hackney)
-      assert ConfigDB.from_binary(binary) == Tesla.Adapter.Hackney
+      assert ConfigDB.to_elixir_types("Tesla.Adapter.Hackney") == Tesla.Adapter.Hackney
     end
 
     test "ExSyslogger module" do
-      binary = ConfigDB.transform("ExSyslogger")
-      assert binary == :erlang.term_to_binary(ExSyslogger)
-      assert ConfigDB.from_binary(binary) == ExSyslogger
+      assert ConfigDB.to_elixir_types("ExSyslogger") == ExSyslogger
     end
 
     test "Quack.Logger module" do
-      binary = ConfigDB.transform("Quack.Logger")
-      assert binary == :erlang.term_to_binary(Quack.Logger)
-      assert ConfigDB.from_binary(binary) == Quack.Logger
+      assert ConfigDB.to_elixir_types("Quack.Logger") == Quack.Logger
     end
 
     test "Swoosh.Adapters modules" do
-      binary = ConfigDB.transform("Swoosh.Adapters.SMTP")
-      assert binary == :erlang.term_to_binary(Swoosh.Adapters.SMTP)
-      assert ConfigDB.from_binary(binary) == Swoosh.Adapters.SMTP
-      binary = ConfigDB.transform("Swoosh.Adapters.AmazonSES")
-      assert binary == :erlang.term_to_binary(Swoosh.Adapters.AmazonSES)
-      assert ConfigDB.from_binary(binary) == Swoosh.Adapters.AmazonSES
+      assert ConfigDB.to_elixir_types("Swoosh.Adapters.SMTP") == Swoosh.Adapters.SMTP
+      assert ConfigDB.to_elixir_types("Swoosh.Adapters.AmazonSES") == Swoosh.Adapters.AmazonSES
     end
 
     test "sigil" do
-      binary = ConfigDB.transform("~r[comp[lL][aA][iI][nN]er]")
-      assert binary == :erlang.term_to_binary(~r/comp[lL][aA][iI][nN]er/)
-      assert ConfigDB.from_binary(binary) == ~r/comp[lL][aA][iI][nN]er/
+      assert ConfigDB.to_elixir_types("~r[comp[lL][aA][iI][nN]er]") == ~r/comp[lL][aA][iI][nN]er/
     end
 
     test "link sigil" do
-      binary = ConfigDB.transform("~r/https:\/\/example.com/")
-      assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/)
-      assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/
+      assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/") == ~r/https:\/\/example.com/
     end
 
     test "link sigil with um modifiers" do
-      binary = ConfigDB.transform("~r/https:\/\/example.com/um")
-      assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/um)
-      assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/um
+      assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/um") ==
+               ~r/https:\/\/example.com/um
     end
 
     test "link sigil with i modifier" do
-      binary = ConfigDB.transform("~r/https:\/\/example.com/i")
-      assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/i)
-      assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/i
+      assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/i") == ~r/https:\/\/example.com/i
     end
 
     test "link sigil with s modifier" do
-      binary = ConfigDB.transform("~r/https:\/\/example.com/s")
-      assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/s)
-      assert ConfigDB.from_binary(binary) == ~r/https:\/\/example.com/s
+      assert ConfigDB.to_elixir_types("~r/https:\/\/example.com/s") == ~r/https:\/\/example.com/s
     end
 
     test "raise if valid delimiter not found" do
       assert_raise ArgumentError, "valid delimiter for Regex expression not found", fn ->
-        ConfigDB.transform("~r/https://[]{}<>\"'()|example.com/s")
+        ConfigDB.to_elixir_types("~r/https://[]{}<>\"'()|example.com/s")
       end
     end
 
     test "2 child tuple" do
-      binary = ConfigDB.transform(%{"tuple" => ["v1", ":v2"]})
-      assert binary == :erlang.term_to_binary({"v1", :v2})
-      assert ConfigDB.from_binary(binary) == {"v1", :v2}
+      assert ConfigDB.to_elixir_types(%{"tuple" => ["v1", ":v2"]}) == {"v1", :v2}
     end
 
     test "proxy tuple with localhost" do
-      binary =
-        ConfigDB.transform(%{
-          "tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]
-        })
-
-      assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, :localhost, 1234}})
-      assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, :localhost, 1234}}
+      assert ConfigDB.to_elixir_types(%{
+               "tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]
+             }) == {:proxy_url, {:socks5, :localhost, 1234}}
     end
 
     test "proxy tuple with domain" do
-      binary =
-        ConfigDB.transform(%{
-          "tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]
-        })
-
-      assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, 'domain.com', 1234}})
-      assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, 'domain.com', 1234}}
+      assert ConfigDB.to_elixir_types(%{
+               "tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]
+             }) == {:proxy_url, {:socks5, 'domain.com', 1234}}
     end
 
     test "proxy tuple with ip" do
-      binary =
-        ConfigDB.transform(%{
-          "tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]
-        })
-
-      assert binary == :erlang.term_to_binary({:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}})
-      assert ConfigDB.from_binary(binary) == {:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}}
+      assert ConfigDB.to_elixir_types(%{
+               "tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]
+             }) == {:proxy_url, {:socks5, {127, 0, 0, 1}, 1234}}
     end
 
     test "tuple with n childs" do
-      binary =
-        ConfigDB.transform(%{
-          "tuple" => [
-            "v1",
-            ":v2",
-            "Pleroma.Bookmark",
-            150,
-            false,
-            "Phoenix.Socket.V1.JSONSerializer"
-          ]
-        })
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
-               )
-
-      assert ConfigDB.from_binary(binary) ==
-               {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
+      assert ConfigDB.to_elixir_types(%{
+               "tuple" => [
+                 "v1",
+                 ":v2",
+                 "Pleroma.Bookmark",
+                 150,
+                 false,
+                 "Phoenix.Socket.V1.JSONSerializer"
+               ]
+             }) == {"v1", :v2, Pleroma.Bookmark, 150, false, Phoenix.Socket.V1.JSONSerializer}
     end
 
     test "map with string key" do
-      binary = ConfigDB.transform(%{"key" => "value"})
-      assert binary == :erlang.term_to_binary(%{"key" => "value"})
-      assert ConfigDB.from_binary(binary) == %{"key" => "value"}
+      assert ConfigDB.to_elixir_types(%{"key" => "value"}) == %{"key" => "value"}
     end
 
     test "map with atom key" do
-      binary = ConfigDB.transform(%{":key" => "value"})
-      assert binary == :erlang.term_to_binary(%{key: "value"})
-      assert ConfigDB.from_binary(binary) == %{key: "value"}
+      assert ConfigDB.to_elixir_types(%{":key" => "value"}) == %{key: "value"}
     end
 
     test "list of strings" do
-      binary = ConfigDB.transform(["v1", "v2", "v3"])
-      assert binary == :erlang.term_to_binary(["v1", "v2", "v3"])
-      assert ConfigDB.from_binary(binary) == ["v1", "v2", "v3"]
+      assert ConfigDB.to_elixir_types(["v1", "v2", "v3"]) == ["v1", "v2", "v3"]
     end
 
     test "list of modules" do
-      binary = ConfigDB.transform(["Pleroma.Repo", "Pleroma.Activity"])
-      assert binary == :erlang.term_to_binary([Pleroma.Repo, Pleroma.Activity])
-      assert ConfigDB.from_binary(binary) == [Pleroma.Repo, Pleroma.Activity]
+      assert ConfigDB.to_elixir_types(["Pleroma.Repo", "Pleroma.Activity"]) == [
+               Pleroma.Repo,
+               Pleroma.Activity
+             ]
     end
 
     test "list of atoms" do
-      binary = ConfigDB.transform([":v1", ":v2", ":v3"])
-      assert binary == :erlang.term_to_binary([:v1, :v2, :v3])
-      assert ConfigDB.from_binary(binary) == [:v1, :v2, :v3]
+      assert ConfigDB.to_elixir_types([":v1", ":v2", ":v3"]) == [:v1, :v2, :v3]
     end
 
     test "list of mixed values" do
-      binary =
-        ConfigDB.transform([
-          "v1",
-          ":v2",
-          "Pleroma.Repo",
-          "Phoenix.Socket.V1.JSONSerializer",
-          15,
-          false
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary([
-                 "v1",
-                 :v2,
-                 Pleroma.Repo,
-                 Phoenix.Socket.V1.JSONSerializer,
-                 15,
-                 false
-               ])
-
-      assert ConfigDB.from_binary(binary) == [
+      assert ConfigDB.to_elixir_types([
+               "v1",
+               ":v2",
+               "Pleroma.Repo",
+               "Phoenix.Socket.V1.JSONSerializer",
+               15,
+               false
+             ]) == [
                "v1",
                :v2,
                Pleroma.Repo,
@@ -470,40 +379,23 @@ test "list of mixed values" do
     end
 
     test "simple keyword" do
-      binary = ConfigDB.transform([%{"tuple" => [":key", "value"]}])
-      assert binary == :erlang.term_to_binary([{:key, "value"}])
-      assert ConfigDB.from_binary(binary) == [{:key, "value"}]
-      assert ConfigDB.from_binary(binary) == [key: "value"]
+      assert ConfigDB.to_elixir_types([%{"tuple" => [":key", "value"]}]) == [key: "value"]
     end
 
     test "keyword with partial_chain key" do
-      binary =
-        ConfigDB.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}])
-
-      assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1)
-      assert ConfigDB.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1]
+      assert ConfigDB.to_elixir_types([
+               %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}
+             ]) == [partial_chain: &:hackney_connect.partial_chain/1]
     end
 
     test "keyword" do
-      binary =
-        ConfigDB.transform([
-          %{"tuple" => [":types", "Pleroma.PostgresTypes"]},
-          %{"tuple" => [":telemetry_event", ["Pleroma.Repo.Instrumenter"]]},
-          %{"tuple" => [":migration_lock", nil]},
-          %{"tuple" => [":key1", 150]},
-          %{"tuple" => [":key2", "string"]}
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 types: Pleroma.PostgresTypes,
-                 telemetry_event: [Pleroma.Repo.Instrumenter],
-                 migration_lock: nil,
-                 key1: 150,
-                 key2: "string"
-               )
-
-      assert ConfigDB.from_binary(binary) == [
+      assert ConfigDB.to_elixir_types([
+               %{"tuple" => [":types", "Pleroma.PostgresTypes"]},
+               %{"tuple" => [":telemetry_event", ["Pleroma.Repo.Instrumenter"]]},
+               %{"tuple" => [":migration_lock", nil]},
+               %{"tuple" => [":key1", 150]},
+               %{"tuple" => [":key2", "string"]}
+             ]) == [
                types: Pleroma.PostgresTypes,
                telemetry_event: [Pleroma.Repo.Instrumenter],
                migration_lock: nil,
@@ -513,85 +405,55 @@ test "keyword" do
     end
 
     test "complex keyword with nested mixed childs" do
-      binary =
-        ConfigDB.transform([
-          %{"tuple" => [":uploader", "Pleroma.Uploaders.Local"]},
-          %{"tuple" => [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
-          %{"tuple" => [":link_name", true]},
-          %{"tuple" => [":proxy_remote", false]},
-          %{"tuple" => [":common_map", %{":key" => "value"}]},
-          %{
-            "tuple" => [
-              ":proxy_opts",
-              [
-                %{"tuple" => [":redirect_on_failure", false]},
-                %{"tuple" => [":max_body_length", 1_048_576]},
-                %{
-                  "tuple" => [
-                    ":http",
-                    [%{"tuple" => [":follow_redirect", true]}, %{"tuple" => [":pool", ":upload"]}]
-                  ]
-                }
-              ]
-            ]
-          }
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 uploader: Pleroma.Uploaders.Local,
-                 filters: [Pleroma.Upload.Filter.Dedupe],
-                 link_name: true,
-                 proxy_remote: false,
-                 common_map: %{key: "value"},
-                 proxy_opts: [
-                   redirect_on_failure: false,
-                   max_body_length: 1_048_576,
-                   http: [
-                     follow_redirect: true,
-                     pool: :upload
+      assert ConfigDB.to_elixir_types([
+               %{"tuple" => [":uploader", "Pleroma.Uploaders.Local"]},
+               %{"tuple" => [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
+               %{"tuple" => [":link_name", true]},
+               %{"tuple" => [":proxy_remote", false]},
+               %{"tuple" => [":common_map", %{":key" => "value"}]},
+               %{
+                 "tuple" => [
+                   ":proxy_opts",
+                   [
+                     %{"tuple" => [":redirect_on_failure", false]},
+                     %{"tuple" => [":max_body_length", 1_048_576]},
+                     %{
+                       "tuple" => [
+                         ":http",
+                         [
+                           %{"tuple" => [":follow_redirect", true]},
+                           %{"tuple" => [":pool", ":upload"]}
+                         ]
+                       ]
+                     }
                    ]
                  ]
-               )
-
-      assert ConfigDB.from_binary(binary) ==
-               [
-                 uploader: Pleroma.Uploaders.Local,
-                 filters: [Pleroma.Upload.Filter.Dedupe],
-                 link_name: true,
-                 proxy_remote: false,
-                 common_map: %{key: "value"},
-                 proxy_opts: [
-                   redirect_on_failure: false,
-                   max_body_length: 1_048_576,
-                   http: [
-                     follow_redirect: true,
-                     pool: :upload
-                   ]
+               }
+             ]) == [
+               uploader: Pleroma.Uploaders.Local,
+               filters: [Pleroma.Upload.Filter.Dedupe],
+               link_name: true,
+               proxy_remote: false,
+               common_map: %{key: "value"},
+               proxy_opts: [
+                 redirect_on_failure: false,
+                 max_body_length: 1_048_576,
+                 http: [
+                   follow_redirect: true,
+                   pool: :upload
                  ]
                ]
+             ]
     end
 
     test "common keyword" do
-      binary =
-        ConfigDB.transform([
-          %{"tuple" => [":level", ":warn"]},
-          %{"tuple" => [":meta", [":all"]]},
-          %{"tuple" => [":path", ""]},
-          %{"tuple" => [":val", nil]},
-          %{"tuple" => [":webhook_url", "https://hooks.slack.com/services/YOUR-KEY-HERE"]}
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 level: :warn,
-                 meta: [:all],
-                 path: "",
-                 val: nil,
-                 webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
-               )
-
-      assert ConfigDB.from_binary(binary) == [
+      assert ConfigDB.to_elixir_types([
+               %{"tuple" => [":level", ":warn"]},
+               %{"tuple" => [":meta", [":all"]]},
+               %{"tuple" => [":path", ""]},
+               %{"tuple" => [":val", nil]},
+               %{"tuple" => [":webhook_url", "https://hooks.slack.com/services/YOUR-KEY-HERE"]}
+             ]) == [
                level: :warn,
                meta: [:all],
                path: "",
@@ -601,98 +463,73 @@ test "common keyword" do
     end
 
     test "complex keyword with sigil" do
-      binary =
-        ConfigDB.transform([
-          %{"tuple" => [":federated_timeline_removal", []]},
-          %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]},
-          %{"tuple" => [":replace", []]}
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 federated_timeline_removal: [],
-                 reject: [~r/comp[lL][aA][iI][nN]er/],
-                 replace: []
-               )
-
-      assert ConfigDB.from_binary(binary) ==
-               [federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []]
+      assert ConfigDB.to_elixir_types([
+               %{"tuple" => [":federated_timeline_removal", []]},
+               %{"tuple" => [":reject", ["~r/comp[lL][aA][iI][nN]er/"]]},
+               %{"tuple" => [":replace", []]}
+             ]) == [
+               federated_timeline_removal: [],
+               reject: [~r/comp[lL][aA][iI][nN]er/],
+               replace: []
+             ]
     end
 
     test "complex keyword with tuples with more than 2 values" do
-      binary =
-        ConfigDB.transform([
-          %{
-            "tuple" => [
-              ":http",
-              [
-                %{
-                  "tuple" => [
-                    ":key1",
-                    [
-                      %{
-                        "tuple" => [
-                          ":_",
-                          [
-                            %{
-                              "tuple" => [
-                                "/api/v1/streaming",
-                                "Pleroma.Web.MastodonAPI.WebsocketHandler",
-                                []
-                              ]
-                            },
-                            %{
-                              "tuple" => [
-                                "/websocket",
-                                "Phoenix.Endpoint.CowboyWebSocket",
-                                %{
-                                  "tuple" => [
-                                    "Phoenix.Transports.WebSocket",
-                                    %{
-                                      "tuple" => [
-                                        "Pleroma.Web.Endpoint",
-                                        "Pleroma.Web.UserSocket",
-                                        []
-                                      ]
-                                    }
-                                  ]
-                                }
-                              ]
-                            },
-                            %{
-                              "tuple" => [
-                                ":_",
-                                "Phoenix.Endpoint.Cowboy2Handler",
-                                %{"tuple" => ["Pleroma.Web.Endpoint", []]}
-                              ]
-                            }
-                          ]
-                        ]
-                      }
-                    ]
-                  ]
-                }
-              ]
-            ]
-          }
-        ])
-
-      assert binary ==
-               :erlang.term_to_binary(
-                 http: [
-                   key1: [
-                     _: [
-                       {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
-                       {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
-                        {Phoenix.Transports.WebSocket,
-                         {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, []}}},
-                       {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
-                     ]
+      assert ConfigDB.to_elixir_types([
+               %{
+                 "tuple" => [
+                   ":http",
+                   [
+                     %{
+                       "tuple" => [
+                         ":key1",
+                         [
+                           %{
+                             "tuple" => [
+                               ":_",
+                               [
+                                 %{
+                                   "tuple" => [
+                                     "/api/v1/streaming",
+                                     "Pleroma.Web.MastodonAPI.WebsocketHandler",
+                                     []
+                                   ]
+                                 },
+                                 %{
+                                   "tuple" => [
+                                     "/websocket",
+                                     "Phoenix.Endpoint.CowboyWebSocket",
+                                     %{
+                                       "tuple" => [
+                                         "Phoenix.Transports.WebSocket",
+                                         %{
+                                           "tuple" => [
+                                             "Pleroma.Web.Endpoint",
+                                             "Pleroma.Web.UserSocket",
+                                             []
+                                           ]
+                                         }
+                                       ]
+                                     }
+                                   ]
+                                 },
+                                 %{
+                                   "tuple" => [
+                                     ":_",
+                                     "Phoenix.Endpoint.Cowboy2Handler",
+                                     %{"tuple" => ["Pleroma.Web.Endpoint", []]}
+                                   ]
+                                 }
+                               ]
+                             ]
+                           }
+                         ]
+                       ]
+                     }
                    ]
                  ]
-               )
-
-      assert ConfigDB.from_binary(binary) == [
+               }
+             ]) == [
                http: [
                  key1: [
                    {:_,
diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs
index 473899d1d..f53829e09 100644
--- a/test/config/transfer_task_test.exs
+++ b/test/config/transfer_task_test.exs
@@ -6,9 +6,9 @@ defmodule Pleroma.Config.TransferTaskTest do
   use Pleroma.DataCase
 
   import ExUnit.CaptureLog
+  import Pleroma.Factory
 
   alias Pleroma.Config.TransferTask
-  alias Pleroma.ConfigDB
 
   setup do: clear_config(:configurable_from_database, true)
 
@@ -19,31 +19,11 @@ test "transfer config values from db to env" do
     refute Application.get_env(:postgrex, :test_key)
     initial = Application.get_env(:logger, :level)
 
-    ConfigDB.create(%{
-      group: ":pleroma",
-      key: ":test_key",
-      value: [live: 2, com: 3]
-    })
-
-    ConfigDB.create(%{
-      group: ":idna",
-      key: ":test_key",
-      value: [live: 15, com: 35]
-    })
-
-    ConfigDB.create(%{
-      group: ":quack",
-      key: ":test_key",
-      value: [:test_value1, :test_value2]
-    })
-
-    ConfigDB.create(%{
-      group: ":postgrex",
-      key: ":test_key",
-      value: :value
-    })
-
-    ConfigDB.create(%{group: ":logger", key: ":level", value: :debug})
+    insert(:config, key: :test_key, value: [live: 2, com: 3])
+    insert(:config, group: :idna, key: :test_key, value: [live: 15, com: 35])
+    insert(:config, group: :quack, key: :test_key, value: [:test_value1, :test_value2])
+    insert(:config, group: :postgrex, key: :test_key, value: :value)
+    insert(:config, group: :logger, key: :level, value: :debug)
 
     TransferTask.start_link([])
 
@@ -66,17 +46,8 @@ test "transfer config values for 1 group and some keys" do
     level = Application.get_env(:quack, :level)
     meta = Application.get_env(:quack, :meta)
 
-    ConfigDB.create(%{
-      group: ":quack",
-      key: ":level",
-      value: :info
-    })
-
-    ConfigDB.create(%{
-      group: ":quack",
-      key: ":meta",
-      value: [:none]
-    })
+    insert(:config, group: :quack, key: :level, value: :info)
+    insert(:config, group: :quack, key: :meta, value: [:none])
 
     TransferTask.start_link([])
 
@@ -95,17 +66,8 @@ test "transfer config values with full subkey update" do
     clear_config(:emoji)
     clear_config(:assets)
 
-    ConfigDB.create(%{
-      group: ":pleroma",
-      key: ":emoji",
-      value: [groups: [a: 1, b: 2]]
-    })
-
-    ConfigDB.create(%{
-      group: ":pleroma",
-      key: ":assets",
-      value: [mascots: [a: 1, b: 2]]
-    })
+    insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
+    insert(:config, key: :assets, value: [mascots: [a: 1, b: 2]])
 
     TransferTask.start_link([])
 
@@ -122,12 +84,7 @@ test "transfer config values with full subkey update" do
 
     test "don't restart if no reboot time settings were changed" do
       clear_config(:emoji)
-
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":emoji",
-        value: [groups: [a: 1, b: 2]]
-      })
+      insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]])
 
       refute String.contains?(
                capture_log(fn -> TransferTask.start_link([]) end),
@@ -137,25 +94,13 @@ test "don't restart if no reboot time settings were changed" do
 
     test "on reboot time key" do
       clear_config(:chat)
-
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":chat",
-        value: [enabled: false]
-      })
-
+      insert(:config, key: :chat, value: [enabled: false])
       assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
     end
 
     test "on reboot time subkey" do
       clear_config(Pleroma.Captcha)
-
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: "Pleroma.Captcha",
-        value: [seconds_valid: 60]
-      })
-
+      insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
       assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
     end
 
@@ -163,17 +108,8 @@ test "don't restart pleroma on reboot time key and subkey if there is false flag
       clear_config(:chat)
       clear_config(Pleroma.Captcha)
 
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":chat",
-        value: [enabled: false]
-      })
-
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: "Pleroma.Captcha",
-        value: [seconds_valid: 60]
-      })
+      insert(:config, key: :chat, value: [enabled: false])
+      insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60])
 
       refute String.contains?(
                capture_log(fn -> TransferTask.load_and_update_env([], false) end),
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 6e3676aca..e517d5bc6 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -396,24 +396,17 @@ def registration_factory do
     }
   end
 
-  def config_factory do
+  def config_factory(attrs \\ %{}) do
     %Pleroma.ConfigDB{
-      key:
-        sequence(:key, fn key ->
-          # Atom dynamic registration hack in tests
-          "some_key_#{key}"
-          |> String.to_atom()
-          |> inspect()
-        end),
-      group: ":pleroma",
+      key: sequence(:key, &String.to_atom("some_key_#{&1}")),
+      group: :pleroma,
       value:
         sequence(
           :value,
-          fn key ->
-            :erlang.term_to_binary(%{another_key: "#{key}somevalue", another: "#{key}somevalue"})
-          end
+          &%{another_key: "#{&1}somevalue", another: "#{&1}somevalue"}
         )
     }
+    |> merge_attributes(attrs)
   end
 
   def marker_factory do
diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs
index 04bc947a9..e1bddfebf 100644
--- a/test/tasks/config_test.exs
+++ b/test/tasks/config_test.exs
@@ -5,6 +5,8 @@
 defmodule Mix.Tasks.Pleroma.ConfigTest do
   use Pleroma.DataCase
 
+  import Pleroma.Factory
+
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
 
@@ -49,24 +51,19 @@ test "filtered settings are migrated to db" do
       refute ConfigDB.get_by_params(%{group: ":pleroma", key: "Pleroma.Repo"})
       refute ConfigDB.get_by_params(%{group: ":postgrex", key: ":json_library"})
 
-      assert ConfigDB.from_binary(config1.value) == [key: "value", key2: [Repo]]
-      assert ConfigDB.from_binary(config2.value) == [key: "value2", key2: ["Activity"]]
-      assert ConfigDB.from_binary(config3.value) == :info
+      assert config1.value == [key: "value", key2: [Repo]]
+      assert config2.value == [key: "value2", key2: ["Activity"]]
+      assert config3.value == :info
     end
 
     test "config table is truncated before migration" do
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":first_setting",
-        value: [key: "value", key2: ["Activity"]]
-      })
-
+      insert(:config, key: :first_setting, value: [key: "value", key2: ["Activity"]])
       assert Repo.aggregate(ConfigDB, :count, :id) == 1
 
       Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
 
       config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"})
-      assert ConfigDB.from_binary(config.value) == [key: "value", key2: [Repo]]
+      assert config.value == [key: "value", key2: [Repo]]
     end
   end
 
@@ -82,19 +79,9 @@ test "config table is truncated before migration" do
     end
 
     test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":setting_first",
-        value: [key: "value", key2: ["Activity"]]
-      })
-
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":setting_second",
-        value: [key: "value2", key2: [Repo]]
-      })
-
-      ConfigDB.create(%{group: ":quack", key: ":level", value: :info})
+      insert(:config, key: :setting_first, value: [key: "value", key2: ["Activity"]])
+      insert(:config, key: :setting_second, value: [key: "value2", key2: [Repo]])
+      insert(:config, group: :quack, key: :level, value: :info)
 
       Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "--env", "temp", "-d"])
 
@@ -107,9 +94,8 @@ test "settings are migrated to file and deleted from db", %{temp_file: temp_file
     end
 
     test "load a settings with large values and pass to file", %{temp_file: temp_file} do
-      ConfigDB.create(%{
-        group: ":pleroma",
-        key: ":instance",
+      insert(:config,
+        key: :instance,
         value: [
           name: "Pleroma",
           email: "example@example.com",
@@ -163,7 +149,6 @@ test "load a settings with large values and pass to file", %{temp_file: temp_fil
           extended_nickname_format: true,
           multi_factor_authentication: [
             totp: [
-              # digits 6 or 8
               digits: 6,
               period: 30
             ],
@@ -173,7 +158,7 @@ test "load a settings with large values and pass to file", %{temp_file: temp_fil
             ]
           ]
         ]
-      })
+      )
 
       Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "--env", "temp", "-d"])
 
diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs
index b6a463e8c..62ca30487 100644
--- a/test/upload/filter/mogrify_test.exs
+++ b/test/upload/filter/mogrify_test.exs
@@ -6,21 +6,17 @@ defmodule Pleroma.Upload.Filter.MogrifyTest do
   use Pleroma.DataCase
   import Mock
 
-  alias Pleroma.Config
-  alias Pleroma.Upload
   alias Pleroma.Upload.Filter
 
-  setup do: clear_config([Filter.Mogrify, :args])
-
   test "apply mogrify filter" do
-    Config.put([Filter.Mogrify, :args], [{"tint", "40"}])
+    clear_config(Filter.Mogrify, args: [{"tint", "40"}])
 
     File.cp!(
       "test/fixtures/image.jpg",
       "test/fixtures/image_tmp.jpg"
     )
 
-    upload = %Upload{
+    upload = %Pleroma.Upload{
       name: "an… image.jpg",
       content_type: "image/jpg",
       path: Path.absname("test/fixtures/image_tmp.jpg"),
diff --git a/test/web/admin_api/controllers/config_controller_test.exs b/test/web/admin_api/controllers/config_controller_test.exs
index 780de8d18..064ef9bc7 100644
--- a/test/web/admin_api/controllers/config_controller_test.exs
+++ b/test/web/admin_api/controllers/config_controller_test.exs
@@ -57,12 +57,12 @@ test "with settings only in db", %{conn: conn} do
         ]
       } = json_response_and_validate_schema(conn, 200)
 
-      assert key1 == config1.key
-      assert key2 == config2.key
+      assert key1 == inspect(config1.key)
+      assert key2 == inspect(config2.key)
     end
 
     test "db is added to settings that are in db", %{conn: conn} do
-      _config = insert(:config, key: ":instance", value: ConfigDB.to_binary(name: "Some name"))
+      _config = insert(:config, key: ":instance", value: [name: "Some name"])
 
       %{"configs" => configs} =
         conn
@@ -83,7 +83,7 @@ test "merged default setting with db settings", %{conn: conn} do
 
       config3 =
         insert(:config,
-          value: ConfigDB.to_binary(k1: :v1, k2: :v2)
+          value: [k1: :v1, k2: :v2]
         )
 
       %{"configs" => configs} =
@@ -93,42 +93,45 @@ test "merged default setting with db settings", %{conn: conn} do
 
       assert length(configs) > 3
 
+      saved_configs = [config1, config2, config3]
+      keys = Enum.map(saved_configs, &inspect(&1.key))
+
       received_configs =
         Enum.filter(configs, fn %{"group" => group, "key" => key} ->
-          group == ":pleroma" and key in [config1.key, config2.key, config3.key]
+          group == ":pleroma" and key in keys
         end)
 
       assert length(received_configs) == 3
 
       db_keys =
         config3.value
-        |> ConfigDB.from_binary()
         |> Keyword.keys()
-        |> ConfigDB.convert()
+        |> ConfigDB.to_json_types()
+
+      keys = Enum.map(saved_configs -- [config3], &inspect(&1.key))
+
+      values = Enum.map(saved_configs, &ConfigDB.to_json_types(&1.value))
+
+      mapset_keys = MapSet.new(keys ++ db_keys)
 
       Enum.each(received_configs, fn %{"value" => value, "db" => db} ->
-        assert db in [[config1.key], [config2.key], db_keys]
+        db = MapSet.new(db)
+        assert MapSet.subset?(db, mapset_keys)
 
-        assert value in [
-                 ConfigDB.from_binary_with_convert(config1.value),
-                 ConfigDB.from_binary_with_convert(config2.value),
-                 ConfigDB.from_binary_with_convert(config3.value)
-               ]
+        assert value in values
       end)
     end
 
     test "subkeys with full update right merge", %{conn: conn} do
-      config1 =
-        insert(:config,
-          key: ":emoji",
-          value: ConfigDB.to_binary(groups: [a: 1, b: 2], key: [a: 1])
-        )
+      insert(:config,
+        key: ":emoji",
+        value: [groups: [a: 1, b: 2], key: [a: 1]]
+      )
 
-      config2 =
-        insert(:config,
-          key: ":assets",
-          value: ConfigDB.to_binary(mascots: [a: 1, b: 2], key: [a: 1])
-        )
+      insert(:config,
+        key: ":assets",
+        value: [mascots: [a: 1, b: 2], key: [a: 1]]
+      )
 
       %{"configs" => configs} =
         conn
@@ -137,14 +140,14 @@ test "subkeys with full update right merge", %{conn: conn} do
 
       vals =
         Enum.filter(configs, fn %{"group" => group, "key" => key} ->
-          group == ":pleroma" and key in [config1.key, config2.key]
+          group == ":pleroma" and key in [":emoji", ":assets"]
         end)
 
       emoji = Enum.find(vals, fn %{"key" => key} -> key == ":emoji" end)
       assets = Enum.find(vals, fn %{"key" => key} -> key == ":assets" end)
 
-      emoji_val = ConfigDB.transform_with_out_binary(emoji["value"])
-      assets_val = ConfigDB.transform_with_out_binary(assets["value"])
+      emoji_val = ConfigDB.to_elixir_types(emoji["value"])
+      assets_val = ConfigDB.to_elixir_types(assets["value"])
 
       assert emoji_val[:groups] == [a: 1, b: 2]
       assert assets_val[:mascots] == [a: 1, b: 2]
@@ -277,7 +280,8 @@ test "create new config setting in db", %{conn: conn} do
                    "value" => %{"tuple" => ["string", "Pleroma.Captcha.NotReal", []]},
                    "db" => [":key5"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:pleroma, :key1) == "value1"
@@ -357,7 +361,8 @@ test "save configs setting without explicit key", %{conn: conn} do
                    "value" => "https://hooks.slack.com/services/KEY",
                    "db" => [":webhook_url"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:quack, :level) == :info
@@ -366,14 +371,14 @@ test "save configs setting without explicit key", %{conn: conn} do
     end
 
     test "saving config with partial update", %{conn: conn} do
-      config = insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: 2))
+      insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: 2))
 
       conn =
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/config", %{
           configs: [
-            %{group: config.group, key: config.key, value: [%{"tuple" => [":key3", 3]}]}
+            %{group: ":pleroma", key: ":key1", value: [%{"tuple" => [":key3", 3]}]}
           ]
         })
 
@@ -389,7 +394,8 @@ test "saving config with partial update", %{conn: conn} do
                    ],
                    "db" => [":key1", ":key2", ":key3"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -500,8 +506,7 @@ test "update setting which need reboot, don't change reboot flag until reboot",
     end
 
     test "saving config with nested merge", %{conn: conn} do
-      config =
-        insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2]))
+      insert(:config, key: :key1, value: [key1: 1, key2: [k1: 1, k2: 2]])
 
       conn =
         conn
@@ -509,8 +514,8 @@ test "saving config with nested merge", %{conn: conn} do
         |> post("/api/pleroma/admin/config", %{
           configs: [
             %{
-              group: config.group,
-              key: config.key,
+              group: ":pleroma",
+              key: ":key1",
               value: [
                 %{"tuple" => [":key3", 3]},
                 %{
@@ -548,7 +553,8 @@ test "saving config with nested merge", %{conn: conn} do
                    ],
                    "db" => [":key1", ":key3", ":key2"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -588,7 +594,8 @@ test "saving special atoms", %{conn: conn} do
                    ],
                    "db" => [":ssl_options"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:pleroma, :key1) == [
@@ -600,12 +607,11 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do
       backends = Application.get_env(:logger, :backends)
       on_exit(fn -> Application.put_env(:logger, :backends, backends) end)
 
-      config =
-        insert(:config,
-          group: ":logger",
-          key: ":backends",
-          value: :erlang.term_to_binary([])
-        )
+      insert(:config,
+        group: :logger,
+        key: :backends,
+        value: []
+      )
 
       Pleroma.Config.TransferTask.load_and_update_env([], false)
 
@@ -617,8 +623,8 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do
         |> post("/api/pleroma/admin/config", %{
           configs: [
             %{
-              group: config.group,
-              key: config.key,
+              group: ":logger",
+              key: ":backends",
               value: [":console"]
             }
           ]
@@ -634,7 +640,8 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do
                    ],
                    "db" => [":backends"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:logger, :backends) == [
@@ -643,19 +650,18 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do
     end
 
     test "saving full setting if value is not keyword", %{conn: conn} do
-      config =
-        insert(:config,
-          group: ":tesla",
-          key: ":adapter",
-          value: :erlang.term_to_binary(Tesla.Adapter.Hackey)
-        )
+      insert(:config,
+        group: :tesla,
+        key: :adapter,
+        value: Tesla.Adapter.Hackey
+      )
 
       conn =
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/config", %{
           configs: [
-            %{group: config.group, key: config.key, value: "Tesla.Adapter.Httpc"}
+            %{group: ":tesla", key: ":adapter", value: "Tesla.Adapter.Httpc"}
           ]
         })
 
@@ -667,7 +673,8 @@ test "saving full setting if value is not keyword", %{conn: conn} do
                    "value" => "Tesla.Adapter.Httpc",
                    "db" => [":adapter"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -677,13 +684,13 @@ test "update config setting & delete with fallback to default value", %{
       token: token
     } do
       ueberauth = Application.get_env(:ueberauth, Ueberauth)
-      config1 = insert(:config, key: ":keyaa1")
-      config2 = insert(:config, key: ":keyaa2")
+      insert(:config, key: :keyaa1)
+      insert(:config, key: :keyaa2)
 
       config3 =
         insert(:config,
-          group: ":ueberauth",
-          key: "Ueberauth"
+          group: :ueberauth,
+          key: Ueberauth
         )
 
       conn =
@@ -691,8 +698,8 @@ test "update config setting & delete with fallback to default value", %{
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/config", %{
           configs: [
-            %{group: config1.group, key: config1.key, value: "another_value"},
-            %{group: config2.group, key: config2.key, value: "another_value"}
+            %{group: ":pleroma", key: ":keyaa1", value: "another_value"},
+            %{group: ":pleroma", key: ":keyaa2", value: "another_value"}
           ]
         })
 
@@ -700,22 +707,23 @@ test "update config setting & delete with fallback to default value", %{
                "configs" => [
                  %{
                    "group" => ":pleroma",
-                   "key" => config1.key,
+                   "key" => ":keyaa1",
                    "value" => "another_value",
                    "db" => [":keyaa1"]
                  },
                  %{
                    "group" => ":pleroma",
-                   "key" => config2.key,
+                   "key" => ":keyaa2",
                    "value" => "another_value",
                    "db" => [":keyaa2"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:pleroma, :keyaa1) == "another_value"
       assert Application.get_env(:pleroma, :keyaa2) == "another_value"
-      assert Application.get_env(:ueberauth, Ueberauth) == ConfigDB.from_binary(config3.value)
+      assert Application.get_env(:ueberauth, Ueberauth) == config3.value
 
       conn =
         build_conn()
@@ -724,7 +732,7 @@ test "update config setting & delete with fallback to default value", %{
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/config", %{
           configs: [
-            %{group: config2.group, key: config2.key, delete: true},
+            %{group: ":pleroma", key: ":keyaa2", delete: true},
             %{
               group: ":ueberauth",
               key: "Ueberauth",
@@ -734,7 +742,8 @@ test "update config setting & delete with fallback to default value", %{
         })
 
       assert json_response_and_validate_schema(conn, 200) == %{
-               "configs" => []
+               "configs" => [],
+               "need_reboot" => false
              }
 
       assert Application.get_env(:ueberauth, Ueberauth) == ueberauth
@@ -801,7 +810,8 @@ test "common config example", %{conn: conn} do
                      ":name"
                    ]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -935,7 +945,8 @@ test "tuples with more than two values", %{conn: conn} do
                    ],
                    "db" => [":http"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -1000,7 +1011,8 @@ test "settings with nesting map", %{conn: conn} do
                      ],
                      "db" => [":key2", ":key3"]
                    }
-                 ]
+                 ],
+                 "need_reboot" => false
                }
     end
 
@@ -1027,7 +1039,8 @@ test "value as map", %{conn: conn} do
                      "value" => %{"key" => "some_val"},
                      "db" => [":key1"]
                    }
-                 ]
+                 ],
+                 "need_reboot" => false
                }
     end
 
@@ -1077,16 +1090,16 @@ test "queues key as atom", %{conn: conn} do
                      ":background"
                    ]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
     test "delete part of settings by atom subkeys", %{conn: conn} do
-      config =
-        insert(:config,
-          key: ":keyaa1",
-          value: :erlang.term_to_binary(subkey1: "val1", subkey2: "val2", subkey3: "val3")
-        )
+      insert(:config,
+        key: :keyaa1,
+        value: [subkey1: "val1", subkey2: "val2", subkey3: "val3"]
+      )
 
       conn =
         conn
@@ -1094,8 +1107,8 @@ test "delete part of settings by atom subkeys", %{conn: conn} do
         |> post("/api/pleroma/admin/config", %{
           configs: [
             %{
-              group: config.group,
-              key: config.key,
+              group: ":pleroma",
+              key: ":keyaa1",
               subkeys: [":subkey1", ":subkey3"],
               delete: true
             }
@@ -1110,7 +1123,8 @@ test "delete part of settings by atom subkeys", %{conn: conn} do
                    "value" => [%{"tuple" => [":subkey2", "val2"]}],
                    "db" => [":subkey2"]
                  }
-               ]
+               ],
+               "need_reboot" => false
              }
     end
 
@@ -1236,6 +1250,90 @@ test "doesn't set keys not in the whitelist", %{conn: conn} do
       assert Application.get_env(:pleroma, Pleroma.Captcha.NotReal) == "value5"
       assert Application.get_env(:not_real, :anything) == "value6"
     end
+
+    test "args for Pleroma.Upload.Filter.Mogrify with custom tuples", %{conn: conn} do
+      clear_config(Pleroma.Upload.Filter.Mogrify)
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/pleroma/admin/config", %{
+               configs: [
+                 %{
+                   group: ":pleroma",
+                   key: "Pleroma.Upload.Filter.Mogrify",
+                   value: [
+                     %{"tuple" => [":args", ["auto-orient", "strip"]]}
+                   ]
+                 }
+               ]
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "configs" => [
+                 %{
+                   "group" => ":pleroma",
+                   "key" => "Pleroma.Upload.Filter.Mogrify",
+                   "value" => [
+                     %{"tuple" => [":args", ["auto-orient", "strip"]]}
+                   ],
+                   "db" => [":args"]
+                 }
+               ],
+               "need_reboot" => false
+             }
+
+      assert Config.get(Pleroma.Upload.Filter.Mogrify) == [args: ["auto-orient", "strip"]]
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/pleroma/admin/config", %{
+               configs: [
+                 %{
+                   group: ":pleroma",
+                   key: "Pleroma.Upload.Filter.Mogrify",
+                   value: [
+                     %{
+                       "tuple" => [
+                         ":args",
+                         [
+                           "auto-orient",
+                           "strip",
+                           "{\"implode\", \"1\"}",
+                           "{\"resize\", \"3840x1080>\"}"
+                         ]
+                       ]
+                     }
+                   ]
+                 }
+               ]
+             })
+             |> json_response(200) == %{
+               "configs" => [
+                 %{
+                   "group" => ":pleroma",
+                   "key" => "Pleroma.Upload.Filter.Mogrify",
+                   "value" => [
+                     %{
+                       "tuple" => [
+                         ":args",
+                         [
+                           "auto-orient",
+                           "strip",
+                           "{\"implode\", \"1\"}",
+                           "{\"resize\", \"3840x1080>\"}"
+                         ]
+                       ]
+                     }
+                   ],
+                   "db" => [":args"]
+                 }
+               ],
+               "need_reboot" => false
+             }
+
+      assert Config.get(Pleroma.Upload.Filter.Mogrify) == [
+               args: ["auto-orient", "strip", {"implode", "1"}, {"resize", "3840x1080>"}]
+             ]
+    end
   end
 
   describe "GET /api/pleroma/admin/config/descriptions" do

From 23decaab81b900bff0f6eacad7ea6a894239e4ce Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sun, 31 May 2020 12:38:24 +0300
Subject: [PATCH 219/283] fix for updated hackney

warning: :hackney_connect.partial_chain/1 is undefined or private
---
 test/config/config_db_test.exs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs
index a04575c6f..8d753e255 100644
--- a/test/config/config_db_test.exs
+++ b/test/config/config_db_test.exs
@@ -382,12 +382,6 @@ test "simple keyword" do
       assert ConfigDB.to_elixir_types([%{"tuple" => [":key", "value"]}]) == [key: "value"]
     end
 
-    test "keyword with partial_chain key" do
-      assert ConfigDB.to_elixir_types([
-               %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}
-             ]) == [partial_chain: &:hackney_connect.partial_chain/1]
-    end
-
     test "keyword" do
       assert ConfigDB.to_elixir_types([
                %{"tuple" => [":types", "Pleroma.PostgresTypes"]},

From e1603ac8fee2a660c3dc510dee5967e0fd1bbd98 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sun, 31 May 2020 13:25:04 +0300
Subject: [PATCH 220/283] fix attemps to merge map

---
 lib/pleroma/config/config_db.ex | 3 ++-
 test/config/config_db_test.exs  | 8 ++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 39b37c42e..70be17ecf 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -135,7 +135,8 @@ def update_or_create(params) do
 
     with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts),
          {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config},
-         {_, true, config} <- {:can_be_merged, is_list(params[:value]), config} do
+         {_, true, config} <-
+           {:can_be_merged, is_list(params[:value]) and is_list(config.value), config} do
       new_value = merge_group(config.group, config.key, config.value, params[:value])
       update(config, %{value: new_value})
     else
diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs
index 8d753e255..3895e2cda 100644
--- a/test/config/config_db_test.exs
+++ b/test/config/config_db_test.exs
@@ -43,7 +43,7 @@ test "common" do
 
       params = [
         %{group: :pleroma, key: key2, value: "another_value"},
-        %{group: :pleroma, key: config.key, value: "new_value"}
+        %{group: :pleroma, key: config.key, value: [a: 1, b: 2, c: "new_value"]}
       ]
 
       assert Repo.all(ConfigDB) |> length() == 1
@@ -55,7 +55,7 @@ test "common" do
       config1 = ConfigDB.get_by_params(%{group: config.group, key: config.key})
       config2 = ConfigDB.get_by_params(%{group: :pleroma, key: key2})
 
-      assert config1.value == "new_value"
+      assert config1.value == [a: 1, b: 2, c: "new_value"]
       assert config2.value == "another_value"
     end
 
@@ -398,6 +398,10 @@ test "keyword" do
              ]
     end
 
+    test "trandformed keyword" do
+      assert ConfigDB.to_elixir_types(a: 1, b: 2, c: "string") == [a: 1, b: 2, c: "string"]
+    end
+
     test "complex keyword with nested mixed childs" do
       assert ConfigDB.to_elixir_types([
                %{"tuple" => [":uploader", "Pleroma.Uploaders.Local"]},

From 32c6576b600e2f24310f429f4b2391f95a5b5ba0 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sun, 31 May 2020 14:42:15 +0300
Subject: [PATCH 221/283] naming

---
 lib/pleroma/config/config_db.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 70be17ecf..30bd51b05 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -331,7 +331,7 @@ def string_to_elixir_types("~r" <> _pattern = regex) do
   def string_to_elixir_types(":" <> atom), do: String.to_atom(atom)
 
   def string_to_elixir_types(value) do
-    if is_module_name?(value) do
+    if module_name?(value) do
       String.to_existing_atom("Elixir." <> value)
     else
       value
@@ -373,8 +373,8 @@ defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
     end
   end
 
-  @spec is_module_name?(String.t()) :: boolean()
-  def is_module_name?(string) do
+  @spec module_name?(String.t()) :: boolean()
+  def module_name?(string) do
     Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
       string in ["Oban", "Ueberauth", "ExSyslogger"]
   end

From aca6a7543ae97da2d1af8a6f9c547a0088d9e240 Mon Sep 17 00:00:00 2001
From: Steven Fuchs <steven.fuchs@dockyard.com>
Date: Tue, 16 Jun 2020 13:18:29 +0000
Subject: [PATCH 222/283] Upgrade to Elixir 1.9

---
 .gitlab-ci.yml          | 2 +-
 elixir_buildpack.config | 4 ++--
 mix.exs                 | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aad28a2d8..bc7b289a2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: elixir:1.8.1
+image: elixir:1.9.4
 
 variables: &global_variables
   POSTGRES_DB: pleroma_test
diff --git a/elixir_buildpack.config b/elixir_buildpack.config
index c23b08fb8..946408c12 100644
--- a/elixir_buildpack.config
+++ b/elixir_buildpack.config
@@ -1,2 +1,2 @@
-elixir_version=1.8.2
-erlang_version=21.3.7
+elixir_version=1.9.4
+erlang_version=22.3.4.1
diff --git a/mix.exs b/mix.exs
index 03b060bc0..6040c994e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -5,7 +5,7 @@ def project do
     [
       app: :pleroma,
       version: version("2.0.50"),
-      elixir: "~> 1.8",
+      elixir: "~> 1.9",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix, :gettext] ++ Mix.compilers(),
       elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())],

From 3c2cee33adcd79a76b192a66f6f2d3772e2fda99 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 16 Jun 2020 17:50:33 +0300
Subject: [PATCH 223/283] moving custom ecto types in context folders

---
 lib/pleroma/config/config_db.ex                    |  6 +++---
 .../activity_pub/object_validators}/date_time.ex   |  6 +++++-
 .../activity_pub/object_validators}/object_id.ex   |  6 +++++-
 .../activity_pub/object_validators}/recipients.ex  |  8 ++++++--
 .../activity_pub/object_validators}/safe_text.ex   |  2 +-
 .../activity_pub/object_validators}/uri.ex         |  6 +++++-
 .../{config/type => ecto_type/config}/atom.ex      |  6 +++++-
 .../type => ecto_type/config}/binary_value.ex      |  6 +++++-
 lib/pleroma/signature.ex                           |  4 ++--
 lib/pleroma/user.ex                                |  4 ++--
 lib/pleroma/web/activity_pub/object_validator.ex   |  4 ++--
 .../object_validators/announce_validator.ex        | 14 +++++++-------
 .../object_validators/chat_message_validator.ex    | 12 ++++++------
 .../create_chat_message_validator.ex               | 10 +++++-----
 .../object_validators/create_note_validator.ex     |  6 +++---
 .../object_validators/delete_validator.ex          | 14 +++++++-------
 .../object_validators/emoji_react_validator.ex     |  8 ++++----
 .../object_validators/like_validator.ex            | 14 +++++++-------
 .../object_validators/note_validator.ex            | 12 ++++++------
 .../object_validators/undo_validator.ex            |  8 ++++----
 .../object_validators/url_object_validator.ex      |  8 ++++++--
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  4 ++--
 .../object_validators/types/date_time_test.exs     |  2 +-
 .../object_validators/types/object_id_test.exs     |  2 +-
 .../object_validators/types/recipients_test.exs    |  2 +-
 .../object_validators/types/safe_text_test.exs     |  2 +-
 26 files changed, 102 insertions(+), 74 deletions(-)
 rename lib/pleroma/{web/activity_pub/object_validators/types => ecto_type/activity_pub/object_validators}/date_time.ex (77%)
 rename lib/pleroma/{web/activity_pub/object_validators/types => ecto_type/activity_pub/object_validators}/object_id.ex (69%)
 rename lib/pleroma/{web/activity_pub/object_validators/types => ecto_type/activity_pub/object_validators}/recipients.ex (64%)
 rename lib/pleroma/{web/activity_pub/object_validators/types => ecto_type/activity_pub/object_validators}/safe_text.ex (85%)
 rename lib/pleroma/{web/activity_pub/object_validators/types => ecto_type/activity_pub/object_validators}/uri.ex (63%)
 rename lib/pleroma/{config/type => ecto_type/config}/atom.ex (66%)
 rename lib/pleroma/{config/type => ecto_type/config}/binary_value.ex (65%)

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 30bd51b05..2f4eb8581 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -23,9 +23,9 @@ defmodule Pleroma.ConfigDB do
   ]
 
   schema "config" do
-    field(:key, Pleroma.Config.Type.Atom)
-    field(:group, Pleroma.Config.Type.Atom)
-    field(:value, Pleroma.Config.Type.BinaryValue)
+    field(:key, Pleroma.EctoType.Config.Atom)
+    field(:group, Pleroma.EctoType.Config.Atom)
+    field(:value, Pleroma.EctoType.Config.BinaryValue)
     field(:db, {:array, :string}, virtual: true, default: [])
 
     timestamps()
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex
similarity index 77%
rename from lib/pleroma/web/activity_pub/object_validators/types/date_time.ex
rename to lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex
index 4f412fcde..d852c0abd 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime do
   @moduledoc """
   The AP standard defines the date fields in AP as xsd:DateTime. Elixir's
   DateTime can't parse this, but it can parse the related iso8601. This
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex
similarity index 69%
rename from lib/pleroma/web/activity_pub/object_validators/types/object_id.ex
rename to lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex
index f71f76370..8034235b0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID do
   use Ecto.Type
 
   def type, do: :string
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
similarity index 64%
rename from lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
rename to lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
index 408e0f6ee..205527a96 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
@@ -1,7 +1,11 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
   use Ecto.Type
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID
 
   def type, do: {:array, ObjectID}
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex
similarity index 85%
rename from lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
rename to lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex
index 95c948123..7f0405c7b 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText do
   use Ecto.Type
 
   alias Pleroma.HTML
diff --git a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex
similarity index 63%
rename from lib/pleroma/web/activity_pub/object_validators/types/uri.ex
rename to lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex
index 24845bcc0..2054c26be 100644
--- a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Uri do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Uri do
   use Ecto.Type
 
   def type, do: :string
diff --git a/lib/pleroma/config/type/atom.ex b/lib/pleroma/ecto_type/config/atom.ex
similarity index 66%
rename from lib/pleroma/config/type/atom.ex
rename to lib/pleroma/ecto_type/config/atom.ex
index 387869284..df565d432 100644
--- a/lib/pleroma/config/type/atom.ex
+++ b/lib/pleroma/ecto_type/config/atom.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Config.Type.Atom do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.Config.Atom do
   use Ecto.Type
 
   def type, do: :atom
diff --git a/lib/pleroma/config/type/binary_value.ex b/lib/pleroma/ecto_type/config/binary_value.ex
similarity index 65%
rename from lib/pleroma/config/type/binary_value.ex
rename to lib/pleroma/ecto_type/config/binary_value.ex
index 17c5524a3..bbd2608c5 100644
--- a/lib/pleroma/config/type/binary_value.ex
+++ b/lib/pleroma/ecto_type/config/binary_value.ex
@@ -1,4 +1,8 @@
-defmodule Pleroma.Config.Type.BinaryValue do
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.Config.BinaryValue do
   use Ecto.Type
 
   def type, do: :term
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index d01728361..3aa6909d2 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -5,10 +5,10 @@
 defmodule Pleroma.Signature do
   @behaviour HTTPSignatures.Adapter
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   def key_id_to_actor_id(key_id) do
     uri =
@@ -24,7 +24,7 @@ def key_id_to_actor_id(key_id) do
 
     maybe_ap_id = URI.to_string(uri)
 
-    case Types.ObjectID.cast(maybe_ap_id) do
+    case ObjectValidators.ObjectID.cast(maybe_ap_id) do
       {:ok, ap_id} ->
         {:ok, ap_id}
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 52ac9052b..686ab0123 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.User do
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Delivery
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Emoji
   alias Pleroma.FollowingRelationship
   alias Pleroma.Formatter
@@ -30,7 +31,6 @@ defmodule Pleroma.User do
   alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
@@ -115,7 +115,7 @@ defmodule Pleroma.User do
     field(:is_admin, :boolean, default: false)
     field(:show_role, :boolean, default: true)
     field(:settings, :map, default: nil)
-    field(:uri, Types.Uri, default: nil)
+    field(:uri, ObjectValidators.Uri, default: nil)
     field(:hide_followers_count, :boolean, default: false)
     field(:hide_follows_count, :boolean, default: false)
     field(:hide_followers, :boolean, default: false)
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index c01c5f780..3d699e8a5 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   """
 
   alias Pleroma.Object
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
@@ -17,7 +18,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
 
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
@@ -120,7 +120,7 @@ def stringify_keys(object) when is_list(object) do
   def stringify_keys(object), do: object
 
   def fetch_actor(object) do
-    with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do
+    with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
       User.get_or_fetch_by_ap_id(actor)
     end
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
index 40f861f47..6f757f49c 100644
--- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
 
@@ -19,14 +19,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:type, :string)
-    field(:object, Types.ObjectID)
-    field(:actor, Types.ObjectID)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
     field(:context, :string, autogenerate: {Utils, :generate_context_id, []})
-    field(:to, Types.Recipients, default: [])
-    field(:cc, Types.Recipients, default: [])
-    field(:published, Types.DateTime)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:published, ObjectValidators.DateTime)
   end
 
   def cast_and_validate(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 138736f23..c481d79e0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1]
@@ -16,12 +16,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
   @derive Jason.Encoder
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
-    field(:to, Types.Recipients, default: [])
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
     field(:type, :string)
-    field(:content, Types.SafeText)
-    field(:actor, Types.ObjectID)
-    field(:published, Types.DateTime)
+    field(:content, ObjectValidators.SafeText)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
     field(:emoji, :map, default: %{})
 
     embeds_one(:attachment, AttachmentValidator)
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
index fc582400b..7269f9ff0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -7,9 +7,9 @@
 # - doesn't embed, will only get the object id
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
   use Ecto.Schema
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
 
   alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -17,11 +17,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
-    field(:actor, Types.ObjectID)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:actor, ObjectValidators.ObjectID)
     field(:type, :string)
-    field(:to, Types.Recipients, default: [])
-    field(:object, Types.ObjectID)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:object, ObjectValidators.ObjectID)
   end
 
   def cast_and_apply(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
index 926804ce7..316bd0c07 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
@@ -5,16 +5,16 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
 
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
-    field(:actor, Types.ObjectID)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:actor, ObjectValidators.ObjectID)
     field(:type, :string)
     field(:to, {:array, :string})
     field(:cc, {:array, :string})
diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
index e5d08eb5c..93a7b0e0b 100644
--- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
   use Ecto.Schema
 
   alias Pleroma.Activity
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:type, :string)
-    field(:actor, Types.ObjectID)
-    field(:to, Types.Recipients, default: [])
-    field(:cc, Types.Recipients, default: [])
-    field(:deleted_activity_id, Types.ObjectID)
-    field(:object, Types.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:deleted_activity_id, ObjectValidators.ObjectID)
+    field(:object, ObjectValidators.ObjectID)
   end
 
   def cast_data(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
index e87519c59..a543af1f8 100644
--- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:type, :string)
-    field(:object, Types.ObjectID)
-    field(:actor, Types.ObjectID)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
     field(:context, :string)
     field(:content, :string)
     field(:to, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
index 034f25492..493e4c247 100644
--- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.Utils
 
   import Ecto.Changeset
@@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:type, :string)
-    field(:object, Types.ObjectID)
-    field(:actor, Types.ObjectID)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
     field(:context, :string)
-    field(:to, Types.Recipients, default: [])
-    field(:cc, Types.Recipients, default: [])
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
   end
 
   def cast_and_validate(data) do
@@ -67,7 +67,7 @@ def fix_recipients(cng) do
 
     with {[], []} <- {to, cc},
          %Object{data: %{"actor" => actor}} <- Object.get_cached_by_ap_id(object),
-         {:ok, actor} <- Types.ObjectID.cast(actor) do
+         {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
       cng
       |> put_change(:to, [actor])
     else
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 462a5620a..a10728ac6 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -5,14 +5,14 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
   use Ecto.Schema
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
 
   import Ecto.Changeset
 
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:to, {:array, :string}, default: [])
     field(:cc, {:array, :string}, default: [])
     field(:bto, {:array, :string}, default: [])
@@ -22,10 +22,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:type, :string)
     field(:content, :string)
     field(:context, :string)
-    field(:actor, Types.ObjectID)
-    field(:attributedTo, Types.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:attributedTo, ObjectValidators.ObjectID)
     field(:summary, :string)
-    field(:published, Types.DateTime)
+    field(:published, ObjectValidators.DateTime)
     # TODO: Write type
     field(:emoji, :map, default: %{})
     field(:sensitive, :boolean, default: false)
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
     field(:inRepyTo, :string)
-    field(:uri, Types.Uri)
+    field(:uri, ObjectValidators.Uri)
 
     field(:likes, {:array, :string}, default: [])
     field(:announcements, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
index d0ba418e8..e8d2d39c1 100644
--- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
   use Ecto.Schema
 
   alias Pleroma.Activity
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
   @primary_key false
 
   embedded_schema do
-    field(:id, Types.ObjectID, primary_key: true)
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
     field(:type, :string)
-    field(:object, Types.ObjectID)
-    field(:actor, Types.ObjectID)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
     field(:to, {:array, :string}, default: [])
     field(:cc, {:array, :string}, default: [])
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
index 47e231150..f64fac46d 100644
--- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
@@ -1,14 +1,18 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
   use Ecto.Schema
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
 
   import Ecto.Changeset
   @primary_key false
 
   embedded_schema do
     field(:type, :string)
-    field(:href, Types.Uri)
+    field(:href, ObjectValidators.Uri)
     field(:mediaType, :string)
   end
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 985921aa0..851f474b8 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.FollowingRelationship
   alias Pleroma.Maps
   alias Pleroma.Notification
@@ -18,7 +19,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.ObjectValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
@@ -725,7 +725,7 @@ def handle_incoming(
     else
       {:error, {:validate_object, _}} = e ->
         # Check if we have a create activity for this
-        with {:ok, object_id} <- Types.ObjectID.cast(data["object"]),
+        with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
              %Activity{data: %{"actor" => actor}} <-
                Activity.create_by_object_ap_id(object_id) |> Repo.one(),
              # We have one, insert a tombstone and retry
diff --git a/test/web/activity_pub/object_validators/types/date_time_test.exs b/test/web/activity_pub/object_validators/types/date_time_test.exs
index 3e17a9497..43be8e936 100644
--- a/test/web/activity_pub/object_validators/types/date_time_test.exs
+++ b/test/web/activity_pub/object_validators/types/date_time_test.exs
@@ -1,5 +1,5 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTimeTest do
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime
   use Pleroma.DataCase
 
   test "it validates an xsd:Datetime" do
diff --git a/test/web/activity_pub/object_validators/types/object_id_test.exs b/test/web/activity_pub/object_validators/types/object_id_test.exs
index c8911948e..e0ab76379 100644
--- a/test/web/activity_pub/object_validators/types/object_id_test.exs
+++ b/test/web/activity_pub/object_validators/types/object_id_test.exs
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ObjectValidators.Types.ObjectIDTest do
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID
   use Pleroma.DataCase
 
   @uris [
diff --git a/test/web/activity_pub/object_validators/types/recipients_test.exs b/test/web/activity_pub/object_validators/types/recipients_test.exs
index f278f039b..053916bdd 100644
--- a/test/web/activity_pub/object_validators/types/recipients_test.exs
+++ b/test/web/activity_pub/object_validators/types/recipients_test.exs
@@ -1,5 +1,5 @@
 defmodule Pleroma.Web.ObjectValidators.Types.RecipientsTest do
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients
   use Pleroma.DataCase
 
   test "it asserts that all elements of the list are object ids" do
diff --git a/test/web/activity_pub/object_validators/types/safe_text_test.exs b/test/web/activity_pub/object_validators/types/safe_text_test.exs
index d4a574554..9c08606f6 100644
--- a/test/web/activity_pub/object_validators/types/safe_text_test.exs
+++ b/test/web/activity_pub/object_validators/types/safe_text_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeTextTest do
   use Pleroma.DataCase
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText
 
   test "it lets normal text go through" do
     text = "hey how are you"

From ed189568f3c2c6fc6ae9ba4d676e95902b3019d1 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 21 Mar 2020 09:47:05 +0300
Subject: [PATCH 224/283] moving mrf settings from instance to separate group

---
 CHANGELOG.md                                  |  5 +-
 config/config.exs                             |  8 ++-
 config/description.exs                        | 64 +++++++++++--------
 docs/configuration/cheatsheet.md              | 45 +++++++------
 docs/configuration/mrf.md                     | 14 ++--
 lib/pleroma/config/config_db.ex               |  8 ++-
 lib/pleroma/config/deprecation_warnings.ex    | 44 +++++++++++++
 lib/pleroma/web/activity_pub/mrf.ex           |  4 +-
 .../web/activity_pub/mrf/simple_policy.ex     | 28 ++++----
 .../web/mastodon_api/views/instance_view.ex   |  2 +-
 ...rf_config_move_from_instance_namespace.exs | 39 +++++++++++
 test/config/deprecation_warnings_test.exs     | 57 +++++++++++++++++
 test/tasks/config_test.exs                    |  5 +-
 test/web/activity_pub/mrf/mrf_test.exs        |  4 +-
 test/web/federator_test.exs                   |  4 +-
 test/web/node_info_test.exs                   | 46 ++++---------
 16 files changed, 256 insertions(+), 121 deletions(-)
 create mode 100644 priv/repo/migrations/20200323122421_mrf_config_move_from_instance_namespace.exs
 create mode 100644 test/config/deprecation_warnings_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2629bf84..12e8d58e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,10 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
-
 ### Changed
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard
+- Configuration: `rewrite_policy` renamed to `policies` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
+- Configuration: `mrf_transparency` renamed to `transparency` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
+- Configuration: `mrf_transparency_exclusions` renamed to `transparency_exclusions` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
+
 <details>
   <summary>API Changes</summary>
 - **Breaking:** Emoji API: changed methods and renamed routes.
diff --git a/config/config.exs b/config/config.exs
index 6a7bb9e06..3d6336a66 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -209,7 +209,6 @@
     Pleroma.Web.ActivityPub.Publisher
   ],
   allow_relay: true,
-  rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
   public: true,
   quarantined_instances: [],
   managed_config: true,
@@ -220,8 +219,6 @@
     "text/markdown",
     "text/bbcode"
   ],
-  mrf_transparency: true,
-  mrf_transparency_exclusions: [],
   autofollowed_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
@@ -685,6 +682,11 @@
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
+config :pleroma, :mrf,
+  policies: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
+  transparency: true,
+  transparency_exclusions: []
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/config/description.exs b/config/description.exs
index b21d7840c..2ab95e5ab 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -689,17 +689,6 @@
         type: :boolean,
         description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance"
       },
-      %{
-        key: :rewrite_policy,
-        type: [:module, {:list, :module}],
-        description:
-          "A list of enabled MRF policies. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
-        suggestions:
-          Generator.list_modules_in_dir(
-            "lib/pleroma/web/activity_pub/mrf",
-            "Elixir.Pleroma.Web.ActivityPub.MRF."
-          )
-      },
       %{
         key: :public,
         type: :boolean,
@@ -742,23 +731,6 @@
           "text/bbcode"
         ]
       },
-      %{
-        key: :mrf_transparency,
-        label: "MRF transparency",
-        type: :boolean,
-        description:
-          "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
-      },
-      %{
-        key: :mrf_transparency_exclusions,
-        label: "MRF transparency exclusions",
-        type: {:list, :string},
-        description:
-          "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
-        suggestions: [
-          "exclusion.com"
-        ]
-      },
       %{
         key: :extended_nickname_format,
         type: :boolean,
@@ -3325,5 +3297,41 @@
         suggestions: [false]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: :mrf,
+    type: :group,
+    description: "General MRF settings",
+    children: [
+      %{
+        key: :policies,
+        type: [:module, {:list, :module}],
+        description:
+          "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
+        suggestions:
+          Generator.list_modules_in_dir(
+            "lib/pleroma/web/activity_pub/mrf",
+            "Elixir.Pleroma.Web.ActivityPub.MRF."
+          )
+      },
+      %{
+        key: :transparency,
+        label: "MRF transparency",
+        type: :boolean,
+        description:
+          "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
+      },
+      %{
+        key: :transparency_exclusions,
+        label: "MRF transparency exclusions",
+        type: {:list, :string},
+        description:
+          "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
+        suggestions: [
+          "exclusion.com"
+        ]
+      }
+    ]
   }
 ]
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index fad67fc4d..e9af604e2 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -36,26 +36,10 @@ To add configuration to your config file, you can copy it from the base config.
 * `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes.
 * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance.
-* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
-    * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default).
-    * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production.
-    * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certain instances (See [`:mrf_simple`](#mrf_simple)).
-    * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive).
-    * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (See [`:mrf_subchain`](#mrf_subchain)).
-    * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See [`:mrf_rejectnonpublic`](#mrf_rejectnonpublic)).
-    * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:.
-    * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links.
-    * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed.
-    * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
-    * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
-    * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
-    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Adds expiration to all local Create activities (see [`:mrf_activity_expiration`](#mrf_activity_expiration)).
 * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
 * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
 * `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
-* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
-* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
@@ -78,11 +62,30 @@ To add configuration to your config file, you can copy it from the base config.
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
 * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 
+## Message rewrite facility
+
+### :mrf
+* `policies`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
+    * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default).
+    * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production.
+    * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See [`:mrf_simple`](#mrf_simple)).
+    * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive).
+    * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (See [`:mrf_subchain`](#mrf_subchain)).
+    * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See [`:mrf_rejectnonpublic`](#mrf_rejectnonpublic)).
+    * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:.
+    * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links.
+    * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed.
+    * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
+    * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
+    * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
+* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
+* `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
+
 ## Federation
 ### MRF policies
 
 !!! note
-    Configuring MRF policies is not enough for them to take effect. You have to enable them by specifying their module in `rewrite_policy` under [:instance](#instance) section.
+    Configuring MRF policies is not enough for them to take effect. You have to enable them by specifying their module in `policies` under [:mrf](#mrf) section.
 
 #### :mrf_simple
 * `media_removal`: List of instances to remove media from.
@@ -969,13 +972,13 @@ config :pleroma, :database_config_whitelist, [
 
 Restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
 
-* `timelines` - public and federated timelines
-  * `local` - public timeline
+* `timelines`: public and federated timelines
+  * `local`: public timeline
   * `federated`
-* `profiles` - user profiles
+* `profiles`: user profiles
   * `local`
   * `remote`
-* `activities` - statuses
+* `activities`: statuses
   * `local`
   * `remote`
 
diff --git a/docs/configuration/mrf.md b/docs/configuration/mrf.md
index d48d0cc99..31c66e098 100644
--- a/docs/configuration/mrf.md
+++ b/docs/configuration/mrf.md
@@ -34,9 +34,9 @@ config :pleroma, :instance,
 To use `SimplePolicy`, you must enable it. Do so by adding the following to your `:instance` config object, so that it looks like this:
 
 ```elixir
-config :pleroma, :instance,
+config :pleroma, :mrf,
   [...]
-  rewrite_policy: Pleroma.Web.ActivityPub.MRF.SimplePolicy
+  policies: Pleroma.Web.ActivityPub.MRF.SimplePolicy
 ```
 
 Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object. These groups are:
@@ -58,8 +58,8 @@ Servers should be configured as lists.
 This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```elixir
-config :pleroma, :instance,
-  rewrite_policy: [Pleroma.Web.ActivityPub.MRF.SimplePolicy]
+config :pleroma, :mrf,
+  policies: [Pleroma.Web.ActivityPub.MRF.SimplePolicy]
 
 config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
@@ -75,7 +75,7 @@ The effects of MRF policies can be very drastic. It is important to use this fun
 
 ## Writing your own MRF Policy
 
-As discussed above, the MRF system is a modular system that supports pluggable policies. This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `rewrite_policy` config setting.
+As discussed above, the MRF system is a modular system that supports pluggable policies. This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `policies` config setting.
 
 For example, here is a sample policy module which rewrites all messages to "new message content":
 
@@ -125,8 +125,8 @@ end
 If you save this file as `lib/pleroma/web/activity_pub/mrf/rewrite_policy.ex`, it will be included when you next rebuild Pleroma.  You can enable it in the configuration like so:
 
 ```elixir
-config :pleroma, :instance,
-  rewrite_policy: [
+config :pleroma, :mrf,
+  policies: [
     Pleroma.Web.ActivityPub.MRF.SimplePolicy,
     Pleroma.Web.ActivityPub.MRF.RewritePolicy
   ]
diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 30bd51b05..c0f3fe888 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -54,13 +54,13 @@ def changeset(config, params \\ %{}) do
 
   defp create(params) do
     %ConfigDB{}
-    |> changeset(params)
+    |> changeset(params, transform?)
     |> Repo.insert()
   end
 
   defp update(%ConfigDB{} = config, %{value: value}) do
     config
-    |> changeset(%{value: value})
+    |> changeset(%{value: value}, transform?)
     |> Repo.update()
   end
 
@@ -167,7 +167,9 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do
     end)
   end
 
-  @spec delete(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
+  @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()}
+  def delete(%ConfigDB{} = config), do: Repo.delete(config)
+
   def delete(params) do
     search_opts = Map.delete(params, :subkeys)
 
diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index b68ded01f..0a6c724fb 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -3,9 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.DeprecationWarnings do
+  alias Pleroma.Config
+
   require Logger
   alias Pleroma.Config
 
+  @type config_namespace() :: [atom()]
+  @type config_map() :: {config_namespace(), config_namespace(), String.t()}
+
+  @mrf_config_map [
+    {[:instance, :rewrite_policy], [:mrf, :policies],
+     "\n* `config :pleroma, :instance, rewrite_policy` is now `config :pleroma, :mrf, policies`"},
+    {[:instance, :mrf_transparency], [:mrf, :transparency],
+     "\n* `config :pleroma, :instance, mrf_transparency` is now `config :pleroma, :mrf, transparency`"},
+    {[:instance, :mrf_transparency_exclusions], [:mrf, :transparency_exclusions],
+     "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
+  ]
+
   def check_hellthread_threshold do
     if Config.get([:mrf_hellthread, :threshold]) do
       Logger.warn("""
@@ -39,5 +53,35 @@ def mrf_user_allowlist do
   def warn do
     check_hellthread_threshold()
     mrf_user_allowlist()
+    check_old_mrf_config()
+  end
+
+  def check_old_mrf_config do
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old namespaces for MRF configuration. They should work for now, but you are advised to change to new namespaces to prevent possible issues later:
+    """
+
+    move_namespace_and_warn(@mrf_config_map, warning_preface)
+  end
+
+  @spec move_namespace_and_warn([config_map()], String.t()) :: :ok
+  def move_namespace_and_warn(config_map, warning_preface) do
+    warning =
+      Enum.reduce(config_map, "", fn
+        {old, new, err_msg}, acc ->
+          old_config = Config.get(old)
+
+          if old_config do
+            Config.put(new, old_config)
+            acc <> err_msg
+          else
+            acc
+          end
+      end)
+
+    if warning != "" do
+      Logger.warn(warning_preface <> warning)
+    end
   end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 5a4a76085..206d6af52 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -16,7 +16,7 @@ def filter(policies, %{} = object) do
   def filter(%{} = object), do: get_policies() |> filter(object)
 
   def get_policies do
-    Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies()
+    Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
   end
 
   defp get_policies(policy) when is_atom(policy), do: [policy]
@@ -51,7 +51,7 @@ def describe(policies) do
       get_policies()
       |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end)
 
-    exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions])
+    exclusions = Pleroma.Config.get([:mrf, :transparency_exclusions])
 
     base =
       %{
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index b7dcb1b86..9cea6bcf9 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -3,21 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.MRF
   @moduledoc "Filter activities depending on their origin instance"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  alias Pleroma.Config
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.MRF
+
   require Pleroma.Constants
 
   defp check_accept(%{host: actor_host} = _actor_info, object) do
     accepts =
-      Pleroma.Config.get([:mrf_simple, :accept])
+      Config.get([:mrf_simple, :accept])
       |> MRF.subdomains_regex()
 
     cond do
       accepts == [] -> {:ok, object}
-      actor_host == Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object}
+      actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object}
       MRF.subdomain_match?(accepts, actor_host) -> {:ok, object}
       true -> {:reject, nil}
     end
@@ -25,7 +27,7 @@ defp check_accept(%{host: actor_host} = _actor_info, object) do
 
   defp check_reject(%{host: actor_host} = _actor_info, object) do
     rejects =
-      Pleroma.Config.get([:mrf_simple, :reject])
+      Config.get([:mrf_simple, :reject])
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(rejects, actor_host) do
@@ -41,7 +43,7 @@ defp check_media_removal(
        )
        when length(child_attachment) > 0 do
     media_removal =
-      Pleroma.Config.get([:mrf_simple, :media_removal])
+      Config.get([:mrf_simple, :media_removal])
       |> MRF.subdomains_regex()
 
     object =
@@ -65,7 +67,7 @@ defp check_media_nsfw(
          } = object
        ) do
     media_nsfw =
-      Pleroma.Config.get([:mrf_simple, :media_nsfw])
+      Config.get([:mrf_simple, :media_nsfw])
       |> MRF.subdomains_regex()
 
     object =
@@ -85,7 +87,7 @@ defp check_media_nsfw(_actor_info, object), do: {:ok, object}
 
   defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
     timeline_removal =
-      Pleroma.Config.get([:mrf_simple, :federated_timeline_removal])
+      Config.get([:mrf_simple, :federated_timeline_removal])
       |> MRF.subdomains_regex()
 
     object =
@@ -108,7 +110,7 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
 
   defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
     report_removal =
-      Pleroma.Config.get([:mrf_simple, :report_removal])
+      Config.get([:mrf_simple, :report_removal])
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(report_removal, actor_host) do
@@ -122,7 +124,7 @@ defp check_report_removal(_actor_info, object), do: {:ok, object}
 
   defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do
     avatar_removal =
-      Pleroma.Config.get([:mrf_simple, :avatar_removal])
+      Config.get([:mrf_simple, :avatar_removal])
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(avatar_removal, actor_host) do
@@ -136,7 +138,7 @@ defp check_avatar_removal(_actor_info, object), do: {:ok, object}
 
   defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do
     banner_removal =
-      Pleroma.Config.get([:mrf_simple, :banner_removal])
+      Config.get([:mrf_simple, :banner_removal])
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(banner_removal, actor_host) do
@@ -197,10 +199,10 @@ def filter(object), do: {:ok, object}
 
   @impl true
   def describe do
-    exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions])
+    exclusions = Config.get([:mrf, :transparency_exclusions])
 
     mrf_simple =
-      Pleroma.Config.get(:mrf_simple)
+      Config.get(:mrf_simple)
       |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end)
       |> Enum.into(%{})
 
diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index c498fe632..4f0ae4e8f 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -78,7 +78,7 @@ def features do
   def federation do
     quarantined = Config.get([:instance, :quarantined_instances], [])
 
-    if Config.get([:instance, :mrf_transparency]) do
+    if Config.get([:mrf, :transparency]) do
       {:ok, data} = MRF.describe()
 
       data
diff --git a/priv/repo/migrations/20200323122421_mrf_config_move_from_instance_namespace.exs b/priv/repo/migrations/20200323122421_mrf_config_move_from_instance_namespace.exs
new file mode 100644
index 000000000..6f6094613
--- /dev/null
+++ b/priv/repo/migrations/20200323122421_mrf_config_move_from_instance_namespace.exs
@@ -0,0 +1,39 @@
+defmodule Pleroma.Repo.Migrations.MrfConfigMoveFromInstanceNamespace do
+  use Ecto.Migration
+
+  alias Pleroma.ConfigDB
+
+  @old_keys [:rewrite_policy, :mrf_transparency, :mrf_transparency_exclusions]
+  def change do
+    config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":instance"})
+
+    if config do
+      old_instance = ConfigDB.from_binary(config.value)
+
+      mrf =
+        old_instance
+        |> Keyword.take(@old_keys)
+        |> Keyword.new(fn
+          {:rewrite_policy, policies} -> {:policies, policies}
+          {:mrf_transparency, transparency} -> {:transparency, transparency}
+          {:mrf_transparency_exclusions, exclusions} -> {:transparency_exclusions, exclusions}
+        end)
+
+      if mrf != [] do
+        {:ok, _} =
+          ConfigDB.create(
+            %{group: ":pleroma", key: ":mrf", value: ConfigDB.to_binary(mrf)},
+            false
+          )
+
+        new_instance = Keyword.drop(old_instance, @old_keys)
+
+        if new_instance != [] do
+          {:ok, _} = ConfigDB.update(config, %{value: ConfigDB.to_binary(new_instance)}, false)
+        else
+          {:ok, _} = ConfigDB.delete(config)
+        end
+      end
+    end
+  end
+end
diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs
new file mode 100644
index 000000000..548ee87b0
--- /dev/null
+++ b/test/config/deprecation_warnings_test.exs
@@ -0,0 +1,57 @@
+defmodule Pleroma.Config.DeprecationWarningsTest do
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
+
+  import ExUnit.CaptureLog
+
+  test "check_old_mrf_config/0" do
+    clear_config([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.NoOpPolicy)
+    clear_config([:instance, :mrf_transparency], true)
+    clear_config([:instance, :mrf_transparency_exclusions], [])
+
+    assert capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_old_mrf_config() end) =~
+             """
+             !!!DEPRECATION WARNING!!!
+             Your config is using old namespaces for MRF configuration. They should work for now, but you are advised to change to new namespaces to prevent possible issues later:
+
+             * `config :pleroma, :instance, rewrite_policy` is now `config :pleroma, :mrf, policies`
+             * `config :pleroma, :instance, mrf_transparency` is now `config :pleroma, :mrf, transparency`
+             * `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`
+             """
+  end
+
+  test "move_namespace_and_warn/2" do
+    old_group1 = [:group, :key]
+    old_group2 = [:group, :key2]
+    old_group3 = [:group, :key3]
+
+    new_group1 = [:another_group, :key4]
+    new_group2 = [:another_group, :key5]
+    new_group3 = [:another_group, :key6]
+
+    clear_config(old_group1, 1)
+    clear_config(old_group2, 2)
+    clear_config(old_group3, 3)
+
+    clear_config(new_group1)
+    clear_config(new_group2)
+    clear_config(new_group3)
+
+    config_map = [
+      {old_group1, new_group1, "\n error :key"},
+      {old_group2, new_group2, "\n error :key2"},
+      {old_group3, new_group3, "\n error :key3"}
+    ]
+
+    assert capture_log(fn ->
+             Pleroma.Config.DeprecationWarnings.move_namespace_and_warn(
+               config_map,
+               "Warning preface"
+             )
+           end) =~ "Warning preface\n error :key\n error :key2\n error :key3"
+
+    assert Pleroma.Config.get(new_group1) == 1
+    assert Pleroma.Config.get(new_group2) == 2
+    assert Pleroma.Config.get(new_group3) == 3
+  end
+end
diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs
index e1bddfebf..bae171074 100644
--- a/test/tasks/config_test.exs
+++ b/test/tasks/config_test.exs
@@ -120,14 +120,11 @@ test "load a settings with large values and pass to file", %{temp_file: temp_fil
           federation_reachability_timeout_days: 7,
           federation_publisher_modules: [Pleroma.Web.ActivityPub.Publisher],
           allow_relay: true,
-          rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
           public: true,
           quarantined_instances: [],
           managed_config: true,
           static_dir: "instance/static/",
           allowed_post_formats: ["text/plain", "text/html", "text/markdown", "text/bbcode"],
-          mrf_transparency: true,
-          mrf_transparency_exclusions: [],
           autofollowed_nicknames: [],
           max_pinned_statuses: 1,
           attachment_links: false,
@@ -174,7 +171,7 @@ test "load a settings with large values and pass to file", %{temp_file: temp_fil
         end
 
       assert file ==
-               "#{header}\n\nconfig :pleroma, :instance,\n  name: \"Pleroma\",\n  email: \"example@example.com\",\n  notify_email: \"noreply@example.com\",\n  description: \"A Pleroma instance, an alternative fediverse server\",\n  limit: 5000,\n  chat_limit: 5000,\n  remote_limit: 100_000,\n  upload_limit: 16_000_000,\n  avatar_upload_limit: 2_000_000,\n  background_upload_limit: 4_000_000,\n  banner_upload_limit: 4_000_000,\n  poll_limits: %{\n    max_expiration: 31_536_000,\n    max_option_chars: 200,\n    max_options: 20,\n    min_expiration: 0\n  },\n  registrations_open: true,\n  federating: true,\n  federation_incoming_replies_max_depth: 100,\n  federation_reachability_timeout_days: 7,\n  federation_publisher_modules: [Pleroma.Web.ActivityPub.Publisher],\n  allow_relay: true,\n  rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,\n  public: true,\n  quarantined_instances: [],\n  managed_config: true,\n  static_dir: \"instance/static/\",\n  allowed_post_formats: [\"text/plain\", \"text/html\", \"text/markdown\", \"text/bbcode\"],\n  mrf_transparency: true,\n  mrf_transparency_exclusions: [],\n  autofollowed_nicknames: [],\n  max_pinned_statuses: 1,\n  attachment_links: false,\n  welcome_user_nickname: nil,\n  welcome_message: nil,\n  max_report_comment_size: 1000,\n  safe_dm_mentions: false,\n  healthcheck: false,\n  remote_post_retention_days: 90,\n  skip_thread_containment: true,\n  limit_to_local_content: :unauthenticated,\n  user_bio_length: 5000,\n  user_name_length: 100,\n  max_account_fields: 10,\n  max_remote_account_fields: 20,\n  account_field_name_length: 512,\n  account_field_value_length: 2048,\n  external_user_synchronization: true,\n  extended_nickname_format: true,\n  multi_factor_authentication: [\n    totp: [digits: 6, period: 30],\n    backup_codes: [number: 2, length: 6]\n  ]\n"
+               "#{header}\n\nconfig :pleroma, :instance,\n  name: \"Pleroma\",\n  email: \"example@example.com\",\n  notify_email: \"noreply@example.com\",\n  description: \"A Pleroma instance, an alternative fediverse server\",\n  limit: 5000,\n  chat_limit: 5000,\n  remote_limit: 100_000,\n  upload_limit: 16_000_000,\n  avatar_upload_limit: 2_000_000,\n  background_upload_limit: 4_000_000,\n  banner_upload_limit: 4_000_000,\n  poll_limits: %{\n    max_expiration: 31_536_000,\n    max_option_chars: 200,\n    max_options: 20,\n    min_expiration: 0\n  },\n  registrations_open: true,\n  federating: true,\n  federation_incoming_replies_max_depth: 100,\n  federation_reachability_timeout_days: 7,\n  federation_publisher_modules: [Pleroma.Web.ActivityPub.Publisher],\n  allow_relay: true,\n  public: true,\n  quarantined_instances: [],\n  managed_config: true,\n  static_dir: \"instance/static/\",\n  allowed_post_formats: [\"text/plain\", \"text/html\", \"text/markdown\", \"text/bbcode\"],\n  autofollowed_nicknames: [],\n  max_pinned_statuses: 1,\n  attachment_links: false,\n  welcome_user_nickname: nil,\n  welcome_message: nil,\n  max_report_comment_size: 1000,\n  safe_dm_mentions: false,\n  healthcheck: false,\n  remote_post_retention_days: 90,\n  skip_thread_containment: true,\n  limit_to_local_content: :unauthenticated,\n  user_bio_length: 5000,\n  user_name_length: 100,\n  max_account_fields: 10,\n  max_remote_account_fields: 20,\n  account_field_name_length: 512,\n  account_field_value_length: 2048,\n  external_user_synchronization: true,\n  extended_nickname_format: true,\n  multi_factor_authentication: [\n    totp: [digits: 6, period: 30],\n    backup_codes: [number: 2, length: 6]\n  ]\n"
     end
   end
 end
diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs
index c941066f2..a63b25423 100644
--- a/test/web/activity_pub/mrf/mrf_test.exs
+++ b/test/web/activity_pub/mrf/mrf_test.exs
@@ -60,8 +60,6 @@ test "matches are case-insensitive" do
   end
 
   describe "describe/0" do
-    setup do: clear_config([:instance, :rewrite_policy])
-
     test "it works as expected with noop policy" do
       expected = %{
         mrf_policies: ["NoOpPolicy"],
@@ -72,7 +70,7 @@ test "it works as expected with noop policy" do
     end
 
     test "it works as expected with mock policy" do
-      Pleroma.Config.put([:instance, :rewrite_policy], [MRFModuleMock])
+      clear_config([:mrf, :policies], [MRFModuleMock])
 
       expected = %{
         mrf_policies: ["MRFModuleMock"],
diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs
index de90aa6e0..592fdccd1 100644
--- a/test/web/federator_test.exs
+++ b/test/web/federator_test.exs
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.FederatorTest do
 
   setup_all do: clear_config([:instance, :federating], true)
   setup do: clear_config([:instance, :allow_relay])
-  setup do: clear_config([:instance, :rewrite_policy])
+  setup do: clear_config([:mrf, :policies])
   setup do: clear_config([:mrf_keyword])
 
   describe "Publish an activity" do
@@ -158,7 +158,7 @@ test "it does not crash if MRF rejects the post" do
       Pleroma.Config.put([:mrf_keyword, :reject], ["lain"])
 
       Pleroma.Config.put(
-        [:instance, :rewrite_policy],
+        [:mrf, :policies],
         Pleroma.Web.ActivityPub.MRF.KeywordPolicy
       )
 
diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs
index 00925caad..06b33607f 100644
--- a/test/web/node_info_test.exs
+++ b/test/web/node_info_test.exs
@@ -67,10 +67,10 @@ test "returns software.repository field in nodeinfo 2.1", %{conn: conn} do
   end
 
   test "returns fieldsLimits field", %{conn: conn} do
-    Config.put([:instance, :max_account_fields], 10)
-    Config.put([:instance, :max_remote_account_fields], 15)
-    Config.put([:instance, :account_field_name_length], 255)
-    Config.put([:instance, :account_field_value_length], 2048)
+    clear_config([:instance, :max_account_fields], 10)
+    clear_config([:instance, :max_remote_account_fields], 15)
+    clear_config([:instance, :account_field_name_length], 255)
+    clear_config([:instance, :account_field_value_length], 2048)
 
     response =
       conn
@@ -84,8 +84,7 @@ test "returns fieldsLimits field", %{conn: conn} do
   end
 
   test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do
-    option = Config.get([:instance, :safe_dm_mentions])
-    Config.put([:instance, :safe_dm_mentions], true)
+    clear_config([:instance, :safe_dm_mentions], true)
 
     response =
       conn
@@ -102,8 +101,6 @@ test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do
       |> json_response(:ok)
 
     refute "safe_dm_mentions" in response["metadata"]["features"]
-
-    Config.put([:instance, :safe_dm_mentions], option)
   end
 
   describe "`metadata/federation/enabled`" do
@@ -156,14 +153,11 @@ test "it shows default features flags", %{conn: conn} do
   end
 
   test "it shows MRF transparency data if enabled", %{conn: conn} do
-    config = Config.get([:instance, :rewrite_policy])
-    Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy])
-
-    option = Config.get([:instance, :mrf_transparency])
-    Config.put([:instance, :mrf_transparency], true)
+    clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy])
+    clear_config([:mrf, :transparency], true)
 
     simple_config = %{"reject" => ["example.com"]}
-    Config.put(:mrf_simple, simple_config)
+    clear_config(:mrf_simple, simple_config)
 
     response =
       conn
@@ -171,26 +165,17 @@ test "it shows MRF transparency data if enabled", %{conn: conn} do
       |> json_response(:ok)
 
     assert response["metadata"]["federation"]["mrf_simple"] == simple_config
-
-    Config.put([:instance, :rewrite_policy], config)
-    Config.put([:instance, :mrf_transparency], option)
-    Config.put(:mrf_simple, %{})
   end
 
   test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do
-    config = Config.get([:instance, :rewrite_policy])
-    Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy])
-
-    option = Config.get([:instance, :mrf_transparency])
-    Config.put([:instance, :mrf_transparency], true)
-
-    exclusions = Config.get([:instance, :mrf_transparency_exclusions])
-    Config.put([:instance, :mrf_transparency_exclusions], ["other.site"])
+    clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy])
+    clear_config([:mrf, :transparency], true)
+    clear_config([:mrf, :transparency_exclusions], ["other.site"])
 
     simple_config = %{"reject" => ["example.com", "other.site"]}
-    expected_config = %{"reject" => ["example.com"]}
+    clear_config(:mrf_simple, simple_config)
 
-    Config.put(:mrf_simple, simple_config)
+    expected_config = %{"reject" => ["example.com"]}
 
     response =
       conn
@@ -199,10 +184,5 @@ test "it performs exclusions from MRF transparency data if configured", %{conn:
 
     assert response["metadata"]["federation"]["mrf_simple"] == expected_config
     assert response["metadata"]["federation"]["exclusions"] == true
-
-    Config.put([:instance, :rewrite_policy], config)
-    Config.put([:instance, :mrf_transparency], option)
-    Config.put([:instance, :mrf_transparency_exclusions], exclusions)
-    Config.put(:mrf_simple, %{})
   end
 end

From b66e6eb521c2901da119179016c99751cb5e6f95 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 16 Jun 2020 19:03:45 +0300
Subject: [PATCH 225/283] fixes for tests

---
 docs/configuration/storing_remote_media.md                 | 4 ++--
 lib/pleroma/config/config_db.ex                            | 4 ++--
 test/web/activity_pub/activity_pub_test.exs                | 4 ++--
 test/workers/cron/purge_expired_activities_worker_test.exs | 6 +-----
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/docs/configuration/storing_remote_media.md b/docs/configuration/storing_remote_media.md
index 7e91fe7d9..c01985d25 100644
--- a/docs/configuration/storing_remote_media.md
+++ b/docs/configuration/storing_remote_media.md
@@ -33,6 +33,6 @@ as soon as the post is received by your instance.
 Add to your `prod.secret.exs`:
 
 ```
-config :pleroma, :instance,
-  rewrite_policy: [Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+config :pleroma, :mrf,
+  policies: [Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
 ```
diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index c0f3fe888..134116863 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -54,13 +54,13 @@ def changeset(config, params \\ %{}) do
 
   defp create(params) do
     %ConfigDB{}
-    |> changeset(params, transform?)
+    |> changeset(params)
     |> Repo.insert()
   end
 
   defp update(%ConfigDB{} = config, %{value: value}) do
     config
-    |> changeset(%{value: value}, transform?)
+    |> changeset(%{value: value})
     |> Repo.update()
   end
 
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 7693f6400..1c684df1a 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -2055,11 +2055,11 @@ test "it just returns the input if the user has no following/follower addresses"
   end
 
   describe "global activity expiration" do
-    setup do: clear_config([:instance, :rewrite_policy])
+    setup do: clear_config([:mrf, :policies])
 
     test "creates an activity expiration for local Create activities" do
       Pleroma.Config.put(
-        [:instance, :rewrite_policy],
+        [:mrf, :policies],
         Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
       )
 
diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs
index 6d2991a60..b1db59fdf 100644
--- a/test/workers/cron/purge_expired_activities_worker_test.exs
+++ b/test/workers/cron/purge_expired_activities_worker_test.exs
@@ -13,7 +13,6 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do
 
   setup do
     clear_config([ActivityExpiration, :enabled])
-    clear_config([:instance, :rewrite_policy])
   end
 
   test "deletes an expiration activity" do
@@ -42,10 +41,7 @@ test "deletes an expiration activity" do
   test "works with ActivityExpirationPolicy" do
     Pleroma.Config.put([ActivityExpiration, :enabled], true)
 
-    Pleroma.Config.put(
-      [:instance, :rewrite_policy],
-      Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
-    )
+    clear_config([:mrf, :policies], Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy)
 
     user = insert(:user)
 

From 5c0e1039ce41a2717598992a590658d4d079451c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 16 Jun 2020 23:45:59 +0300
Subject: [PATCH 226/283] Chunk the notification type backfill migration

Long-term we want that migration to be done entirely in SQL,
but for now this is a hotfix to not cause OOMs on large databases.

This is using a homegrown version of `Repo.stream`, it's worse in
terms of performance than the upstream since it doesn't use the same
prepared query for chunk queries, but unlike the upstream it supports
preloads.
---
 .../migration_helper/notification_backfill.ex |  2 +-
 lib/pleroma/repo.ex                           | 28 +++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex
index 09647d12a..b3770307a 100644
--- a/lib/pleroma/migration_helper/notification_backfill.ex
+++ b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -18,7 +18,7 @@ def fill_in_notification_types do
       )
 
     query
-    |> Repo.all()
+    |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
       type =
         notification.activity
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f62138466..6d85d70bc 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Repo do
     adapter: Ecto.Adapters.Postgres,
     migration_timestamps: [type: :naive_datetime_usec]
 
+  import Ecto.Query
   require Logger
 
   defmodule Instrumenter do
@@ -78,6 +79,33 @@ def check_migrations_applied!() do
       :ok
     end
   end
+
+  def chunk_stream(query, chunk_size) do
+    # We don't actually need start and end funcitons of resource streaming,
+    # but it seems to be the only way to not fetch records one-by-one and
+    # have individual records be the elements of the stream, instead of
+    # lists of records
+    Stream.resource(
+      fn -> 0 end,
+      fn
+        last_id ->
+          query
+          |> order_by(asc: :id)
+          |> where([r], r.id > ^last_id)
+          |> limit(^chunk_size)
+          |> all()
+          |> case do
+            [] ->
+              {:halt, last_id}
+
+            records ->
+              last_id = List.last(records).id
+              {records, last_id}
+          end
+      end,
+      fn _ -> :ok end
+    )
+  end
 end
 
 defmodule Pleroma.Repo.UnappliedMigrationsError do

From 55d8263c0040b32075b9bb90ab1d6693e627f6bf Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Jun 2020 02:27:28 +0300
Subject: [PATCH 227/283] Update OTP releases to official images of 1.10.3

This is necessary since we bumped required version of elixir to 1.9.

The dlsym bug should be gone by now.
---
 .gitlab-ci.yml | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bc7b289a2..b4bd59b43 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -170,8 +170,7 @@ stop_review_app:
 
 amd64:
   stage: release
-  # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0
+  image: elixir:1.10.3
   only: &release-only
   - stable@pleroma/pleroma
   - develop@pleroma/pleroma
@@ -208,8 +207,7 @@ amd64-musl:
   stage: release
   artifacts: *release-artifacts
   only: *release-only
-  # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0-alpine
+  image: elixir:1.10.3-alpine 
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
@@ -225,8 +223,7 @@ arm:
   only: *release-only
   tags:
     - arm32
-  # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0-arm
+  image: elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -238,8 +235,7 @@ arm-musl:
   only: *release-only
   tags:
     - arm32
-  # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0-arm-alpine
+  image: elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl
@@ -251,8 +247,7 @@ arm64:
   only: *release-only
   tags:
     - arm
-  # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0-arm64
+  image: elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -265,7 +260,7 @@ arm64-musl:
   tags:
     - arm
   # TODO: Replace with upstream image when 1.9.0 comes out
-  image: rinpatch/elixir:1.9.0-rc.0-arm64-alpine
+  image: elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl

From 281ecd6b30a165843c5b6a1899894646dc25c0f9 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Jun 2020 02:29:32 +0300
Subject: [PATCH 228/283] CHANGELOG.md: mention minimal elixir version update

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2f291ad2a..3ee13904f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - In Conversations, return only direct messages as `last_status`
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard

From 02a5648febb8a508116c29e2271e1ade2ffafb2d Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 09:15:35 +0300
Subject: [PATCH 229/283] fixed migration the settings to DB

---
 lib/mix/tasks/pleroma/config.ex               |  1 +
 lib/pleroma/config/loader.ex                  |  1 -
 ...510135645_add_fts_index_to_objects_two.exs | 33 +++++++++++--------
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index f1b3a8766..65691f9c1 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -52,6 +52,7 @@ def migrate_to_db(file_path \\ nil) do
 
   defp do_migrate_to_db(config_file) do
     if File.exists?(config_file) do
+      shell_info("Running migrate settings from file: #{Path.expand(config_file)}")
       Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
       Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
 
diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex
index 0f3ecf1ed..76559e70c 100644
--- a/lib/pleroma/config/loader.ex
+++ b/lib/pleroma/config/loader.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Config.Loader do
     Pleroma.Web.Endpoint,
     :env,
     :configurable_from_database,
-    :database,
     :swarm
   ]
 
diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 6227769dc..79bde163d 100644
--- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -2,24 +2,29 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
   use Ecto.Migration
 
   def up do
-    execute("create extension if not exists rum")
-    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
-    alter table(:objects) do
-      add(:fts_content, :tsvector)
-    end
+    if Pleroma.Config.get([:database, :rum_enabled]) do
+      execute("create extension if not exists rum")
+      drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+      alter table(:objects) do
+        add(:fts_content, :tsvector)
+      end
 
-    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
-    begin
+      execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+      begin
       new.fts_content := to_tsvector('english', new.data->>'content');
       return new;
+      end
+      $$ LANGUAGE plpgsql")
+      execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+      execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+      FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+      execute("UPDATE objects SET updated_at = NOW()")
+    else
+        raise Ecto.MigrationError,
+          message: "Migration is not allowed. You can change this behavior by setting `database/rum_enabled` to true."
     end
-    $$ LANGUAGE plpgsql")
-    execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
-
-    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
-    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
-
-    execute("UPDATE objects SET updated_at = NOW()")
   end
 
   def down do

From a77b0388f48084bd3a420855a232bf2e504f0bce Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 17 Jun 2020 10:31:06 +0300
Subject: [PATCH 230/283] credo fix

---
 lib/pleroma/web/activity_pub/object_validator.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 3d699e8a5..6a83a2c33 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   the system.
   """
 
-  alias Pleroma.Object
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator

From abda3f2d92eda0888e018cea0a2fffb21d9e0a60 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 17 Jun 2020 10:47:20 +0300
Subject: [PATCH 231/283] suggestion for changelog

---
 CHANGELOG.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 12e8d58e6..6095cf139 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,9 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Changed
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard
-- Configuration: `rewrite_policy` renamed to `policies` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
-- Configuration: `mrf_transparency` renamed to `transparency` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
-- Configuration: `mrf_transparency_exclusions` renamed to `transparency_exclusions` and moved from `instance` to `mrf` group. Old config namespace is deprecated.
+- Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
 
 <details>
   <summary>API Changes</summary>

From 9a82de219c264f467b485316570c5425e3fe2f00 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 17 Jun 2020 10:50:05 +0300
Subject: [PATCH 232/283] formatting

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6095cf139..fab87b569 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
+
 ### Changed
 - MFR policy to set global expiration for all local Create activities
 - OGP rich media parser merged with TwitterCard

From 90613348ed8078e1906f7ffd18eebfa1a3b7f25a Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:56:13 +0000
Subject: [PATCH 233/283] Apply suggestion to docs/API/admin_api.md

---
 docs/API/admin_api.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 6659b605d..8a3d60187 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1253,7 +1253,7 @@ Loads json generated from `config/descriptions.exs`.
 
 - Authentication: required
 - Params:
-  - `urls`
+  - `urls` (array)
 
 - Response:
 

From abfb1c756b62c24589d2881d77bf5974a80809d3 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:56:17 +0000
Subject: [PATCH 234/283] Apply suggestion to docs/API/admin_api.md

---
 docs/API/admin_api.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 8a3d60187..c7f56cf5f 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -1273,8 +1273,8 @@ Loads json generated from `config/descriptions.exs`.
 
 - Authentication: required
 - Params:
-  - `urls`
-  - `ban`
+  - `urls` (array)
+  - `ban` (boolean)
 
 - Response:
 

From 74fd761637f737822d01aed945b6e0c75ced7008 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:56:30 +0000
Subject: [PATCH 235/283] Apply suggestion to
 lib/pleroma/web/media_proxy/invalidation.ex

---
 lib/pleroma/web/media_proxy/invalidation.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index 83ff8589c..6da7eb720 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -32,6 +32,6 @@ defp do_purge(urls) do
   def prepare_urls(urls) do
     urls
     |> List.wrap()
-    |> Enum.map(&MediaProxy.url(&1))
+    |> Enum.map(&MediaProxy.url/1)
   end
 end

From 1b45bc7b2ac53e56a4868da7e2b5b198d16306ab Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:08 +0000
Subject: [PATCH 236/283] Apply suggestion to
 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs

---
 .../admin_api/controllers/media_proxy_cache_controller_test.exs  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
index 76a96f46f..ddaf39f14 100644
--- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
 
   setup do
     on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
-    :ok
   end
 
   setup do

From 793a53f1ec18a42f15f58494e40ed3c37d35f95c Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:16 +0000
Subject: [PATCH 237/283] Apply suggestion to
 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs

---
 .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
index ddaf39f14..81e20d001 100644
--- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -72,7 +72,7 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
     end
   end
 
-  describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do
+  describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
     test "deleted MediaProxy URLs from banned", %{conn: conn} do
       MediaProxy.put_in_deleted_urls([
         "http://localhost:4001/media/a688346.jpg",

From 6d33a3a51bb8ff0afdf7f4f9880f8f5c5f2dfebc Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:28 +0000
Subject: [PATCH 238/283] Apply suggestion to
 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs

---
 .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
index 81e20d001..42a3c0dd8 100644
--- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -93,7 +93,7 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
     end
   end
 
-  describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do
+  describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
     test "perform invalidates cache of MediaProxy", %{conn: conn} do
       urls = [
         "http://example.com/media/a688346.jpg",

From 11b22a42293ec2ac0e66897bf4b29b5363913c19 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:33 +0000
Subject: [PATCH 239/283] Apply suggestion to
 test/web/media_proxy/invalidations/http_test.exs

---
 test/web/media_proxy/invalidations/http_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs
index 09e7ca0fb..9d181dd8b 100644
--- a/test/web/media_proxy/invalidations/http_test.exs
+++ b/test/web/media_proxy/invalidations/http_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
 
   setup do
     on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
-    :ok
   end
 
   test "logs hasn't error message when request is valid" do

From 2991aae4c4e4ae430539c1e6fac53fb8a0c991e9 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:38 +0000
Subject: [PATCH 240/283] Apply suggestion to
 test/web/media_proxy/invalidations/script_test.exs

---
 test/web/media_proxy/invalidations/script_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs
index c69cec07a..8e155b705 100644
--- a/test/web/media_proxy/invalidations/script_test.exs
+++ b/test/web/media_proxy/invalidations/script_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
 
   setup do
     on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
-    :ok
   end
 
   test "it logger error when script not found" do

From 078d687e6ed66f921d7f54114f2dc6bf4abbf237 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 12:58:50 +0000
Subject: [PATCH 241/283] Apply suggestion to
 test/web/media_proxy/media_proxy_controller_test.exs

---
 test/web/media_proxy/media_proxy_controller_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs
index 2b6b25221..72da98a6a 100644
--- a/test/web/media_proxy/media_proxy_controller_test.exs
+++ b/test/web/media_proxy/media_proxy_controller_test.exs
@@ -12,7 +12,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
 
   setup do
     on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
-    :ok
   end
 
   test "it returns 404 when MediaProxy disabled", %{conn: conn} do

From 44ce97a9c9e90d5906386d1b51dea144cd258c32 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 13:12:32 +0000
Subject: [PATCH 242/283] Apply suggestion to
 lib/pleroma/web/media_proxy/invalidations/script.ex

---
 lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index 0217b119d..b0f44e8e2 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
   require Logger
 
   @impl Pleroma.Web.MediaProxy.Invalidation
-  def purge(urls, opts \\ %{}) do
+  def purge(urls, opts \\ []) do
     args =
       urls
       |> List.wrap()

From 9a371bf5f6245b0f372bec7af15e2f4fc41d4ab7 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 13:12:38 +0000
Subject: [PATCH 243/283] Apply suggestion to
 lib/pleroma/web/media_proxy/invalidations/script.ex

---
 lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex
index b0f44e8e2..d32ffc50b 100644
--- a/lib/pleroma/web/media_proxy/invalidations/script.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/script.ex
@@ -18,7 +18,7 @@ def purge(urls, opts \\ []) do
       |> Enum.join(" ")
 
     opts
-    |> Keyword.get(:script_path, nil)
+    |> Keyword.get(:script_path)
     |> do_purge([args])
     |> handle_result(urls)
   end

From 96493da7bdab4ff4a51cbebf18df4127ddc47990 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 17 Jun 2020 13:14:01 +0000
Subject: [PATCH 244/283] Apply suggestion to
 test/web/media_proxy/invalidation_test.exs

---
 test/web/media_proxy/invalidation_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
index 3a9fa8c88..bf9af251c 100644
--- a/test/web/media_proxy/invalidation_test.exs
+++ b/test/web/media_proxy/invalidation_test.exs
@@ -13,7 +13,6 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
 
   setup do
     on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
-    :ok
   end
 
   describe "Invalidation.Http" do

From d4b5a9730e8fe7adf5a2eca15bf40fafff85f30d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 17 Jun 2020 18:47:59 +0400
Subject: [PATCH 245/283] Remove `poll` from `notification_type` OpenAPI spec

---
 lib/pleroma/web/api_spec/operations/notification_operation.ex | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index c966b553a..41328b5f2 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -183,7 +183,6 @@ defp notification_type do
         "favourite",
         "reblog",
         "mention",
-        "poll",
         "pleroma:emoji_reaction",
         "pleroma:chat_mention",
         "move",

From 71a5d9bffb33d9424ea28900ea006678617e0096 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 17 Jun 2020 12:54:02 -0500
Subject: [PATCH 246/283] Empty list as default

---
 lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex
index 3694b56e8..bb81d8888 100644
--- a/lib/pleroma/web/media_proxy/invalidations/http.ex
+++ b/lib/pleroma/web/media_proxy/invalidations/http.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
   require Logger
 
   @impl Pleroma.Web.MediaProxy.Invalidation
-  def purge(urls, opts) do
+  def purge(urls, opts \\ []) do
     method = Keyword.get(opts, :method, :purge)
     headers = Keyword.get(opts, :headers, [])
     options = Keyword.get(opts, :options, [])

From c08c9db0c137d36896910194a6dc50a391a8fee2 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 17 Jun 2020 13:02:01 -0500
Subject: [PATCH 247/283] Remove misleading is_ prefix from boolean function

---
 lib/pleroma/web/media_proxy/media_proxy.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 59ca217ab..3dccd6b7f 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -37,15 +37,15 @@ def url(url) when is_nil(url) or url == "", do: nil
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if disabled?() or not is_url_proxiable?(url) do
+    if disabled?() or not url_proxiable?(url) do
       url
     else
       encode_url(url)
     end
   end
 
-  @spec is_url_proxiable?(String.t()) :: boolean()
-  def is_url_proxiable?(url) do
+  @spec url_proxiable?(String.t()) :: boolean()
+  def url_proxiable?(url) do
     if local?(url) or whitelisted?(url) do
       false
     else

From 2731ea1334c2c91315465659a0874829cb9e1e11 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 17 Jun 2020 13:13:55 -0500
Subject: [PATCH 248/283] Change references from "deleted_urls" to
 "banned_urls" as nothing is handled via media deletions anymore; all actions
 are manual operations by an admin to ban the url

---
 lib/pleroma/application.ex                    |  2 +-
 lib/pleroma/plugs/uploaded_media.ex           | 10 ++++----
 .../media_proxy_cache_controller.ex           |  6 ++---
 lib/pleroma/web/media_proxy/media_proxy.ex    | 20 ++++++++--------
 .../web/media_proxy/media_proxy_controller.ex |  4 ++--
 .../media_proxy_cache_controller_test.exs     | 24 +++++++++----------
 test/web/media_proxy/invalidation_test.exs    | 14 +++++------
 .../media_proxy_controller_test.exs           |  6 ++---
 8 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index adebebc7a..4a21bf138 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -149,7 +149,7 @@ defp cachex_children do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
     ]
   end
 
diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index 2f3fde002..40984cfc0 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -49,7 +49,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
     with uploader <- Keyword.fetch!(config, :uploader),
          proxy_remote = Keyword.get(config, :proxy_remote, false),
          {:ok, get_method} <- uploader.get_file(file),
-         false <- media_is_deleted(conn, get_method) do
+         false <- media_is_banned(conn, get_method) do
       get_media(conn, get_method, proxy_remote, opts)
     else
       _ ->
@@ -61,13 +61,13 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
 
   def call(conn, _opts), do: conn
 
-  defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do
-    MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path)
+  defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
+    MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path)
   end
 
-  defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url)
+  defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
 
-  defp media_is_deleted(_, _), do: false
+  defp media_is_banned(_, _), do: false
 
   defp get_media(conn, {:static_dir, directory}, _, opts) do
     static_opts =
diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
index e3fa0ac28..e2759d59f 100644
--- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
 
   def index(%{assigns: %{user: _}} = conn, params) do
     cursor =
-      :deleted_urls_cache
+      :banned_urls_cache
       |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
       |> :qlc.cursor()
 
@@ -47,7 +47,7 @@ def index(%{assigns: %{user: _}} = conn, params) do
   end
 
   def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
-    MediaProxy.remove_from_deleted_urls(urls)
+    MediaProxy.remove_from_banned_urls(urls)
     render(conn, "index.json", urls: urls)
   end
 
@@ -55,7 +55,7 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _
     MediaProxy.Invalidation.purge(urls)
 
     if ban do
-      MediaProxy.put_in_deleted_urls(urls)
+      MediaProxy.put_in_banned_urls(urls)
     end
 
     render(conn, "index.json", urls: urls)
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 3dccd6b7f..077fabe47 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -10,27 +10,27 @@ defmodule Pleroma.Web.MediaProxy do
 
   @base64_opts [padding: false]
 
-  @spec in_deleted_urls(String.t()) :: boolean()
-  def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1)
+  @spec in_banned_urls(String.t()) :: boolean()
+  def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
 
-  def remove_from_deleted_urls(urls) when is_list(urls) do
-    Cachex.execute!(:deleted_urls_cache, fn cache ->
+  def remove_from_banned_urls(urls) when is_list(urls) do
+    Cachex.execute!(:banned_urls_cache, fn cache ->
       Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
     end)
   end
 
-  def remove_from_deleted_urls(url) when is_binary(url) do
-    Cachex.del(:deleted_urls_cache, url(url))
+  def remove_from_banned_urls(url) when is_binary(url) do
+    Cachex.del(:banned_urls_cache, url(url))
   end
 
-  def put_in_deleted_urls(urls) when is_list(urls) do
-    Cachex.execute!(:deleted_urls_cache, fn cache ->
+  def put_in_banned_urls(urls) when is_list(urls) do
+    Cachex.execute!(:banned_urls_cache, fn cache ->
       Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
     end)
   end
 
-  def put_in_deleted_urls(url) when is_binary(url) do
-    Cachex.put(:deleted_urls_cache, url(url), true)
+  def put_in_banned_urls(url) when is_binary(url) do
+    Cachex.put(:banned_urls_cache, url(url), true)
   end
 
   def url(url) when is_nil(url) or url == "", do: nil
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index ff0158d83..9a64b0ef3 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -14,11 +14,11 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
     with config <- Pleroma.Config.get([:media_proxy], []),
          true <- Keyword.get(config, :enabled, false),
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)},
+         {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
          :ok <- filename_matches(params, conn.request_path, url) do
       ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
     else
-      error when error in [false, {:in_deleted_urls, true}] ->
+      error when error in [false, {:in_banned_urls, true}] ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
index 42a3c0dd8..5ab6cb78a 100644
--- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
+++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
   setup do: clear_config([:media_proxy])
 
   setup do
-    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
   setup do
@@ -34,14 +34,14 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
 
   describe "GET /api/pleroma/admin/media_proxy_caches" do
     test "shows banned MediaProxy URLs", %{conn: conn} do
-      MediaProxy.put_in_deleted_urls([
+      MediaProxy.put_in_banned_urls([
         "http://localhost:4001/media/a688346.jpg",
         "http://localhost:4001/media/fb1f4d.jpg"
       ])
 
-      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg")
-      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg")
-      MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg")
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
 
       response =
         conn
@@ -74,7 +74,7 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
 
   describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
     test "deleted MediaProxy URLs from banned", %{conn: conn} do
-      MediaProxy.put_in_deleted_urls([
+      MediaProxy.put_in_banned_urls([
         "http://localhost:4001/media/a688346.jpg",
         "http://localhost:4001/media/fb1f4d.jpg"
       ])
@@ -88,8 +88,8 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
         |> json_response_and_validate_schema(200)
 
       assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
-      refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg")
-      assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg")
+      refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
+      assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
     end
   end
 
@@ -114,8 +114,8 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do
 
         assert response["urls"] == urls
 
-        refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
-        refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+        refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
+        refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
       end
     end
 
@@ -137,8 +137,8 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c
 
         assert response["urls"] == urls
 
-        assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
-        assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+        assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
+        assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
       end
     end
   end
diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
index bf9af251c..926ae74ca 100644
--- a/test/web/media_proxy/invalidation_test.exs
+++ b/test/web/media_proxy/invalidation_test.exs
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do
   setup do: clear_config([:media_proxy])
 
   setup do
-    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
   describe "Invalidation.Http" do
@@ -23,7 +23,7 @@ test "perform request to clear cache" do
 
       Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
       image_url = "http://example.com/media/example.jpg"
-      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+      Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
 
       mock(fn
         %{
@@ -35,9 +35,9 @@ test "perform request to clear cache" do
       end)
 
       assert capture_log(fn ->
-               assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
                assert Invalidation.purge([image_url]) == {:ok, [image_url]}
-               assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+               assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
              end) =~ "Running cache purge: [\"#{image_url}\"]"
     end
   end
@@ -50,13 +50,13 @@ test "run script to clear cache" do
       Config.put([Invalidation.Script], script_path: "purge-nginx")
 
       image_url = "http://example.com/media/example.jpg"
-      Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+      Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
 
       with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
         assert capture_log(fn ->
-                 assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+                 assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
                  assert Invalidation.purge([image_url]) == {:ok, [image_url]}
-                 assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+                 assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
                end) =~ "Running cache purge: [\"#{image_url}\"]"
       end
     end
diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs
index 72da98a6a..d61cef83b 100644
--- a/test/web/media_proxy/media_proxy_controller_test.exs
+++ b/test/web/media_proxy/media_proxy_controller_test.exs
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
   setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
 
   setup do
-    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
   test "it returns 404 when MediaProxy disabled", %{conn: conn} do
@@ -71,11 +71,11 @@ test "it performs ReverseProxy.call when signature valid", %{conn: conn} do
     end
   end
 
-  test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do
+  test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do
     Config.put([:media_proxy, :enabled], true)
     Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
     url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
-    Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png")
+    Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png")
 
     with_mock Pleroma.ReverseProxy,
       call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do

From 4044f24e2e4935757e038e7f06373ed1c9172560 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 18 Jun 2020 05:02:33 +0300
Subject: [PATCH 249/283] fix test

---
 lib/pleroma/web/media_proxy/invalidation.ex        | 3 ++-
 test/web/media_proxy/invalidations/http_test.exs   | 2 +-
 test/web/media_proxy/invalidations/script_test.exs | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index 6da7eb720..5808861e6 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -26,7 +26,8 @@ def purge(urls) do
 
   defp do_purge(urls) do
     provider = Config.get([:media_proxy, :invalidation, :provider])
-    provider.purge(urls, Config.get(provider))
+    options = Config.get(provider)
+    provider.purge(urls, options)
   end
 
   def prepare_urls(urls) do
diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs
index 9d181dd8b..a1bef5237 100644
--- a/test/web/media_proxy/invalidations/http_test.exs
+++ b/test/web/media_proxy/invalidations/http_test.exs
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
   import Tesla.Mock
 
   setup do
-    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
   test "logs hasn't error message when request is valid" do
diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs
index 8e155b705..51833ab18 100644
--- a/test/web/media_proxy/invalidations/script_test.exs
+++ b/test/web/media_proxy/invalidations/script_test.exs
@@ -5,7 +5,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
   import ExUnit.CaptureLog
 
   setup do
-    on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+    on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
   end
 
   test "it logger error when script not found" do

From c9b5e3fedabd0b6ef3bb9e6108385ffa3857af54 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 18 Jun 2020 05:29:31 +0300
Subject: [PATCH 250/283] revert  'database'  option to rejected keys

---
 lib/pleroma/config/loader.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex
index 76559e70c..0f3ecf1ed 100644
--- a/lib/pleroma/config/loader.ex
+++ b/lib/pleroma/config/loader.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Config.Loader do
     Pleroma.Web.Endpoint,
     :env,
     :configurable_from_database,
+    :database,
     :swarm
   ]
 

From e4c61f1741f32fec3201f7d9a8403bc1bc329710 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 18 Jun 2020 05:45:15 +0300
Subject: [PATCH 251/283] added test

---
 test/fixtures/config/temp.secret.exs | 2 ++
 test/tasks/config_test.exs           | 1 +
 2 files changed, 3 insertions(+)

diff --git a/test/fixtures/config/temp.secret.exs b/test/fixtures/config/temp.secret.exs
index dc950ca30..fa8c7c7e8 100644
--- a/test/fixtures/config/temp.secret.exs
+++ b/test/fixtures/config/temp.secret.exs
@@ -9,3 +9,5 @@
 config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox
 
 config :postgrex, :json_library, Poison
+
+config :pleroma, :database, rum_enabled: true
diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs
index e1bddfebf..99038e544 100644
--- a/test/tasks/config_test.exs
+++ b/test/tasks/config_test.exs
@@ -50,6 +50,7 @@ test "filtered settings are migrated to db" do
       config3 = ConfigDB.get_by_params(%{group: ":quack", key: ":level"})
       refute ConfigDB.get_by_params(%{group: ":pleroma", key: "Pleroma.Repo"})
       refute ConfigDB.get_by_params(%{group: ":postgrex", key: ":json_library"})
+      refute ConfigDB.get_by_params(%{group: ":pleroma", key: ":database"})
 
       assert config1.value == [key: "value", key2: [Repo]]
       assert config2.value == [key: "value2", key2: ["Activity"]]

From 3becdafd335f95d9320d287ecf9a55ea1b1765cd Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 18 Jun 2020 14:32:21 +0300
Subject: [PATCH 252/283] emoji packs pagination

---
 lib/pleroma/emoji/pack.ex                     | 19 ++++++++++++----
 .../pleroma_emoji_pack_operation.ex           | 14 ++++++++++++
 .../controllers/emoji_pack_controller.ex      |  4 ++--
 .../emoji_pack_controller_test.exs            | 22 +++++++++++++++++++
 4 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index 14a5185be..5660c4c9d 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Emoji.Pack do
 
   alias Pleroma.Emoji
 
-  @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
+  @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
          dir <- Path.join(emoji_path(), name),
@@ -120,8 +120,8 @@ def list_remote(url) do
     end
   end
 
-  @spec list_local() :: {:ok, map()}
-  def list_local do
+  @spec list_local(keyword()) :: {:ok, map()}
+  def list_local(opts) do
     with {:ok, results} <- list_packs_dir() do
       packs =
         results
@@ -132,6 +132,17 @@ def list_local do
           end
         end)
         |> Enum.reject(&is_nil/1)
+
+      packs =
+        case opts[:page] do
+          1 ->
+            Enum.take(packs, opts[:page_size])
+
+          _ ->
+            packs
+            |> Enum.take(opts[:page] * opts[:page_size])
+            |> Enum.take(-opts[:page_size])
+        end
         |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
 
       {:ok, packs}
@@ -146,7 +157,7 @@ def get_archive(name) do
     end
   end
 
-  @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
+  @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
   def download(name, url, as) do
     uri = url |> String.trim() |> URI.parse()
 
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
index 567688ff5..0d842382b 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
@@ -33,6 +33,20 @@ def index_operation do
       tags: ["Emoji Packs"],
       summary: "Lists local custom emoji packs",
       operationId: "PleromaAPI.EmojiPackController.index",
+      parameters: [
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of statuses to return"
+        )
+      ],
       responses: %{
         200 => emoji_packs_response()
       }
diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
index d1efdeb5d..5654b3fbe 100644
--- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
@@ -37,13 +37,13 @@ def remote(conn, %{url: url}) do
     end
   end
 
-  def index(conn, _params) do
+  def index(conn, params) do
     emoji_path =
       [:instance, :static_dir]
       |> Pleroma.Config.get!()
       |> Path.join("emoji")
 
-    with {:ok, packs} <- Pack.list_local() do
+    with {:ok, packs} <- Pack.list_local(page: params.page, page_size: params.page_size) do
       json(conn, packs)
     else
       {:error, :create_dir, e} ->
diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
index ee3d281a0..aafca6359 100644
--- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
+++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
@@ -39,6 +39,28 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
     non_shared = resp["test_pack_nonshared"]
     assert non_shared["pack"]["share-files"] == false
     assert non_shared["pack"]["can-download"] == false
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1")
+      |> json_response_and_validate_schema(200)
+
+    [pack1] = Map.keys(resp)
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1&page=2")
+      |> json_response_and_validate_schema(200)
+
+    [pack2] = Map.keys(resp)
+
+    resp =
+      conn
+      |> get("/api/pleroma/emoji/packs?page_size=1&page=3")
+      |> json_response_and_validate_schema(200)
+
+    [pack3] = Map.keys(resp)
+    assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3
   end
 
   describe "GET /api/pleroma/emoji/packs/remote" do

From 4975ed86bcca330373a68c9e6c6798a6b2167b14 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 18 Jun 2020 18:50:03 +0300
Subject: [PATCH 253/283] emoji pagination for pack show action

---
 docs/API/pleroma_api.md                       |  12 ++-
 lib/pleroma/emoji/pack.ex                     |  37 +++++----
 .../pleroma_emoji_pack_operation.ex           |  16 +++-
 .../controllers/emoji_pack_controller.ex      |   4 +-
 .../emoji/test_pack/blank2.png                | Bin 0 -> 95 bytes
 .../instance_static/emoji/test_pack/pack.json |   3 +-
 .../emoji/test_pack_nonshared/nonshared.zip   | Bin 256 -> 548 bytes
 .../emoji/test_pack_nonshared/pack.json       |   2 +-
 .../emoji_pack_controller_test.exs            |  72 +++++++++++++-----
 9 files changed, 104 insertions(+), 42 deletions(-)
 create mode 100644 test/instance_static/emoji/test_pack/blank2.png

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index 70d4755b7..d8d3ba85f 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -450,17 +450,25 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
 ## `GET /api/pleroma/emoji/packs`
+
 ### Lists local custom emoji packs
+
 * Method `GET`
 * Authentication: not required
-* Params: None
+* Params:
+  * `page`: page number for packs (default 1)
+  * `page_size`: page size for packs (default 50)
 * Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents
 
 ## `GET /api/pleroma/emoji/packs/:name`
+
 ### Get pack.json for the pack
+
 * Method `GET`
 * Authentication: not required
-* Params: None
+* Params:
+  * `page`: page number for files (default 1)
+  * `page_size`: page size for files (default 50)
 * Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist
 
 ## `GET /api/pleroma/emoji/packs/:name/archive`
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index 5660c4c9d..c033572c1 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -26,10 +26,27 @@ def create(name) do
     end
   end
 
-  @spec show(String.t()) :: {:ok, t()} | {:error, atom()}
-  def show(name) do
+  defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
+
+  defp paginate(entities, page, page_size) do
+    entities
+    |> Enum.take(page * page_size)
+    |> Enum.take(-page_size)
+  end
+
+  @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
+  def show(opts) do
+    name = opts[:name]
+
     with :ok <- validate_not_empty([name]),
          {:ok, pack} <- load_pack(name) do
+      shortcodes =
+        pack.files
+        |> Map.keys()
+        |> paginate(opts[:page], opts[:page_size])
+
+      pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
+
       {:ok, validate_pack(pack)}
     end
   end
@@ -132,17 +149,7 @@ def list_local(opts) do
           end
         end)
         |> Enum.reject(&is_nil/1)
-
-      packs =
-        case opts[:page] do
-          1 ->
-            Enum.take(packs, opts[:page_size])
-
-          _ ->
-            packs
-            |> Enum.take(opts[:page] * opts[:page_size])
-            |> Enum.take(-opts[:page_size])
-        end
+        |> paginate(opts[:page], opts[:page_size])
         |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
 
       {:ok, packs}
@@ -307,7 +314,9 @@ defp downloadable?(pack) do
     # Otherwise, they'd have to download it from external-src
     pack.pack["share-files"] &&
       Enum.all?(pack.files, fn {_, file} ->
-        File.exists?(Path.join(pack.path, file))
+        pack.path
+        |> Path.join(file)
+        |> File.exists?()
       end)
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
index 0d842382b..e8abe654d 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
@@ -58,7 +58,21 @@ def show_operation do
       tags: ["Emoji Packs"],
       summary: "Show emoji pack",
       operationId: "PleromaAPI.EmojiPackController.show",
-      parameters: [name_param()],
+      parameters: [
+        name_param(),
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of statuses to return"
+        )
+      ],
       responses: %{
         200 => Operation.response("Emoji Pack", "application/json", emoji_pack()),
         400 => Operation.response("Bad Request", "application/json", ApiError),
diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
index 5654b3fbe..078fb88dd 100644
--- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
@@ -60,10 +60,10 @@ def index(conn, params) do
     end
   end
 
-  def show(conn, %{name: name}) do
+  def show(conn, %{name: name, page: page, page_size: page_size}) do
     name = String.trim(name)
 
-    with {:ok, pack} <- Pack.show(name) do
+    with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do
       json(conn, pack)
     else
       {:error, :not_found} ->
diff --git a/test/instance_static/emoji/test_pack/blank2.png b/test/instance_static/emoji/test_pack/blank2.png
new file mode 100644
index 0000000000000000000000000000000000000000..8f50fa02340e7e09e562f86e00b6e4bd6ad1d565
GIT binary patch
literal 95
zcmeAS@N?(olHy`uVBq!ia0vp^4Is=2Bp6=1#-sr$rjj7PU<QV=$!9HqJPA)1$B+uf
q<ORkOtcw#wdYS?axZDnEFfed5Ffe}4nR*bYhQZU-&t;ucLK6U?=oVoB

literal 0
HcmV?d00001

diff --git a/test/instance_static/emoji/test_pack/pack.json b/test/instance_static/emoji/test_pack/pack.json
index 481891b08..5b33fbb32 100644
--- a/test/instance_static/emoji/test_pack/pack.json
+++ b/test/instance_static/emoji/test_pack/pack.json
@@ -1,6 +1,7 @@
 {
     "files": {
-        "blank": "blank.png"
+        "blank": "blank.png",
+        "blank2": "blank2.png"
     },
     "pack": {
         "description": "Test description",
diff --git a/test/instance_static/emoji/test_pack_nonshared/nonshared.zip b/test/instance_static/emoji/test_pack_nonshared/nonshared.zip
index 148446c642ea24b494bc3e25ccd772faaf2f2a13..59bff37f0895f17fecf4285015c5dcd18fcc8b35 100644
GIT binary patch
literal 548
zcmWIWW@Zs#-~hstas2)aP!JEKIT;ifl5!IBvh@n`(nCXd8Q6EphQ<a|ypHn$;?fFk
z21b^zj0_Aw?F<aBc|H_Be>&+=QbNLmuU`r{nJ)1voZ(QBh}(T^38Ut+NecTD*xELo
zOxJC&;q(_jK7s4l6V_uwYr2J9s%A0q?zqF3WnfTXVqj2rerA=xlFOH`o==?{>?F((
z;LXkvAKoY$0ki|;r~sVK<$^ia2*th8K(|H>q<a~eM3@l)jO-Us0K>qNMi7hW;12M{
gZ7e7tU|>n(dYG|91xtW8D;r2J6A*%Q#xsI=0P0(p8vp<R

delta 152
zcmZ3&(!k^x;LXe;!oa}5!EiE;-#<RQQ8WU`iw9y(1{sE=oW#6ry@I^-&=5`r=3TO(
zv8RBzw1S&~k>v$50|Stl=o%x-$Rx*%%Mgjlb&OIvtPtHOIvE%Oyjj_RHZd>)p+AtG
I4dO5W0E3SliU0rr

diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.json b/test/instance_static/emoji/test_pack_nonshared/pack.json
index 93d643a5f..09f6274d1 100644
--- a/test/instance_static/emoji/test_pack_nonshared/pack.json
+++ b/test/instance_static/emoji/test_pack_nonshared/pack.json
@@ -4,7 +4,7 @@
         "homepage": "https://pleroma.social",
         "description": "Test description",
         "fallback-src": "https://nonshared-pack",
-        "fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF",
+        "fallback-src-sha256": "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D",
         "share-files": false
     },
     "files": {
diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
index aafca6359..f6239cae5 100644
--- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
+++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
@@ -31,7 +31,7 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
     shared = resp["test_pack"]
-    assert shared["files"] == %{"blank" => "blank.png"}
+    assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"}
     assert Map.has_key?(shared["pack"], "download-sha256")
     assert shared["pack"]["can-download"]
     assert shared["pack"]["share-files"]
@@ -354,7 +354,7 @@ test "for a pack with a fallback source", ctx do
         Map.put(
           new_data,
           "fallback-src-sha256",
-          "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF"
+          "1967BB4E42BCC34BCC12D57BE7811D3B7BE52F965BCE45C87BD377B9499CE11D"
         )
 
       assert ctx[:admin_conn]
@@ -420,7 +420,7 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -429,7 +429,8 @@ test "don't rewrite old emoji", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank2" => "dir/blank.png"
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -453,7 +454,7 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -462,7 +463,8 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank2" => "dir/blank.png"
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -470,14 +472,15 @@ test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
-               new_shortcode: "blank3",
+               shortcode: "blank3",
+               new_shortcode: "blank4",
                new_filename: "dir_2/blank_3.png",
                force: true
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank3" => "dir_2/blank_3.png"
+               "blank2" => "blank2.png",
+               "blank4" => "dir_2/blank_3.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
@@ -503,7 +506,7 @@ test "add file with not loaded pack", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank2",
+               shortcode: "blank3",
                filename: "dir/blank.png",
                file: %Plug.Upload{
                  filename: "blank.png",
@@ -557,7 +560,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank" => "blank.png",
-               "blank4" => "dir/blank.png"
+               "blank4" => "dir/blank.png",
+               "blank2" => "blank2.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
@@ -571,7 +575,8 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank3" => "dir_2/blank_3.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
 
       refute File.exists?("#{@emoji_path}/test_pack/dir/")
@@ -579,7 +584,10 @@ test "new with shortcode as file with update", %{admin_conn: admin_conn} do
 
       assert admin_conn
              |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(200) == %{"blank" => "blank.png"}
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
 
       refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
 
@@ -603,7 +611,8 @@ test "new with shortcode from url", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "blank_url" => "blank_url.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
 
       assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
@@ -624,15 +633,16 @@ test "new without shortcode", %{admin_conn: admin_conn} do
              })
              |> json_response_and_validate_schema(200) == %{
                "shortcode" => "shortcode.png",
-               "blank" => "blank.png"
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
              }
     end
 
     test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
       assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank2")
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
              |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank2\" does not exist"
+               "error" => "Emoji \"blank3\" does not exist"
              }
     end
 
@@ -640,12 +650,12 @@ test "update non existing emoji", %{admin_conn: admin_conn} do
       assert admin_conn
              |> put_req_header("content-type", "multipart/form-data")
              |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
-               new_shortcode: "blank3",
+               shortcode: "blank3",
+               new_shortcode: "blank4",
                new_filename: "dir_2/blank_3.png"
              })
              |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank2\" does not exist"
+               "error" => "Emoji \"blank3\" does not exist"
              }
     end
 
@@ -768,7 +778,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
   describe "GET /api/pleroma/emoji/packs/:name" do
     test "shows pack.json", %{conn: conn} do
       assert %{
-               "files" => %{"blank" => "blank.png"},
+               "files" => files,
                "pack" => %{
                  "can-download" => true,
                  "description" => "Test description",
@@ -781,6 +791,26 @@ test "shows pack.json", %{conn: conn} do
                conn
                |> get("/api/pleroma/emoji/packs/test_pack")
                |> json_response_and_validate_schema(200)
+
+      assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"}
+
+      assert %{
+               "files" => files
+             } =
+               conn
+               |> get("/api/pleroma/emoji/packs/test_pack?page_size=1")
+               |> json_response_and_validate_schema(200)
+
+      assert files |> Map.keys() |> length() == 1
+
+      assert %{
+               "files" => files
+             } =
+               conn
+               |> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2")
+               |> json_response_and_validate_schema(200)
+
+      assert files |> Map.keys() |> length() == 1
     end
 
     test "non existing pack", %{conn: conn} do

From 3e3f9253e6db17b691c7393ad7a5f89df84348ea Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 19 Jun 2020 10:17:24 +0300
Subject: [PATCH 254/283] adding overall count for packs and files

---
 docs/API/pleroma_api.md                       | 22 ++++++++++++--
 lib/pleroma/emoji/pack.ex                     | 20 +++++++++----
 .../controllers/emoji_pack_controller.ex      |  4 +--
 .../emoji_pack_controller_test.exs            | 30 ++++++++++++-------
 4 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index d8d3ba85f..e5bc29eb2 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -458,7 +458,17 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params:
   * `page`: page number for packs (default 1)
   * `page_size`: page size for packs (default 50)
-* Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents
+* Response: `packs` key with JSON hashmap of pack name to pack contents and `count` key for count of packs.
+
+```json
+{
+  "packs": {
+    "pack_name": {...}, // pack contents
+    ...
+  },
+  "count": 0 // packs count
+}
+```
 
 ## `GET /api/pleroma/emoji/packs/:name`
 
@@ -469,7 +479,15 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params:
   * `page`: page number for files (default 1)
   * `page_size`: page size for files (default 50)
-* Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist
+* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
+
+```json
+{
+  "files": {...},
+  "files_count": 0, // emoji count in pack
+  "pack": {...}
+}
+```
 
 ## `GET /api/pleroma/emoji/packs/:name/archive`
 ### Requests a local pack archive from the instance
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index c033572c1..2dca21c93 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -1,6 +1,7 @@
 defmodule Pleroma.Emoji.Pack do
-  @derive {Jason.Encoder, only: [:files, :pack]}
+  @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
   defstruct files: %{},
+            files_count: 0,
             pack_file: nil,
             path: nil,
             pack: %{},
@@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do
 
   @type t() :: %__MODULE__{
           files: %{String.t() => Path.t()},
+          files_count: non_neg_integer(),
           pack_file: Path.t(),
           path: Path.t(),
           pack: map(),
@@ -137,10 +139,10 @@ def list_remote(url) do
     end
   end
 
-  @spec list_local(keyword()) :: {:ok, map()}
+  @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
   def list_local(opts) do
     with {:ok, results} <- list_packs_dir() do
-      packs =
+      all_packs =
         results
         |> Enum.map(fn name ->
           case load_pack(name) do
@@ -149,10 +151,13 @@ def list_local(opts) do
           end
         end)
         |> Enum.reject(&is_nil/1)
+
+      packs =
+        all_packs
         |> paginate(opts[:page], opts[:page_size])
         |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
 
-      {:ok, packs}
+      {:ok, packs, length(all_packs)}
     end
   end
 
@@ -215,7 +220,12 @@ def load_pack(name) do
         |> Map.put(:path, Path.dirname(pack_file))
         |> Map.put(:name, name)
 
-      {:ok, pack}
+      files_count =
+        pack.files
+        |> Map.keys()
+        |> length()
+
+      {:ok, Map.put(pack, :files_count, files_count)}
     else
       {:error, :not_found}
     end
diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
index 078fb88dd..33ecd1f70 100644
--- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
@@ -43,8 +43,8 @@ def index(conn, params) do
       |> Pleroma.Config.get!()
       |> Path.join("emoji")
 
-    with {:ok, packs} <- Pack.list_local(page: params.page, page_size: params.page_size) do
-      json(conn, packs)
+    with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do
+      json(conn, %{packs: packs, count: count})
     else
       {:error, :create_dir, e} ->
         conn
diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
index f6239cae5..91312c832 100644
--- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
+++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
@@ -30,13 +30,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackControllerTest do
   test "GET /api/pleroma/emoji/packs", %{conn: conn} do
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    shared = resp["test_pack"]
+    assert resp["count"] == 3
+    shared = resp["packs"]["test_pack"]
     assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"}
     assert Map.has_key?(shared["pack"], "download-sha256")
     assert shared["pack"]["can-download"]
     assert shared["pack"]["share-files"]
 
-    non_shared = resp["test_pack_nonshared"]
+    non_shared = resp["packs"]["test_pack_nonshared"]
     assert non_shared["pack"]["share-files"] == false
     assert non_shared["pack"]["can-download"] == false
 
@@ -45,21 +46,24 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
       |> get("/api/pleroma/emoji/packs?page_size=1")
       |> json_response_and_validate_schema(200)
 
-    [pack1] = Map.keys(resp)
+    assert resp["count"] == 3
+    [pack1] = Map.keys(resp["packs"])
 
     resp =
       conn
       |> get("/api/pleroma/emoji/packs?page_size=1&page=2")
       |> json_response_and_validate_schema(200)
 
-    [pack2] = Map.keys(resp)
+    assert resp["count"] == 3
+    [pack2] = Map.keys(resp["packs"])
 
     resp =
       conn
       |> get("/api/pleroma/emoji/packs?page_size=1&page=3")
       |> json_response_and_validate_schema(200)
 
-    [pack3] = Map.keys(resp)
+    assert resp["count"] == 3
+    [pack3] = Map.keys(resp["packs"])
     assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3
   end
 
@@ -683,7 +687,8 @@ test "creating and deleting a pack", %{admin_conn: admin_conn} do
 
       assert Jason.decode!(File.read!("#{@emoji_path}/test_created/pack.json")) == %{
                "pack" => %{},
-               "files" => %{}
+               "files" => %{},
+               "files_count" => 0
              }
 
       assert admin_conn
@@ -741,14 +746,14 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    refute Map.has_key?(resp, "test_pack_for_import")
+    refute Map.has_key?(resp["packs"], "test_pack_for_import")
 
     assert admin_conn
            |> get("/api/pleroma/emoji/packs/import")
            |> json_response_and_validate_schema(200) == ["test_pack_for_import"]
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
-    assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
+    assert resp["packs"]["test_pack_for_import"]["files"] == %{"blank" => "blank.png"}
 
     File.rm!("#{@emoji_path}/test_pack_for_import/pack.json")
     refute File.exists?("#{@emoji_path}/test_pack_for_import/pack.json")
@@ -768,7 +773,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
 
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
-    assert resp["test_pack_for_import"]["files"] == %{
+    assert resp["packs"]["test_pack_for_import"]["files"] == %{
              "blank" => "blank.png",
              "blank2" => "blank.png",
              "foo" => "blank.png"
@@ -779,6 +784,7 @@ test "filesystem import", %{admin_conn: admin_conn, conn: conn} do
     test "shows pack.json", %{conn: conn} do
       assert %{
                "files" => files,
+               "files_count" => 2,
                "pack" => %{
                  "can-download" => true,
                  "description" => "Test description",
@@ -795,7 +801,8 @@ test "shows pack.json", %{conn: conn} do
       assert files == %{"blank" => "blank.png", "blank2" => "blank2.png"}
 
       assert %{
-               "files" => files
+               "files" => files,
+               "files_count" => 2
              } =
                conn
                |> get("/api/pleroma/emoji/packs/test_pack?page_size=1")
@@ -804,7 +811,8 @@ test "shows pack.json", %{conn: conn} do
       assert files |> Map.keys() |> length() == 1
 
       assert %{
-               "files" => files
+               "files" => files,
+               "files_count" => 2
              } =
                conn
                |> get("/api/pleroma/emoji/packs/test_pack?page_size=1&page=2")

From 0c739b423aad4cc6baa3a59308200ca5a5060716 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 19 Jun 2020 12:31:55 +0300
Subject: [PATCH 255/283] changelog entry

---
 CHANGELOG.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ee13904f..ee657de13 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
+
 - Chats: Added support for federated chats. For details, see the docs.
 - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
 - Instance: Add `background_image` to configuration and `/api/v1/instance`
@@ -34,6 +35,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Notifications: Added `follow_request` notification type.
 - Added `:reject_deletes` group to SimplePolicy
 - MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
+- Support pagination in emoji packs API (for packs and for files in pack)
+
 <details>
   <summary>API Changes</summary>
 - Mastodon API: Extended `/api/v1/instance`.

From 02ca8a363f738ece7b605940690f6a538f6c2fa8 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 19 Jun 2020 14:46:38 +0300
Subject: [PATCH 256/283] default page size for files

---
 docs/API/pleroma_api.md                                         | 2 +-
 .../web/api_spec/operations/pleroma_emoji_pack_operation.ex     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index e5bc29eb2..b7eee5192 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -478,7 +478,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Authentication: not required
 * Params:
   * `page`: page number for files (default 1)
-  * `page_size`: page size for files (default 50)
+  * `page_size`: page size for files (default 30)
 * Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
 
 ```json
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
index e8abe654d..da7cc5154 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
@@ -69,7 +69,7 @@ def show_operation do
         Operation.parameter(
           :page_size,
           :query,
-          %Schema{type: :integer, default: 50},
+          %Schema{type: :integer, default: 30},
           "Number of statuses to return"
         )
       ],

From 5237a2df9f123f661de30a53193b7d9fec69ecae Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 19 Jun 2020 16:14:06 +0300
Subject: [PATCH 257/283] [#1873] Fixes missing :offset pagination param
 support. Added pagination support for hashtags search.

---
 lib/pleroma/pagination.ex                     |  6 ++++
 lib/pleroma/web/api_spec/helpers.ex           |  6 ++++
 .../controllers/search_controller.ex          | 31 ++++++++++++-------
 .../controllers/search_controller_test.exs    | 16 ++++++++++
 .../controllers/status_controller_test.exs    |  2 +-
 5 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex
index 1b99e44f9..9a3795769 100644
--- a/lib/pleroma/pagination.ex
+++ b/lib/pleroma/pagination.ex
@@ -64,6 +64,12 @@ def fetch_paginated(query, params, :offset, table_binding) do
   @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()]
   def paginate(query, options, method \\ :keyset, table_binding \\ nil)
 
+  def paginate(list, options, _method, _table_binding) when is_list(list) do
+    offset = options[:offset] || 0
+    limit = options[:limit] || 0
+    Enum.slice(list, offset, limit)
+  end
+
   def paginate(query, options, :keyset, table_binding) do
     query
     |> restrict(:min_id, options, table_binding)
diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex
index a9cfe0fed..a258e8421 100644
--- a/lib/pleroma/web/api_spec/helpers.ex
+++ b/lib/pleroma/web/api_spec/helpers.ex
@@ -39,6 +39,12 @@ def pagination_params do
         :string,
         "Return the newest items newer than this ID"
       ),
+      Operation.parameter(
+        :offset,
+        :query,
+        %Schema{type: :integer, default: 0},
+        "Return items past this number of items"
+      ),
       Operation.parameter(
         :limit,
         :query,
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 3be0ca095..e50980122 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -107,21 +107,21 @@ defp resource_search(_, "statuses", query, options) do
     )
   end
 
-  defp resource_search(:v2, "hashtags", query, _options) do
+  defp resource_search(:v2, "hashtags", query, options) do
     tags_path = Web.base_url() <> "/tag/"
 
     query
-    |> prepare_tags()
+    |> prepare_tags(options)
     |> Enum.map(fn tag ->
       %{name: tag, url: tags_path <> tag}
     end)
   end
 
-  defp resource_search(:v1, "hashtags", query, _options) do
-    prepare_tags(query)
+  defp resource_search(:v1, "hashtags", query, options) do
+    prepare_tags(query, options)
   end
 
-  defp prepare_tags(query, add_joined_tag \\ true) do
+  defp prepare_tags(query, options) do
     tags =
       query
       |> preprocess_uri_query()
@@ -139,13 +139,20 @@ defp prepare_tags(query, add_joined_tag \\ true) do
 
     tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
 
-    if Enum.empty?(explicit_tags) && add_joined_tag do
-      tags
-      |> Kernel.++([joined_tag(tags)])
-      |> Enum.uniq_by(&String.downcase/1)
-    else
-      tags
-    end
+    tags =
+      if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
+        add_joined_tag(tags)
+      else
+        tags
+      end
+
+    Pleroma.Pagination.paginate(tags, options)
+  end
+
+  defp add_joined_tag(tags) do
+    tags
+    |> Kernel.++([joined_tag(tags)])
+    |> Enum.uniq_by(&String.downcase/1)
   end
 
   # If `query` is a URI, returns last component of its path, otherwise returns `query`
diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs
index c605957b1..826f37fbc 100644
--- a/test/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/web/mastodon_api/controllers/search_controller_test.exs
@@ -151,6 +151,22 @@ test "constructs hashtags from search query", %{conn: conn} do
              ]
     end
 
+    test "supports pagination of hashtags search results", %{conn: conn} do
+      results =
+        conn
+        |> get(
+          "/api/v2/search?#{
+            URI.encode_query(%{q: "#some #text #with #hashtags", limit: 2, offset: 1})
+          }"
+        )
+        |> json_response_and_validate_schema(200)
+
+      assert results["hashtags"] == [
+               %{"name" => "text", "url" => "#{Web.base_url()}/tag/text"},
+               %{"name" => "with", "url" => "#{Web.base_url()}/tag/with"}
+             ]
+    end
+
     test "excludes a blocked users from search results", %{conn: conn} do
       user = insert(:user)
       user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"})
diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs
index 648e6f2ce..a98e939e8 100644
--- a/test/web/mastodon_api/controllers/status_controller_test.exs
+++ b/test/web/mastodon_api/controllers/status_controller_test.exs
@@ -1561,7 +1561,7 @@ test "favorites paginate correctly" do
 
     # Using the header for pagination works correctly
     [next, _] = get_resp_header(result, "link") |> hd() |> String.split(", ")
-    [_, max_id] = Regex.run(~r/max_id=(.*)>;/, next)
+    [_, max_id] = Regex.run(~r/max_id=([^&]+)/, next)
 
     assert max_id == third_favorite.id
 

From d54b0432eae74a830a0294cf48f23933a16382aa Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 19 Jun 2020 15:49:34 +0200
Subject: [PATCH 258/283] README: Add some troubleshooting info for compilation
 issues.

---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 7fc1fd381..6ca3118fb 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,16 @@ Currently Pleroma is not packaged by any OS/Distros, but if you want to package
 ### Docker
 While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.
 
+### Compilation Troubleshooting
+If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things:
+
+- `mix deps.clean --all`
+- `mix local.rebar`
+- `mix local.hex`
+- `rm -r _build`
+
+If you are not developing Pleroma, it is better to use the OTP release, which comes with everything precompiled.
+
 ## Documentation
 - Latest Released revision: <https://docs.pleroma.social>
 - Latest Git revision: <https://docs-develop.pleroma.social>

From b63646169dbed68814bdb867c4b8b3c88a3d2360 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Fri, 19 Jun 2020 21:18:07 +0200
Subject: [PATCH 259/283] Add support for bot field in update_credentials

---
 CHANGELOG.md                                  |  1 +
 lib/pleroma/user.ex                           |  1 +
 .../controllers/account_controller.ex         |  3 +
 .../update_credentials_test.exs               | 67 +++++++++++++++++++
 4 files changed, 72 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ee13904f..8a8798e8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
 - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
 - Mastodon API: Add support for filtering replies in public and home timelines
+- Mastodon API: Support for `bot` field in `/api/v1/accounts/update_credentials`
 - Admin API: endpoints for create/update/delete OAuth Apps.
 - Admin API: endpoint for status view.
 - OTP: Add command to reload emoji packs
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index f0ccc7c79..ae4f96aac 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -465,6 +465,7 @@ def update_changeset(struct, params \\ %{}) do
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
+    |> validate_inclusion(:actor_type, ["Person", "Service"])
     |> put_fields()
     |> put_emoji()
     |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index c38c2b895..adbbac624 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -177,6 +177,9 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
       |> Maps.put_if_present(:pleroma_settings_store, params[:pleroma_settings_store])
       |> Maps.put_if_present(:default_scope, params[:default_scope])
       |> Maps.put_if_present(:default_scope, params["source"]["privacy"])
+      |> Maps.put_if_present(:actor_type, params[:bot], fn bot ->
+        if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
+      end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
 
     changeset = User.update_changeset(user, user_params)
diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
index 76e6d603a..f67d294ba 100644
--- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
+++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs
@@ -400,4 +400,71 @@ test "update fields when invalid request", %{conn: conn} do
                |> json_response_and_validate_schema(403)
     end
   end
+
+  describe "Mark account as bot" do
+    setup do: oauth_access(["write:accounts"])
+    setup :request_content_type
+
+    test "changing actor_type to Service makes account a bot", %{conn: conn} do
+      account =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Service"})
+        |> json_response_and_validate_schema(200)
+
+      assert account["bot"]
+      assert account["source"]["pleroma"]["actor_type"] == "Service"
+    end
+
+    test "changing actor_type to Person makes account a human", %{conn: conn} do
+      account =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Person"})
+        |> json_response_and_validate_schema(200)
+
+      refute account["bot"]
+      assert account["source"]["pleroma"]["actor_type"] == "Person"
+    end
+
+    test "changing actor_type to Application causes error", %{conn: conn} do
+      response =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{actor_type: "Application"})
+        |> json_response_and_validate_schema(403)
+
+      assert %{"error" => "Invalid request"} == response
+    end
+
+    test "changing bot field to true changes actor_type to Service", %{conn: conn} do
+      account =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{bot: "true"})
+        |> json_response_and_validate_schema(200)
+
+      assert account["bot"]
+      assert account["source"]["pleroma"]["actor_type"] == "Service"
+    end
+
+    test "changing bot field to false changes actor_type to Person", %{conn: conn} do
+      account =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{bot: "false"})
+        |> json_response_and_validate_schema(200)
+
+      refute account["bot"]
+      assert account["source"]["pleroma"]["actor_type"] == "Person"
+    end
+
+    test "actor_type field has a higher priority than bot", %{conn: conn} do
+      account =
+        conn
+        |> patch("/api/v1/accounts/update_credentials", %{
+          actor_type: "Person",
+          bot: "true"
+        })
+        |> json_response_and_validate_schema(200)
+
+      refute account["bot"]
+      assert account["source"]["pleroma"]["actor_type"] == "Person"
+    end
+  end
 end

From ac0344dd24d520ab61e835b9caea97529f4c1dad Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Fri, 19 Jun 2020 21:19:00 +0200
Subject: [PATCH 260/283] Only accounts with Service actor_type are considered
 as bots

---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 68beb69b8..6c40b8ccd 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -179,7 +179,7 @@ defp do_render("show.json", %{user: user} = opts) do
         0
       end
 
-    bot = user.actor_type in ["Application", "Service"]
+    bot = user.actor_type == "Service"
 
     emojis =
       Enum.map(user.emoji, fn {shortcode, raw_url} ->

From 3d4cfc9c5f3969e08c32781385c86f310eba70a2 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 18 Jun 2020 19:32:03 +0200
Subject: [PATCH 261/283] Stop filling conversation field on incoming objects
 (legacy, unused)

conversation field is still set for outgoing federation for compatibility.
---
 .../web/activity_pub/object_validators/note_validator.ex    | 1 -
 lib/pleroma/web/activity_pub/transmogrifier.ex              | 6 +++---
 test/web/activity_pub/transmogrifier_test.exs               | 3 ---
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index a10728ac6..56b93dde8 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -41,7 +41,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:announcements, {:array, :string}, default: [])
 
     # see if needed
-    field(:conversation, :string)
     field(:context_id, :string)
   end
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 851f474b8..1c60ef8f5 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -172,8 +172,8 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
         object
         |> Map.put("inReplyTo", replied_object.data["id"])
         |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
-        |> Map.put("conversation", replied_object.data["context"] || object["conversation"])
         |> Map.put("context", replied_object.data["context"] || object["conversation"])
+        |> Map.drop(["conversation"])
       else
         e ->
           Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
@@ -207,7 +207,7 @@ def fix_context(object) do
 
     object
     |> Map.put("context", context)
-    |> Map.put("conversation", context)
+    |> Map.drop(["conversation"])
   end
 
   def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do
@@ -458,7 +458,7 @@ def handle_incoming(
         to: data["to"],
         object: object,
         actor: user,
-        context: object["conversation"],
+        context: object["context"],
         local: false,
         published: data["published"],
         additional:
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 94d8552e8..47d6e843a 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -1571,9 +1571,6 @@ test "returns modified object when allowed incoming reply", %{data: data} do
 
       assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
 
-      assert modified_object["conversation"] ==
-               "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26"
-
       assert modified_object["context"] ==
                "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26"
     end

From 1a704e1f1e0acb73cbfb49acc4f614dd01799c46 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 20 Jun 2020 10:56:28 +0300
Subject: [PATCH 262/283] fix for packs pagination

---
 lib/pleroma/emoji/pack.ex                     |  6 +++---
 .../emoji_pack_controller_test.exs            | 20 ++++++++++++++++---
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index 2dca21c93..787ff8141 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -32,8 +32,8 @@ defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
 
   defp paginate(entities, page, page_size) do
     entities
-    |> Enum.take(page * page_size)
-    |> Enum.take(-page_size)
+    |> Enum.chunk_every(page_size)
+    |> Enum.at(page - 1)
   end
 
   @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
@@ -470,7 +470,7 @@ defp list_packs_dir do
     # with the API so it should be sufficient
     with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
          {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
-      {:ok, results}
+      {:ok, Enum.sort(results)}
     else
       {:create_dir, {:error, e}} -> {:error, :create_dir, e}
       {:ls, {:error, e}} -> {:error, :ls, e}
diff --git a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
index 91312c832..df58a5eb6 100644
--- a/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
+++ b/test/web/pleroma_api/controllers/emoji_pack_controller_test.exs
@@ -31,6 +31,11 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
     resp = conn |> get("/api/pleroma/emoji/packs") |> json_response_and_validate_schema(200)
 
     assert resp["count"] == 3
+
+    assert resp["packs"]
+           |> Map.keys()
+           |> length() == 3
+
     shared = resp["packs"]["test_pack"]
     assert shared["files"] == %{"blank" => "blank.png", "blank2" => "blank2.png"}
     assert Map.has_key?(shared["pack"], "download-sha256")
@@ -47,7 +52,12 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
       |> json_response_and_validate_schema(200)
 
     assert resp["count"] == 3
-    [pack1] = Map.keys(resp["packs"])
+
+    packs = Map.keys(resp["packs"])
+
+    assert length(packs) == 1
+
+    [pack1] = packs
 
     resp =
       conn
@@ -55,7 +65,9 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
       |> json_response_and_validate_schema(200)
 
     assert resp["count"] == 3
-    [pack2] = Map.keys(resp["packs"])
+    packs = Map.keys(resp["packs"])
+    assert length(packs) == 1
+    [pack2] = packs
 
     resp =
       conn
@@ -63,7 +75,9 @@ test "GET /api/pleroma/emoji/packs", %{conn: conn} do
       |> json_response_and_validate_schema(200)
 
     assert resp["count"] == 3
-    [pack3] = Map.keys(resp["packs"])
+    packs = Map.keys(resp["packs"])
+    assert length(packs) == 1
+    [pack3] = packs
     assert [pack1, pack2, pack3] |> Enum.uniq() |> length() == 3
   end
 

From 4cb7b1ebc6b255faae635f6138bf90264e84e1fb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 20 Jun 2020 09:34:34 +0000
Subject: [PATCH 263/283] Apply suggestion to lib/mix/tasks/pleroma/config.ex

---
 lib/mix/tasks/pleroma/config.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 65691f9c1..d5129d410 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -52,7 +52,7 @@ def migrate_to_db(file_path \\ nil) do
 
   defp do_migrate_to_db(config_file) do
     if File.exists?(config_file) do
-      shell_info("Running migrate settings from file: #{Path.expand(config_file)}")
+      shell_info("Migrating settings from file: #{Path.expand(config_file)}")
       Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
       Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
 

From 15ba5392584a2d4e8129a99e825f5025e57e6ebd Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 20 Jun 2020 11:39:06 +0200
Subject: [PATCH 264/283] =?UTF-8?q?cheatsheet.md:=20no=5Fattachment=5Flink?=
 =?UTF-8?q?s=20=E2=86=92=20attachment=5Flinks?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/configuration/cheatsheet.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 6ebdab546..7e5f1cd29 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -60,7 +60,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
-* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses.
+* `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `welcome_message`: A message that will be send to a newly registered users as a direct message.
 * `welcome_user_nickname`: The nickname of the local user that sends the welcome message.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).

From 0e789bc55fed24fd913d6bf1a5c6be135320b0c9 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 20 Jun 2020 09:39:50 +0000
Subject: [PATCH 265/283] Apply suggestion to
 lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

---
 .../web/api_spec/operations/pleroma_emoji_pack_operation.ex     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
index da7cc5154..caa849721 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
@@ -44,7 +44,7 @@ def index_operation do
           :page_size,
           :query,
           %Schema{type: :integer, default: 50},
-          "Number of statuses to return"
+          "Number of emoji packs to return"
         )
       ],
       responses: %{

From c5863438ba9079a01a832fe48e203907fe5b37cd Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 20 Jun 2020 13:53:57 +0300
Subject: [PATCH 266/283] proper error codes for error in adminFE

---
 docs/API/admin_api.md                         | 54 ++++++++++---------
 .../controllers/admin_api_controller.ex       | 29 +++++-----
 .../controllers/fallback_controller.ex        |  6 +++
 .../controllers/admin_api_controller_test.exs |  4 +-
 4 files changed, 50 insertions(+), 43 deletions(-)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index c7f56cf5f..b6fb43dcb 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -488,35 +488,39 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ### Change the user's email, password, display and settings-related fields
 
-- Params:
-  - `email`
-  - `password`
-  - `name`
-  - `bio`
-  - `avatar`
-  - `locked`
-  - `no_rich_text`
-  - `default_scope`
-  - `banner`
-  - `hide_follows`
-  - `hide_followers`
-  - `hide_followers_count`
-  - `hide_follows_count`
-  - `hide_favorites`
-  - `allow_following_move`
-  - `background`
-  - `show_role`
-  - `skip_thread_containment`
-  - `fields`
-  - `discoverable`
-  - `actor_type`
+* Params:
+  * `email`
+  * `password`
+  * `name`
+  * `bio`
+  * `avatar`
+  * `locked`
+  * `no_rich_text`
+  * `default_scope`
+  * `banner`
+  * `hide_follows`
+  * `hide_followers`
+  * `hide_followers_count`
+  * `hide_follows_count`
+  * `hide_favorites`
+  * `allow_following_move`
+  * `background`
+  * `show_role`
+  * `skip_thread_containment`
+  * `fields`
+  * `discoverable`
+  * `actor_type`
 
-- Response:
+* Responses:
+
+Status: 200
 
 ```json
 {"status": "success"}
 ```
 
+Status: 400
+
 ```json
 {"errors":
   {"actor_type": "is invalid"},
@@ -525,8 +529,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
  }
 ```
 
+Status: 404
+
 ```json
-{"error": "Unable to update user."}
+{"error": "Not found"}
 ```
 
 ## `GET /api/pleroma/admin/reports`
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index 5cbf0dd4f..db2413dfe 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -111,8 +111,7 @@ def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames})
       action: "delete"
     })
 
-    conn
-    |> json(nicknames)
+    json(conn, nicknames)
   end
 
   def user_follow(%{assigns: %{user: admin}} = conn, %{
@@ -131,8 +130,7 @@ def user_follow(%{assigns: %{user: admin}} = conn, %{
       })
     end
 
-    conn
-    |> json("ok")
+    json(conn, "ok")
   end
 
   def user_unfollow(%{assigns: %{user: admin}} = conn, %{
@@ -151,8 +149,7 @@ def user_unfollow(%{assigns: %{user: admin}} = conn, %{
       })
     end
 
-    conn
-    |> json("ok")
+    json(conn, "ok")
   end
 
   def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
@@ -191,8 +188,7 @@ def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
           action: "create"
         })
 
-        conn
-        |> json(res)
+        json(conn, res)
 
       {:error, id, changeset, _} ->
         res =
@@ -363,8 +359,8 @@ defp maybe_parse_filters(filters) do
     filters
     |> String.split(",")
     |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Enum.map(&String.to_atom(&1))
-    |> Enum.into(%{}, &{&1, true})
+    |> Enum.map(&String.to_atom/1)
+    |> Map.new(&{&1, true})
   end
 
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
@@ -568,10 +564,10 @@ def update_user_credentials(
       {:error, changeset} ->
         errors = Map.new(changeset.errors, fn {key, {error, _}} -> {key, error} end)
 
-        json(conn, %{errors: errors})
+        {:errors, errors}
 
       _ ->
-        json(conn, %{error: "Unable to update user."})
+        {:error, :not_found}
     end
   end
 
@@ -616,7 +612,7 @@ defp configurable_from_database do
   def reload_emoji(conn, _params) do
     Pleroma.Emoji.reload()
 
-    conn |> json("ok")
+    json(conn, "ok")
   end
 
   def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
@@ -630,7 +626,7 @@ def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}
       action: "confirm_email"
     })
 
-    conn |> json("")
+    json(conn, "")
   end
 
   def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
@@ -644,14 +640,13 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" =
       action: "resend_confirmation_email"
     })
 
-    conn |> json("")
+    json(conn, "")
   end
 
   def stats(conn, _) do
     count = Stats.get_status_visibility_count()
 
-    conn
-    |> json(%{"status_visibility" => count})
+    json(conn, %{"status_visibility" => count})
   end
 
   defp page_params(params) do
diff --git a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex
index 82965936d..34d90db07 100644
--- a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex
@@ -17,6 +17,12 @@ def call(conn, {:error, reason}) do
     |> json(%{error: reason})
   end
 
+  def call(conn, {:errors, errors}) do
+    conn
+    |> put_status(:bad_request)
+    |> json(%{errors: errors})
+  end
+
   def call(conn, {:param_cast, _}) do
     conn
     |> put_status(:bad_request)
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index e3d3ccb8d..3a3eb822d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -1599,14 +1599,14 @@ test "changes actor type from permitted list", %{conn: conn, user: user} do
       assert patch(conn, "/api/pleroma/admin/users/#{user.nickname}/credentials", %{
                "actor_type" => "Application"
              })
-             |> json_response(200) == %{"errors" => %{"actor_type" => "is invalid"}}
+             |> json_response(400) == %{"errors" => %{"actor_type" => "is invalid"}}
     end
 
     test "update non existing user", %{conn: conn} do
       assert patch(conn, "/api/pleroma/admin/users/non-existing/credentials", %{
                "password" => "new_password"
              })
-             |> json_response(200) == %{"error" => "Unable to update user."}
+             |> json_response(404) == %{"error" => "Not found"}
     end
   end
 

From b5f13af7ba66924f6aed448bd519f6becc269922 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 20 Jun 2020 10:59:08 +0000
Subject: [PATCH 267/283] Apply suggestion to
 lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

---
 .../web/api_spec/operations/pleroma_emoji_pack_operation.ex     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
index caa849721..b2b4f8713 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
@@ -70,7 +70,7 @@ def show_operation do
           :page_size,
           :query,
           %Schema{type: :integer, default: 30},
-          "Number of statuses to return"
+          "Number of emoji to return"
         )
       ],
       responses: %{

From 35e9282ffdafd8a04d1c09ec5eff3f176bb389de Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 22 Jun 2020 10:35:11 +0200
Subject: [PATCH 268/283] HellthreadPolicy: Restrict to Notes and Articles.

---
 .../web/activity_pub/mrf/hellthread_policy.ex  |  7 +++++--
 .../mrf/hellthread_policy_test.exs             | 18 +++++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
index 1764bc789..f6b2c4415 100644
--- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
@@ -13,8 +13,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
 
   defp delist_message(message, threshold) when threshold > 0 do
     follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
+    to = message["to"] || []
+    cc = message["cc"] || []
 
-    follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection)
+    follower_collection? = Enum.member?(to ++ cc, follower_collection)
 
     message =
       case get_recipient_count(message) do
@@ -71,7 +73,8 @@ defp get_recipient_count(message) do
   end
 
   @impl true
-  def filter(%{"type" => "Create"} = message) do
+  def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message)
+      when object_type in ~w{Note Article} do
     reject_threshold =
       Pleroma.Config.get(
         [:mrf_hellthread, :reject_threshold],
diff --git a/test/web/activity_pub/mrf/hellthread_policy_test.exs b/test/web/activity_pub/mrf/hellthread_policy_test.exs
index 95ef0b168..6e9daa7f9 100644
--- a/test/web/activity_pub/mrf/hellthread_policy_test.exs
+++ b/test/web/activity_pub/mrf/hellthread_policy_test.exs
@@ -8,6 +8,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do
 
   import Pleroma.Web.ActivityPub.MRF.HellthreadPolicy
 
+  alias Pleroma.Web.CommonAPI
+
   setup do
     user = insert(:user)
 
@@ -20,7 +22,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do
         "https://instance.tld/users/user1",
         "https://instance.tld/users/user2",
         "https://instance.tld/users/user3"
-      ]
+      ],
+      "object" => %{
+        "type" => "Note"
+      }
     }
 
     [user: user, message: message]
@@ -28,6 +33,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicyTest do
 
   setup do: clear_config(:mrf_hellthread)
 
+  test "doesn't die on chat messages" do
+    Pleroma.Config.put([:mrf_hellthread], %{delist_threshold: 2, reject_threshold: 0})
+
+    user = insert(:user)
+    other_user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post_chat_message(user, other_user, "moin")
+
+    assert {:ok, _} = filter(activity.data)
+  end
+
   describe "reject" do
     test "rejects the message if the recipient count is above reject_threshold", %{
       message: message

From 9f7ee5dfa283f8db9a5fcb006630674263425ac8 Mon Sep 17 00:00:00 2001
From: Ilja <domainepublic@spectraltheorem.be>
Date: Mon, 22 Jun 2020 11:41:22 +0200
Subject: [PATCH 269/283] Add include for the "Further reading" section

* I added an include and use this include for the installation guides that already had this section
* I added the "Further reading" section as well as te "Questions" section to the English guides that didn't have it yet
* I added a first point "How Federation Works/Why is my Federated Timeline empty?" to link to lains blogpost about this because we still get this question a lot in the #pleroma support channel
* I reordered the list a bit
---
 docs/installation/alpine_linux_en.md      | 5 +----
 docs/installation/arch_linux_en.md        | 5 +----
 docs/installation/debian_based_en.md      | 5 +----
 docs/installation/debian_based_jp.md      | 5 +----
 docs/installation/further_reading.include | 5 +++++
 docs/installation/gentoo_en.md            | 5 +----
 docs/installation/netbsd_en.md            | 8 ++++++++
 docs/installation/openbsd_en.md           | 8 ++++++++
 docs/installation/otp_en.md               | 5 +----
 9 files changed, 27 insertions(+), 24 deletions(-)
 create mode 100644 docs/installation/further_reading.include

diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md
index 2a9b8f6ff..c726d559f 100644
--- a/docs/installation/alpine_linux_en.md
+++ b/docs/installation/alpine_linux_en.md
@@ -225,10 +225,7 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## Questions
 
diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md
index 8370986ad..bf9cfb488 100644
--- a/docs/installation/arch_linux_en.md
+++ b/docs/installation/arch_linux_en.md
@@ -200,10 +200,7 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## Questions
 
diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md
index 2c20d521a..8ae5044b5 100644
--- a/docs/installation/debian_based_en.md
+++ b/docs/installation/debian_based_en.md
@@ -186,10 +186,7 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## Questions
 
diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md
index 1e5a9be91..42e91cda7 100644
--- a/docs/installation/debian_based_jp.md
+++ b/docs/installation/debian_based_jp.md
@@ -175,10 +175,7 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### その他の設定とカスタマイズ
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## 質問ある？
 
diff --git a/docs/installation/further_reading.include b/docs/installation/further_reading.include
new file mode 100644
index 000000000..46752c722
--- /dev/null
+++ b/docs/installation/further_reading.include
@@ -0,0 +1,5 @@
+* [How Federation Works/Why is my Federated Timeline empty?](https://blog.soykaf.com/post/how-federation-works/)
+* [Backup your instance](../administration/backup.md)
+* [Updating your instance](../administration/updating.md)
+* [Hardening your instance](../configuration/hardening.md)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md
index 1e61373cc..32152aea7 100644
--- a/docs/installation/gentoo_en.md
+++ b/docs/installation/gentoo_en.md
@@ -283,10 +283,7 @@ If you opted to allow sudo for the `pleroma` user but would like to remove the a
 
 #### Further reading
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## Questions
 
diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md
index 6a922a27e..3626acc69 100644
--- a/docs/installation/netbsd_en.md
+++ b/docs/installation/netbsd_en.md
@@ -196,3 +196,11 @@ incorrect timestamps. You should have ntpd running.
 ## Instances running NetBSD
 
 * <https://catgirl.science>
+
+#### Further reading
+
+{! backend/installation/further_reading.include !}
+
+## Questions
+
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md
index e8c5d844c..5dbe24f75 100644
--- a/docs/installation/openbsd_en.md
+++ b/docs/installation/openbsd_en.md
@@ -242,3 +242,11 @@ If your instance is up and running, you can create your first user with administ
 ```
 LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
 ```
+
+#### Further reading
+
+{! backend/installation/further_reading.include !}
+
+## Questions
+
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md
index 86135cd20..e4f822d1c 100644
--- a/docs/installation/otp_en.md
+++ b/docs/installation/otp_en.md
@@ -270,10 +270,7 @@ This will create an account withe the username of 'joeuser' with the email addre
 
 ## Further reading
 
-* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](../configuration/hardening.md)
-* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Updating your instance](../administration/updating.md)
+{! backend/installation/further_reading.include !}
 
 ## Questions
 

From 4f5af68b3e96c5b5b62185f86af39fc2f8955e10 Mon Sep 17 00:00:00 2001
From: Ben Is <srsbzns@cock.li>
Date: Fri, 19 Jun 2020 14:33:58 +0000
Subject: [PATCH 270/283] Added translation using Weblate (Italian)

---
 priv/gettext/it/LC_MESSAGES/errors.po | 578 ++++++++++++++++++++++++++
 1 file changed, 578 insertions(+)
 create mode 100644 priv/gettext/it/LC_MESSAGES/errors.po

diff --git a/priv/gettext/it/LC_MESSAGES/errors.po b/priv/gettext/it/LC_MESSAGES/errors.po
new file mode 100644
index 000000000..18ec03c83
--- /dev/null
+++ b/priv/gettext/it/LC_MESSAGES/errors.po
@@ -0,0 +1,578 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-06-19 14:33+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: Automatically generated\n"
+"Language-Team: none\n"
+"Language: it\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"X-Generator: Translate Toolkit 2.5.1\n"
+
+## This file is a PO Template file.
+##
+## `msgid`s here are often extracted from source code.
+## Add new translations manually only if they're dynamic
+## translations that can't be statically extracted.
+##
+## Run `mix gettext.extract` to bring this file up to
+## date. Leave `msgstr`s empty as changing them here as no
+## effect: edit them in PO (`.po`) files instead.
+## From Ecto.Changeset.cast/4
+msgid "can't be blank"
+msgstr ""
+
+## From Ecto.Changeset.unique_constraint/3
+msgid "has already been taken"
+msgstr ""
+
+## From Ecto.Changeset.put_change/3
+msgid "is invalid"
+msgstr ""
+
+## From Ecto.Changeset.validate_format/3
+msgid "has invalid format"
+msgstr ""
+
+## From Ecto.Changeset.validate_subset/3
+msgid "has an invalid entry"
+msgstr ""
+
+## From Ecto.Changeset.validate_exclusion/3
+msgid "is reserved"
+msgstr ""
+
+## From Ecto.Changeset.validate_confirmation/3
+msgid "does not match confirmation"
+msgstr ""
+
+## From Ecto.Changeset.no_assoc_constraint/3
+msgid "is still associated with this entry"
+msgstr ""
+
+msgid "are still associated with this entry"
+msgstr ""
+
+## From Ecto.Changeset.validate_length/3
+msgid "should be %{count} character(s)"
+msgid_plural "should be %{count} character(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+msgid "should have %{count} item(s)"
+msgid_plural "should have %{count} item(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+msgid "should be at least %{count} character(s)"
+msgid_plural "should be at least %{count} character(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+msgid "should have at least %{count} item(s)"
+msgid_plural "should have at least %{count} item(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+msgid "should be at most %{count} character(s)"
+msgid_plural "should be at most %{count} character(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+msgid "should have at most %{count} item(s)"
+msgid_plural "should have at most %{count} item(s)"
+msgstr[0] ""
+msgstr[1] ""
+
+## From Ecto.Changeset.validate_number/3
+msgid "must be less than %{number}"
+msgstr ""
+
+msgid "must be greater than %{number}"
+msgstr ""
+
+msgid "must be less than or equal to %{number}"
+msgstr ""
+
+msgid "must be greater than or equal to %{number}"
+msgstr ""
+
+msgid "must be equal to %{number}"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:421
+#, elixir-format
+msgid "Account not found"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:249
+#, elixir-format
+msgid "Already voted"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:360
+#, elixir-format
+msgid "Bad request"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:425
+#, elixir-format
+msgid "Can't delete object"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/status_controller.ex:196
+#, elixir-format
+msgid "Can't delete this post"
+msgstr ""
+
+#: lib/pleroma/web/controller_helper.ex:95
+#: lib/pleroma/web/controller_helper.ex:101
+#, elixir-format
+msgid "Can't display this activity"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:227
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:254
+#, elixir-format
+msgid "Can't find user"
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:114
+#, elixir-format
+msgid "Can't get favorites"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:437
+#, elixir-format
+msgid "Can't like object"
+msgstr ""
+
+#: lib/pleroma/web/common_api/utils.ex:556
+#, elixir-format
+msgid "Cannot post an empty status without attachments"
+msgstr ""
+
+#: lib/pleroma/web/common_api/utils.ex:504
+#, elixir-format
+msgid "Comment must be up to %{max_size} characters"
+msgstr ""
+
+#: lib/pleroma/config/config_db.ex:222
+#, elixir-format
+msgid "Config with params %{params} not found"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:95
+#, elixir-format
+msgid "Could not delete"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:141
+#, elixir-format
+msgid "Could not favorite"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:370
+#, elixir-format
+msgid "Could not pin"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:112
+#, elixir-format
+msgid "Could not repeat"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:188
+#, elixir-format
+msgid "Could not unfavorite"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:380
+#, elixir-format
+msgid "Could not unpin"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:126
+#, elixir-format
+msgid "Could not unrepeat"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:428
+#: lib/pleroma/web/common_api/common_api.ex:437
+#, elixir-format
+msgid "Could not update state"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:202
+#, elixir-format
+msgid "Error."
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:106
+#, elixir-format
+msgid "Invalid CAPTCHA"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:117
+#: lib/pleroma/web/oauth/oauth_controller.ex:569
+#, elixir-format
+msgid "Invalid credentials"
+msgstr ""
+
+#: lib/pleroma/plugs/ensure_authenticated_plug.ex:38
+#, elixir-format
+msgid "Invalid credentials."
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:265
+#, elixir-format
+msgid "Invalid indices"
+msgstr ""
+
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:1147
+#, elixir-format
+msgid "Invalid parameters"
+msgstr ""
+
+#: lib/pleroma/web/common_api/utils.ex:411
+#, elixir-format
+msgid "Invalid password."
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:187
+#, elixir-format
+msgid "Invalid request"
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:109
+#, elixir-format
+msgid "Kocaptcha service unavailable"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:113
+#, elixir-format
+msgid "Missing parameters"
+msgstr ""
+
+#: lib/pleroma/web/common_api/utils.ex:540
+#, elixir-format
+msgid "No such conversation"
+msgstr ""
+
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:439
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:465 lib/pleroma/web/admin_api/admin_api_controller.ex:507
+#, elixir-format
+msgid "No such permission_group"
+msgstr ""
+
+#: lib/pleroma/plugs/uploaded_media.ex:74
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:485 lib/pleroma/web/admin_api/admin_api_controller.ex:1135
+#: lib/pleroma/web/feed/user_controller.ex:73 lib/pleroma/web/ostatus/ostatus_controller.ex:143
+#, elixir-format
+msgid "Not found"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:241
+#, elixir-format
+msgid "Poll's author can't vote"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20
+#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49
+#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:50 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:290
+#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71
+#, elixir-format
+msgid "Record not found"
+msgstr ""
+
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:1153
+#: lib/pleroma/web/feed/user_controller.ex:79 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:32
+#: lib/pleroma/web/ostatus/ostatus_controller.ex:149
+#, elixir-format
+msgid "Something went wrong"
+msgstr ""
+
+#: lib/pleroma/web/common_api/activity_draft.ex:107
+#, elixir-format
+msgid "The message visibility must be direct"
+msgstr ""
+
+#: lib/pleroma/web/common_api/utils.ex:566
+#, elixir-format
+msgid "The status is over the character limit"
+msgstr ""
+
+#: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31
+#, elixir-format
+msgid "This resource requires authentication."
+msgstr ""
+
+#: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206
+#, elixir-format
+msgid "Throttled"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:266
+#, elixir-format
+msgid "Too many choices"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:442
+#, elixir-format
+msgid "Unhandled activity type"
+msgstr ""
+
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:536
+#, elixir-format
+msgid "You can't revoke your own admin status."
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:218
+#: lib/pleroma/web/oauth/oauth_controller.ex:309
+#, elixir-format
+msgid "Your account is currently disabled"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:180
+#: lib/pleroma/web/oauth/oauth_controller.ex:332
+#, elixir-format
+msgid "Your login is missing a confirmed e-mail address"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:389
+#, elixir-format
+msgid "can't read inbox of %{nickname} as %{as_nickname}"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:472
+#, elixir-format
+msgid "can't update outbox of %{nickname} as %{as_nickname}"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:388
+#, elixir-format
+msgid "conversation is already muted"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:316
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:491
+#, elixir-format
+msgid "error"
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:29
+#, elixir-format
+msgid "mascots can only be images"
+msgstr ""
+
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:60
+#, elixir-format
+msgid "not found"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:395
+#, elixir-format
+msgid "Bad OAuth request."
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:115
+#, elixir-format
+msgid "CAPTCHA already used"
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:112
+#, elixir-format
+msgid "CAPTCHA expired"
+msgstr ""
+
+#: lib/pleroma/plugs/uploaded_media.ex:55
+#, elixir-format
+msgid "Failed"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:411
+#, elixir-format
+msgid "Failed to authenticate: %{message}."
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:442
+#, elixir-format
+msgid "Failed to set up user account."
+msgstr ""
+
+#: lib/pleroma/plugs/oauth_scopes_plug.ex:38
+#, elixir-format
+msgid "Insufficient permissions: %{permissions}."
+msgstr ""
+
+#: lib/pleroma/plugs/uploaded_media.ex:94
+#, elixir-format
+msgid "Internal Error"
+msgstr ""
+
+#: lib/pleroma/web/oauth/fallback_controller.ex:22
+#: lib/pleroma/web/oauth/fallback_controller.ex:29
+#, elixir-format
+msgid "Invalid Username/Password"
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:118
+#, elixir-format
+msgid "Invalid answer data"
+msgstr ""
+
+#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:128
+#, elixir-format
+msgid "Nodeinfo schema version not handled"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:169
+#, elixir-format
+msgid "This action is outside the authorized scopes"
+msgstr ""
+
+#: lib/pleroma/web/oauth/fallback_controller.ex:14
+#, elixir-format
+msgid "Unknown error, please check the details and try again."
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:116
+#: lib/pleroma/web/oauth/oauth_controller.ex:155
+#, elixir-format
+msgid "Unlisted redirect_uri."
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:391
+#, elixir-format
+msgid "Unsupported OAuth provider: %{provider}."
+msgstr ""
+
+#: lib/pleroma/uploaders/uploader.ex:72
+#, elixir-format
+msgid "Uploader callback timeout"
+msgstr ""
+
+#: lib/pleroma/web/uploader_controller.ex:23
+#, elixir-format
+msgid "bad request"
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:103
+#, elixir-format
+msgid "CAPTCHA Error"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:200
+#, elixir-format
+msgid "Could not add reaction emoji"
+msgstr ""
+
+#: lib/pleroma/web/common_api/common_api.ex:211
+#, elixir-format
+msgid "Could not remove reaction emoji"
+msgstr ""
+
+#: lib/pleroma/web/twitter_api/twitter_api.ex:129
+#, elixir-format
+msgid "Invalid CAPTCHA (Missing parameter: %{name})"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92
+#, elixir-format
+msgid "List not found"
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:124
+#, elixir-format
+msgid "Missing parameter: %{name}"
+msgstr ""
+
+#: lib/pleroma/web/oauth/oauth_controller.ex:207
+#: lib/pleroma/web/oauth/oauth_controller.ex:322
+#, elixir-format
+msgid "Password reset is required"
+msgstr ""
+
+#: lib/pleroma/tests/auth_test_controller.ex:9
+#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/admin_api_controller.ex:6
+#: lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/fallback_redirect_controller.ex:6
+#: lib/pleroma/web/feed/tag_controller.ex:6 lib/pleroma/web/feed/user_controller.ex:6
+#: lib/pleroma/web/mailer/subscription_controller.ex:2 lib/pleroma/web/masto_fe_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/app_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/conversation_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/custom_emoji_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/report_controller.ex:8 lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6
+#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 lib/pleroma/web/media_proxy/media_proxy_controller.ex:6
+#: lib/pleroma/web/mongooseim/mongoose_im_controller.ex:6 lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6
+#: lib/pleroma/web/oauth/fallback_controller.ex:6 lib/pleroma/web/oauth/mfa_controller.ex:10
+#: lib/pleroma/web/oauth/oauth_controller.ex:6 lib/pleroma/web/ostatus/ostatus_controller.ex:6
+#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:2
+#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex:6
+#: lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex:6
+#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/static_fe/static_fe_controller.ex:6
+#: lib/pleroma/web/twitter_api/controllers/password_controller.ex:10 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex:6
+#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/twitter_api/twitter_api_controller.ex:6
+#: lib/pleroma/web/uploader_controller.ex:6 lib/pleroma/web/web_finger/web_finger_controller.ex:6
+#, elixir-format
+msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped."
+msgstr ""
+
+#: lib/pleroma/plugs/ensure_authenticated_plug.ex:28
+#, elixir-format
+msgid "Two-factor authentication enabled, you must use a access token."
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:210
+#, elixir-format
+msgid "Unexpected error occurred while adding file to pack."
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:138
+#, elixir-format
+msgid "Unexpected error occurred while creating pack."
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:278
+#, elixir-format
+msgid "Unexpected error occurred while removing file from pack."
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:250
+#, elixir-format
+msgid "Unexpected error occurred while updating file in pack."
+msgstr ""
+
+#: lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex:179
+#, elixir-format
+msgid "Unexpected error occurred while updating pack metadata."
+msgstr ""
+
+#: lib/pleroma/plugs/user_is_admin_plug.ex:40
+#, elixir-format
+msgid "User is not an admin or OAuth admin scope is not granted."
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61
+#, elixir-format
+msgid "Web push subscription is disabled on this Pleroma instance"
+msgstr ""
+
+#: lib/pleroma/web/admin_api/admin_api_controller.ex:502
+#, elixir-format
+msgid "You can't revoke your own admin/moderator status."
+msgstr ""
+
+#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:105
+#, elixir-format
+msgid "authorization required for timeline view"
+msgstr ""

From 68c812eb2ecbfb1d582925c15d90bd1bd4e62b4b Mon Sep 17 00:00:00 2001
From: Ben Is <srsbzns@cock.li>
Date: Fri, 19 Jun 2020 14:35:01 +0000
Subject: [PATCH 271/283] Translated using Weblate (Italian)

Currently translated at 0.9% (1 of 106 strings)

Translation: Pleroma/Pleroma backend
Translate-URL: https://translate.pleroma.social/projects/pleroma/pleroma/it/
---
 priv/gettext/it/LC_MESSAGES/errors.po | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/priv/gettext/it/LC_MESSAGES/errors.po b/priv/gettext/it/LC_MESSAGES/errors.po
index 18ec03c83..726be628b 100644
--- a/priv/gettext/it/LC_MESSAGES/errors.po
+++ b/priv/gettext/it/LC_MESSAGES/errors.po
@@ -3,14 +3,16 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2020-06-19 14:33+0000\n"
-"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
-"Last-Translator: Automatically generated\n"
-"Language-Team: none\n"
+"PO-Revision-Date: 2020-06-19 20:38+0000\n"
+"Last-Translator: Ben Is <srsbzns@cock.li>\n"
+"Language-Team: Italian <https://translate.pleroma.social/projects/pleroma/"
+"pleroma/it/>\n"
 "Language: it\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Generator: Translate Toolkit 2.5.1\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Weblate 4.0.4\n"
 
 ## This file is a PO Template file.
 ##
@@ -23,7 +25,7 @@ msgstr ""
 ## effect: edit them in PO (`.po`) files instead.
 ## From Ecto.Changeset.cast/4
 msgid "can't be blank"
-msgstr ""
+msgstr "non può essere nullo"
 
 ## From Ecto.Changeset.unique_constraint/3
 msgid "has already been taken"

From a3b10a4f643d574b84ecee51fb891e26e7f0dbc2 Mon Sep 17 00:00:00 2001
From: Ilja <domainepublic@spectraltheorem.be>
Date: Mon, 22 Jun 2020 14:13:30 +0200
Subject: [PATCH 272/283] Fix 1586 Docs: provide a index.md

* I renamed the introduction.md to index.md
* I moved over the FE parts to an index file in the FE repo (will do an MR in the FE repo to actually add it)
* While I was at it, I also fixed some broken links
---
 docs/index.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 docs/index.md

diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 000000000..fb9e32816
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,26 @@
+# Introduction to Pleroma
+## What is Pleroma?
+Pleroma is a federated social networking platform, compatible with Mastodon and other ActivityPub implementations. It is free software licensed under the AGPLv3.
+It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.
+It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.
+One account on an instance is enough to talk to the entire fediverse!
+
+## How can I use it?
+
+Pleroma instances are already widely deployed, a list can be found at <https://the-federation.info/pleroma> and <https://fediverse.network/pleroma>.
+
+If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!
+Installation instructions can be found in the installation section of these docs.
+
+## I got an account, now what?
+Great! Now you can explore the fediverse! Open the login page for your Pleroma instance (e.g. <https://pleroma.soykaf.com>) and login with your username and password. (If you don't have an account yet, click on Register)
+
+### Pleroma-FE
+The default front-end used by Pleroma is Pleroma-FE. You can find more information on what it is and how to use it in the [Introduction to Pleroma-FE](../frontend).
+
+### Mastodon interface
+If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
+Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
+The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
+
+Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma. 

From 1e089cdf2905309a5450e2acb32aa6b35a928c29 Mon Sep 17 00:00:00 2001
From: Ilja <domainepublic@spectraltheorem.be>
Date: Mon, 22 Jun 2020 14:18:55 +0200
Subject: [PATCH 273/283] I forgot to git add some files, oops (should be
 squashed with MR)

---
 .../howto_theming_your_instance.md            |  2 +-
 docs/dev.md                                   |  2 +-
 docs/introduction.md                          | 65 -------------------
 3 files changed, 2 insertions(+), 67 deletions(-)
 delete mode 100644 docs/introduction.md

diff --git a/docs/configuration/howto_theming_your_instance.md b/docs/configuration/howto_theming_your_instance.md
index d0daf5b25..cfa00f538 100644
--- a/docs/configuration/howto_theming_your_instance.md
+++ b/docs/configuration/howto_theming_your_instance.md
@@ -60,7 +60,7 @@ Example of `my-awesome-theme.json` where we add the name "My Awesome Theme"
 
 ### Set as default theme
 
-Now we can set the new theme as default in the [Pleroma FE configuration](General-tips-for-customizing-Pleroma-FE.md).
+Now we can set the new theme as default in the [Pleroma FE configuration](../../../frontend/CONFIGURATION).
 
 Example of adding the new theme in the back-end config files
 ```elixir
diff --git a/docs/dev.md b/docs/dev.md
index f1b4cbf8b..9c749c17c 100644
--- a/docs/dev.md
+++ b/docs/dev.md
@@ -20,4 +20,4 @@ This document contains notes and guidelines for Pleroma developers.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 
-See `Authentication` section of [`docs/configuration/cheatsheet.md`](docs/configuration/cheatsheet.md#authentication).
+See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication).
diff --git a/docs/introduction.md b/docs/introduction.md
deleted file mode 100644
index a915c143c..000000000
--- a/docs/introduction.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Introduction to Pleroma
-## What is Pleroma?
-Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.
-It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.
-It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.
-One account on an instance is enough to talk to the entire fediverse!
-
-## How can I use it?
-
-Pleroma instances are already widely deployed, a list can be found at <http://distsn.org/pleroma-instances.html>. Information on all existing fediverse instances can be found at <https://fediverse.network/>.
-
-If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!
-Installation instructions can be found in the installation section of these docs.
-
-## I got an account, now what?
-Great! Now you can explore the fediverse! Open the login page for your Pleroma instance (e.g. <https://pleroma.soykaf.com>) and login with your username and password. (If you don't have an account yet, click on Register)
-
-At this point you will have two columns in front of you.
-
-### Left column
-
-- first block: here you can see your avatar, your nickname and statistics (Statuses, Following, Followers). Clicking your profile pic will open your profile.
-Under that you have a text form which allows you to post new statuses. The number on the bottom of the text form is a character counter, every instance can have a different character limit (the default is 5000).
-If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person.
-Under the text form there are also several visibility options and there is the option to use rich text.
-Under that the icon on the left is for uploading media files and attach them to your post. There is also an emoji-picker and an option to post a poll.
-To post your status, simply press Submit.
-On the top right you will also see a wrench icon. This opens your personal settings.
-
-- second block: Here you can switch between the different timelines:
-   - Timeline: all the people that you follow
-   - Interactions: here you can switch between different timelines where there was interaction with your account. There is Mentions, Repeats and Favorites, and New follows
-   - Direct Messages: these are the Direct Messages sent to you
-   - Public Timeline: all the statutes from the local instance
-   - The Whole Known Network: all public posts the instance knows about, both local and remote!
-   - About: This isn't a Timeline but shows relevant info about the instance. You can find a list of the moderators and admins, Terms of Service, MRF policies and enabled features.
-- Optional third block: This is the Instance panel that can be activated, but is deactivated by default. It's fully customisable and by default has links to the pleroma-fe and Mastodon-fe.
-- fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses.
-
-### Right column
-This is where the interesting stuff happens!
-Depending on the timeline you will see different statuses, but each status has a standard structure:
-
-- Profile pic, name and link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the reply-to status). Clicking on the profile pic will uncollapse the user's profile.
-- A `+` button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime!
-- An arrow icon allows you to open the status on the instance where it's originating from.
-- The text of the status, including mentions and attachements. If you click on a mention, it will automatically open the profile page of that person.
-- Three buttons (left to right): Reply, Repeat, Favorite. There is also a forth button, this is a dropdown menu for simple moderation like muting the conversation or, if you have moderation rights, delete the status from the server.
-
-### Top right
-
-- The magnifier icon opens the search screen where you can search for statuses, people and hashtags. It's also possible to import statusses from remote servers by pasting the url to the post in the search field.
-- The gear icon gives you general settings
-- If you have admin rights, you'll see an icon that opens the admin interface
-- The last icon is to log out
-
-### Bottom right
-On the bottom right you have a chatbox. Here you can communicate with people on the same instance in realtime. It is local-only, for now, but there are plans to make it extendable to the entire fediverse!
-
-### Mastodon interface
-If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
-Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
-The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
-
-Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

From 499324f7bee55de4e08647f71fd4adbfd4bd039f Mon Sep 17 00:00:00 2001
From: Ilja <domainepublic@spectraltheorem.be>
Date: Mon, 22 Jun 2020 14:22:23 +0200
Subject: [PATCH 274/283] Removed a space that was too much

---
 docs/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.md b/docs/index.md
index fb9e32816..1a90d0a8d 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -23,4 +23,4 @@ If the Pleroma interface isn't your thing, or you're just trying something new b
 Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
 The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
 
-Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma. 
+Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

From b0a40fc2e42a186fc6bb383621f291411b2a81be Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 22 Jun 2020 17:27:49 +0300
Subject: [PATCH 275/283] added verify RUM settings before start app

---
 lib/pleroma/application.ex                    |   2 +-
 lib/pleroma/application_requirements.ex       | 103 ++++++++++++++++++
 lib/pleroma/repo.ex                           |  37 +------
 ...510135645_add_fts_index_to_objects_two.exs |  39 +++----
 test/application_requirements_test.exs        |  67 ++++++++++++
 test/repo_test.exs                            |  34 ------
 6 files changed, 189 insertions(+), 93 deletions(-)
 create mode 100644 lib/pleroma/application_requirements.ex
 create mode 100644 test/application_requirements_test.exs

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 9d3d92b38..c30e5aadf 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -39,7 +39,7 @@ def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
     Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
-    Pleroma.Repo.check_migrations_applied!()
+    Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
 
diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex
new file mode 100644
index 000000000..3bba70b7b
--- /dev/null
+++ b/lib/pleroma/application_requirements.ex
@@ -0,0 +1,103 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ApplicationRequirements do
+  @moduledoc """
+  The module represents the collection of validations to runs before start server.
+  """
+
+  defmodule VerifyError, do: defexception([:message])
+
+  import Ecto.Query
+
+  require Logger
+
+  @spec verify!() :: :ok | VerifyError.t()
+  def verify! do
+    :ok
+    |> check_migrations_applied!()
+    |> check_rum!()
+    |> handle_result()
+  end
+
+  defp handle_result(:ok), do: :ok
+  defp handle_result({:error, message}), do: raise(VerifyError, message: message)
+
+  defp check_migrations_applied!(:ok) do
+    unless Pleroma.Config.get(
+             [:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
+             false
+           ) do
+      {_, res, _} =
+        Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+          down_migrations =
+            Ecto.Migrator.migrations(repo)
+            |> Enum.reject(fn
+              {:up, _, _} -> true
+              {:down, _, _} -> false
+            end)
+
+          if length(down_migrations) > 0 do
+            down_migrations_text =
+              Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
+
+            Logger.error(
+              "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
+            )
+
+            {:error, "Unapplied Migrations detected"}
+          else
+            :ok
+          end
+        end)
+
+      res
+    else
+      :ok
+    end
+  end
+
+  defp check_migrations_applied!(result), do: result
+
+  defp check_rum!(:ok) do
+    {_, res, _} =
+      Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+        migrate =
+          from(o in "columns",
+            where: o.table_name == "objects",
+            where: o.column_name == "fts_content"
+          )
+          |> repo.exists?(prefix: "information_schema")
+
+        setting = Pleroma.Config.get([:database, :rum_enabled], false)
+
+        do_check_rum!(setting, migrate)
+      end)
+
+    res
+  end
+
+  defp check_rum!(result), do: result
+
+  defp do_check_rum!(setting, migrate) do
+    case {setting, migrate} do
+      {true, false} ->
+        Logger.error(
+          "Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+        )
+
+        {:error, "Unapplied RUM Migrations detected"}
+
+      {false, true} ->
+        Logger.error(
+          "Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
+        )
+
+        {:error, "RUM Migrations detected"}
+
+      _ ->
+        :ok
+    end
+  end
+end
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index 6d85d70bc..f317e4d58 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -11,9 +11,7 @@ defmodule Pleroma.Repo do
   import Ecto.Query
   require Logger
 
-  defmodule Instrumenter do
-    use Prometheus.EctoInstrumenter
-  end
+  defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
 
   @doc """
   Dynamically loads the repository url from the
@@ -51,35 +49,6 @@ def get_assoc(resource, association) do
     end
   end
 
-  def check_migrations_applied!() do
-    unless Pleroma.Config.get(
-             [:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
-             false
-           ) do
-      Ecto.Migrator.with_repo(__MODULE__, fn repo ->
-        down_migrations =
-          Ecto.Migrator.migrations(repo)
-          |> Enum.reject(fn
-            {:up, _, _} -> true
-            {:down, _, _} -> false
-          end)
-
-        if length(down_migrations) > 0 do
-          down_migrations_text =
-            Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
-
-          Logger.error(
-            "The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
-          )
-
-          raise Pleroma.Repo.UnappliedMigrationsError
-        end
-      end)
-    else
-      :ok
-    end
-  end
-
   def chunk_stream(query, chunk_size) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
@@ -107,7 +76,3 @@ def chunk_stream(query, chunk_size) do
     )
   end
 end
-
-defmodule Pleroma.Repo.UnappliedMigrationsError do
-  defexception message: "Unapplied Migrations detected"
-end
diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
index 79bde163d..757afa129 100644
--- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@@ -2,29 +2,24 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
   use Ecto.Migration
 
   def up do
-    if Pleroma.Config.get([:database, :rum_enabled]) do
-      execute("create extension if not exists rum")
-      drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
-      alter table(:objects) do
-        add(:fts_content, :tsvector)
-      end
-
-      execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
-      begin
-      new.fts_content := to_tsvector('english', new.data->>'content');
-      return new;
-      end
-      $$ LANGUAGE plpgsql")
-      execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
-
-      execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
-      FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
-
-      execute("UPDATE objects SET updated_at = NOW()")
-    else
-        raise Ecto.MigrationError,
-          message: "Migration is not allowed. You can change this behavior by setting `database/rum_enabled` to true."
+    execute("create extension if not exists rum")
+    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    alter table(:objects) do
+      add(:fts_content, :tsvector)
     end
+
+    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+    begin
+    new.fts_content := to_tsvector('english', new.data->>'content');
+    return new;
+    end
+    $$ LANGUAGE plpgsql")
+    execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+    execute("UPDATE objects SET updated_at = NOW()")
   end
 
   def down do
diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs
new file mode 100644
index 000000000..0981fcdeb
--- /dev/null
+++ b/test/application_requirements_test.exs
@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RepoTest do
+  use Pleroma.DataCase
+  import ExUnit.CaptureLog
+  import Mock
+
+  describe "check_rum!" do
+    setup_with_mocks([
+      {Ecto.Migrator, [],
+       [
+         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
+         migrations: fn Pleroma.Repo -> [] end
+       ]}
+    ]) do
+      :ok
+    end
+
+    setup do: clear_config([:database, :rum_enabled])
+
+    test "raises if rum is enabled and  detects unapplied rum migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], true)
+
+      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                   "Unapplied RUM Migrations detected",
+                   fn ->
+                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                   end
+    end
+  end
+
+  describe "check_migrations_applied!" do
+    setup_with_mocks([
+      {Ecto.Migrator, [],
+       [
+         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
+         migrations: fn Pleroma.Repo ->
+           [
+             {:up, 20_191_128_153_944, "fix_missing_following_count"},
+             {:up, 20_191_203_043_610, "create_report_notes"},
+             {:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
+           ]
+         end
+       ]}
+    ]) do
+      :ok
+    end
+
+    setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
+
+    test "raises if it detects unapplied migrations" do
+      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                   "Unapplied Migrations detected",
+                   fn ->
+                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                   end
+    end
+
+    test "doesn't do anything if disabled" do
+      Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
+
+      assert :ok == Pleroma.ApplicationRequirements.verify!()
+    end
+  end
+end
diff --git a/test/repo_test.exs b/test/repo_test.exs
index daffc6542..92e827c95 100644
--- a/test/repo_test.exs
+++ b/test/repo_test.exs
@@ -4,9 +4,7 @@
 
 defmodule Pleroma.RepoTest do
   use Pleroma.DataCase
-  import ExUnit.CaptureLog
   import Pleroma.Factory
-  import Mock
 
   alias Pleroma.User
 
@@ -49,36 +47,4 @@ test "return error if has not assoc " do
       assert Repo.get_assoc(token, :user) == {:error, :not_found}
     end
   end
-
-  describe "check_migrations_applied!" do
-    setup_with_mocks([
-      {Ecto.Migrator, [],
-       [
-         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
-         migrations: fn Pleroma.Repo ->
-           [
-             {:up, 20_191_128_153_944, "fix_missing_following_count"},
-             {:up, 20_191_203_043_610, "create_report_notes"},
-             {:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
-           ]
-         end
-       ]}
-    ]) do
-      :ok
-    end
-
-    setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
-
-    test "raises if it detects unapplied migrations" do
-      assert_raise Pleroma.Repo.UnappliedMigrationsError, fn ->
-        capture_log(&Repo.check_migrations_applied!/0)
-      end
-    end
-
-    test "doesn't do anything if disabled" do
-      Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
-
-      assert :ok == Repo.check_migrations_applied!()
-    end
-  end
 end

From 7e6f43c0d7c625a03ee0216c2d9474253ef87b5a Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 22 Jun 2020 19:03:04 +0400
Subject: [PATCH 276/283] Add `is_muted` to notifications

---
 .../mastodon_api/views/notification_view.ex   |  8 ++---
 .../views/notification_view_test.exs          | 36 +++++++++++++++----
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 3865be280..c97e6d32f 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -84,12 +84,7 @@ def render(
 
     # Note: :relationships contain user mutes (needed for :muted flag in :status)
     status_render_opts = %{relationships: opts[:relationships]}
-
-    account =
-      AccountView.render(
-        "show.json",
-        %{user: actor, for: reading_user}
-      )
+    account = AccountView.render("show.json", %{user: actor, for: reading_user})
 
     response = %{
       id: to_string(notification.id),
@@ -97,6 +92,7 @@ def render(
       created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
       account: account,
       pleroma: %{
+        is_muted: User.mutes?(reading_user, actor),
         is_seen: notification.seen
       }
     }
diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs
index 9c399b2df..8e0e58538 100644
--- a/test/web/mastodon_api/views/notification_view_test.exs
+++ b/test/web/mastodon_api/views/notification_view_test.exs
@@ -49,7 +49,7 @@ test "ChatMessage notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "pleroma:chat_mention",
       account: AccountView.render("show.json", %{user: user, for: recipient}),
       chat_message: MessageReferenceView.render("show.json", %{chat_message_reference: cm_ref}),
@@ -68,7 +68,7 @@ test "Mention notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "mention",
       account:
         AccountView.render("show.json", %{
@@ -92,7 +92,7 @@ test "Favourite notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "favourite",
       account: AccountView.render("show.json", %{user: another_user, for: user}),
       status: StatusView.render("show.json", %{activity: create_activity, for: user}),
@@ -112,7 +112,7 @@ test "Reblog notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "reblog",
       account: AccountView.render("show.json", %{user: another_user, for: user}),
       status: StatusView.render("show.json", %{activity: reblog_activity, for: user}),
@@ -130,7 +130,7 @@ test "Follow notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "follow",
       account: AccountView.render("show.json", %{user: follower, for: followed}),
       created_at: Utils.to_masto_date(notification.inserted_at)
@@ -171,7 +171,7 @@ test "Move notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "move",
       account: AccountView.render("show.json", %{user: old_user, for: follower}),
       target: AccountView.render("show.json", %{user: new_user, for: follower}),
@@ -196,7 +196,7 @@ test "EmojiReact notification" do
 
     expected = %{
       id: to_string(notification.id),
-      pleroma: %{is_seen: false},
+      pleroma: %{is_seen: false, is_muted: false},
       type: "pleroma:emoji_reaction",
       emoji: "☕",
       account: AccountView.render("show.json", %{user: other_user, for: user}),
@@ -206,4 +206,26 @@ test "EmojiReact notification" do
 
     test_notifications_rendering([notification], user, [expected])
   end
+
+  test "muted notification" do
+    user = insert(:user)
+    another_user = insert(:user)
+
+    {:ok, _} = Pleroma.UserRelationship.create_mute(user, another_user)
+    {:ok, create_activity} = CommonAPI.post(user, %{status: "hey"})
+    {:ok, favorite_activity} = CommonAPI.favorite(another_user, create_activity.id)
+    {:ok, [notification]} = Notification.create_notifications(favorite_activity)
+    create_activity = Activity.get_by_id(create_activity.id)
+
+    expected = %{
+      id: to_string(notification.id),
+      pleroma: %{is_seen: false, is_muted: true},
+      type: "favourite",
+      account: AccountView.render("show.json", %{user: another_user, for: user}),
+      status: StatusView.render("show.json", %{activity: create_activity, for: user}),
+      created_at: Utils.to_masto_date(notification.inserted_at)
+    }
+
+    test_notifications_rendering([notification], user, [expected])
+  end
 end

From b3a549e916c2a721da16f60e7665b6eb64d756dd Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 22 Jun 2020 19:18:33 +0400
Subject: [PATCH 277/283] Update NotificationOperation spec

---
 .../web/api_spec/operations/notification_operation.ex  | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex
index 41328b5f2..f09be64cb 100644
--- a/lib/pleroma/web/api_spec/operations/notification_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex
@@ -163,6 +163,13 @@ def notification do
           description:
             "Status that was the object of the notification, e.g. in mentions, reblogs, favourites, or polls.",
           nullable: true
+        },
+        pleroma: %Schema{
+          type: :object,
+          properties: %{
+            is_seen: %Schema{type: :boolean},
+            is_muted: %Schema{type: :boolean}
+          }
         }
       },
       example: %{
@@ -170,7 +177,8 @@ def notification do
         "type" => "mention",
         "created_at" => "2019-11-23T07:49:02.064Z",
         "account" => Account.schema().example,
-        "status" => Status.schema().example
+        "status" => Status.schema().example,
+        "pleroma" => %{"is_seen" => false, "is_muted" => false}
       }
     }
   end

From 8f6ba4b22f48dcd0256d6a9cf7259aa475895b84 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 22 Jun 2020 23:45:29 +0200
Subject: [PATCH 278/283] Add warning against parsing/reusing MastoFE settings
 blob

---
 lib/pleroma/web/masto_fe_controller.ex | 2 +-
 lib/pleroma/web/router.ex              | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex
index d0d8bc8eb..43ec70021 100644
--- a/lib/pleroma/web/masto_fe_controller.ex
+++ b/lib/pleroma/web/masto_fe_controller.ex
@@ -49,7 +49,7 @@ def manifest(conn, _params) do
     |> render("manifest.json")
   end
 
-  @doc "PUT /api/web/settings"
+  @doc "PUT /api/web/settings: Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere"
   def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do
     with {:ok, _} <- User.mastodon_settings_update(user, settings) do
       json(conn, %{})
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index eda74a171..419aa55e4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -467,6 +467,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/web", Pleroma.Web do
     pipe_through(:authenticated_api)
 
+    # Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere
     put("/settings", MastoFEController, :put_settings)
   end
 

From bf8310f3802c46e6305fcb3832bca297582990d9 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 22 Jun 2020 17:35:02 -0500
Subject: [PATCH 279/283] Add missing default config value for :instance,
 instance_thumbnail

Follows up on b7fc61e17b
---
 config/config.exs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/config.exs b/config/config.exs
index 4bf31f3fc..e0888fa9a 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -186,6 +186,7 @@
   notify_email: "noreply@example.com",
   description: "Pleroma: An efficient and flexible fediverse server",
   background_image: "/images/city.jpg",
+  instance_thumbnail: "/instance/thumbnail.jpeg",
   limit: 5_000,
   chat_limit: 5_000,
   remote_limit: 100_000,

From df5e048cbb7d349b34203ccba49a8f646e4d93a3 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 22 Jun 2020 17:39:02 -0500
Subject: [PATCH 280/283] Do not need a function to provide fallback value with
 default defined in config.exs

---
 lib/pleroma/web/mastodon_api/views/instance_view.ex | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index c498fe632..c6b54e570 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -23,7 +23,7 @@ def render("show.json", _) do
         streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Pleroma.Stats.get_stats(),
-      thumbnail: instance_thumbnail(),
+      thumbnail: Keyword.get(instance, :instance_thumbnail),
       languages: ["en"],
       registrations: Keyword.get(instance, :registrations_open),
       # Extra (not present in Mastodon):
@@ -88,9 +88,4 @@ def federation do
     end
     |> Map.put(:enabled, Config.get([:instance, :federating]))
   end
-
-  defp instance_thumbnail do
-    Pleroma.Config.get([:instance, :instance_thumbnail]) ||
-      "#{Pleroma.Web.base_url()}/instance/thumbnail.jpeg"
-  end
 end

From c116b6d6d6e4b12d9d751481926183f19cdb5248 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 23 Jun 2020 04:42:44 +0200
Subject: [PATCH 281/283] ActivityPubController: Update upload_media @doc

Small cherry-pick from https://git.pleroma.social/pleroma/pleroma/-/merge_requests/1810
---
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index f0b5c6e93..220c4fe52 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -514,7 +514,6 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
     {new_user, for_user}
   end
 
-  # TODO: Add support for "object" field
   @doc """
   Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
 
@@ -525,6 +524,8 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
   Response:
   - HTTP Code: 201 Created
   - HTTP Body: ActivityPub object to be inserted into another's `attachment` field
+
+  Note: Will not point to a URL with a `Location` header because no standalone Activity has been created.
   """
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-

From 2715c40e1d36cc844be1dd7d41a0c6a16ca5f7b7 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 23 Jun 2020 06:56:17 +0300
Subject: [PATCH 282/283] added tests

---
 lib/pleroma/application_requirements.ex |  8 +++--
 test/application_requirements_test.exs  | 48 +++++++++++++++++++++----
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex
index 3bba70b7b..88575a498 100644
--- a/lib/pleroma/application_requirements.ex
+++ b/lib/pleroma/application_requirements.ex
@@ -24,7 +24,9 @@ def verify! do
   defp handle_result(:ok), do: :ok
   defp handle_result({:error, message}), do: raise(VerifyError, message: message)
 
-  defp check_migrations_applied!(:ok) do
+  # Checks for pending migrations.
+  #
+  def check_migrations_applied!(:ok) do
     unless Pleroma.Config.get(
              [:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
              false
@@ -58,8 +60,10 @@ defp check_migrations_applied!(:ok) do
     end
   end
 
-  defp check_migrations_applied!(result), do: result
+  def check_migrations_applied!(result), do: result
 
+  # Checks for settings of RUM indexes.
+  #
   defp check_rum!(:ok) do
     {_, res, _} =
       Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs
index 0981fcdeb..b8d073e11 100644
--- a/test/application_requirements_test.exs
+++ b/test/application_requirements_test.exs
@@ -2,25 +2,22 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.RepoTest do
+defmodule Pleroma.ApplicationRequirementsTest do
   use Pleroma.DataCase
   import ExUnit.CaptureLog
   import Mock
 
   describe "check_rum!" do
     setup_with_mocks([
-      {Ecto.Migrator, [],
-       [
-         with_repo: fn repo, fun -> passthrough([repo, fun]) end,
-         migrations: fn Pleroma.Repo -> [] end
-       ]}
+      {Pleroma.ApplicationRequirements, [:passthrough],
+       [check_migrations_applied!: fn _ -> :ok end]}
     ]) do
       :ok
     end
 
     setup do: clear_config([:database, :rum_enabled])
 
-    test "raises if rum is enabled and  detects unapplied rum migrations" do
+    test "raises if rum is enabled and detects unapplied rum migrations" do
       Pleroma.Config.put([:database, :rum_enabled], true)
 
       assert_raise Pleroma.ApplicationRequirements.VerifyError,
@@ -29,6 +26,43 @@ test "raises if rum is enabled and  detects unapplied rum migrations" do
                      capture_log(&Pleroma.ApplicationRequirements.verify!/0)
                    end
     end
+
+    test "raises if rum is disabled and detects rum migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], false)
+
+      with_mocks([
+        {
+          Pleroma.Repo,
+          [:passthrough],
+          [exists?: fn _, _ -> true end]
+        }
+      ]) do
+        assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                     "RUM Migrations detected",
+                     fn ->
+                       capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     end
+      end
+    end
+
+    test "doesn't do anything if rum enabled and applied migrations" do
+      Pleroma.Config.put([:database, :rum_enabled], true)
+
+      with_mocks([
+        {
+          Pleroma.Repo,
+          [:passthrough],
+          [exists?: fn _, _ -> true end]
+        }
+      ]) do
+        assert Pleroma.ApplicationRequirements.verify!() == :ok
+      end
+    end
+
+    test "doesn't do anything if rum disabled" do
+      Pleroma.Config.put([:database, :rum_enabled], false)
+      assert Pleroma.ApplicationRequirements.verify!() == :ok
+    end
   end
 
   describe "check_migrations_applied!" do

From 84aa9c78dd314e93a5153e3584af38b8c218caed Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 23 Jun 2020 09:08:24 +0300
Subject: [PATCH 283/283] fix tests

---
 test/application_requirements_test.exs | 37 +++++++++++---------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs
index b8d073e11..481cdfd73 100644
--- a/test/application_requirements_test.exs
+++ b/test/application_requirements_test.exs
@@ -7,6 +7,8 @@ defmodule Pleroma.ApplicationRequirementsTest do
   import ExUnit.CaptureLog
   import Mock
 
+  alias Pleroma.Repo
+
   describe "check_rum!" do
     setup_with_mocks([
       {Pleroma.ApplicationRequirements, [:passthrough],
@@ -20,23 +22,19 @@ defmodule Pleroma.ApplicationRequirementsTest do
     test "raises if rum is enabled and detects unapplied rum migrations" do
       Pleroma.Config.put([:database, :rum_enabled], true)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError,
-                   "Unapplied RUM Migrations detected",
-                   fn ->
-                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
-                   end
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
+        assert_raise Pleroma.ApplicationRequirements.VerifyError,
+                     "Unapplied RUM Migrations detected",
+                     fn ->
+                       capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+                     end
+      end
     end
 
     test "raises if rum is disabled and detects rum migrations" do
       Pleroma.Config.put([:database, :rum_enabled], false)
 
-      with_mocks([
-        {
-          Pleroma.Repo,
-          [:passthrough],
-          [exists?: fn _, _ -> true end]
-        }
-      ]) do
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
         assert_raise Pleroma.ApplicationRequirements.VerifyError,
                      "RUM Migrations detected",
                      fn ->
@@ -48,20 +46,17 @@ test "raises if rum is disabled and detects rum migrations" do
     test "doesn't do anything if rum enabled and applied migrations" do
       Pleroma.Config.put([:database, :rum_enabled], true)
 
-      with_mocks([
-        {
-          Pleroma.Repo,
-          [:passthrough],
-          [exists?: fn _, _ -> true end]
-        }
-      ]) do
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
         assert Pleroma.ApplicationRequirements.verify!() == :ok
       end
     end
 
     test "doesn't do anything if rum disabled" do
       Pleroma.Config.put([:database, :rum_enabled], false)
-      assert Pleroma.ApplicationRequirements.verify!() == :ok
+
+      with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
+        assert Pleroma.ApplicationRequirements.verify!() == :ok
+      end
     end
   end
 
@@ -70,7 +65,7 @@ test "doesn't do anything if rum disabled" do
       {Ecto.Migrator, [],
        [
          with_repo: fn repo, fun -> passthrough([repo, fun]) end,
-         migrations: fn Pleroma.Repo ->
+         migrations: fn Repo ->
            [
              {:up, 20_191_128_153_944, "fix_missing_following_count"},
              {:up, 20_191_203_043_610, "create_report_notes"},