Merge remote-tracking branch 'remotes/origin/develop' into media-preview-proxy-nostream

docs/API/admin_api.md

@@ -1334,3 +1334,124 @@ Loads json generated from `config/descriptions.exs`.
 { }
 
 ```
+
+## GET /api/pleroma/admin/users/:nickname/chats
+
+### List a user's chats
+
+- Params: None
+
+- Response:
+
+```json
+[
+   {
+      "sender": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "receiver": {
+        "id": "someflakeid",
+        "username": "somenick",
+        ...
+      },
+      "id" : "1",
+      "unread" : 2,
+      "last_message" : {...}, // The last message in that chat
+      "updated_at": "2020-04-21T15:11:46.000Z"
+   }
+]
+```
+
+## GET /api/pleroma/admin/chats/:chat_id
+
+### View a single chat
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "sender": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "receiver": {
+    "id": "someflakeid",
+    "username": "somenick",
+    ...
+  },
+  "id" : "1",
+  "unread" : 2,
+  "last_message" : {...}, // The last message in that chat
+  "updated_at": "2020-04-21T15:11:46.000Z"
+}
+```
+
+## GET /api/pleroma/admin/chats/:chat_id/messages
+
+### List the messages in a chat
+
+- Params: `max_id`, `min_id`
+
+- Response:
+
+```json
+[
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Check this out :firefox:",
+    "created_at": "2020-04-21T15:11:46.000Z",
+    "emojis": [
+      {
+        "shortcode": "firefox",
+        "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+        "url": "https://dontbulling.me/emoji/Firefox.gif",
+        "visible_in_picker": false
+      }
+    ],
+    "id": "13",
+    "unread": true
+  },
+  {
+    "account_id": "someflakeid",
+    "chat_id": "1",
+    "content": "Whats' up?",
+    "created_at": "2020-04-21T15:06:45.000Z",
+    "emojis": [],
+    "id": "12",
+    "unread": false
+  }
+]
+```
+
+## DELETE /api/pleroma/admin/chats/:chat_id/messages/:message_id
+
+### Delete a single message
+
+- Params: None
+
+- Response:
+
+```json
+{
+  "account_id": "someflakeid",
+  "chat_id": "1",
+  "content": "Check this out :firefox:",
+  "created_at": "2020-04-21T15:11:46.000Z",
+  "emojis": [
+    {
+      "shortcode": "firefox",
+      "static_url": "https://dontbulling.me/emoji/Firefox.gif",
+      "url": "https://dontbulling.me/emoji/Firefox.gif",
+      "visible_in_picker": false
+    }
+  ],
+  "id": "13",
+  "unread": false
+}
+```

lib/mix/tasks/pleroma/database.ex

@@ -99,7 +99,7 @@ def run(["fix_likes_collections"]) do
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -145,7 +145,7 @@ def run(["ensure_expiration"]) do
     |> where(local: true)
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
     |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
         expires_at =

lib/mix/tasks/pleroma/user.ex

@@ -179,7 +179,7 @@ def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -370,7 +370,7 @@ def run(["list"]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->

lib/pleroma/chat.ex

@@ -6,7 +6,9 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
 
   import Ecto.Changeset
+  import Ecto.Query
 
+  alias Pleroma.Chat
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -69,4 +71,12 @@ def bump_or_create(user_id, recipient) do
       conflict_target: [:user_id, :recipient]
     )
   end
+
+  @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t()
+  def for_user_query(user_id) do
+    from(c in Chat,
+      where: c.user_id == ^user_id,
+      order_by: [desc: c.updated_at]
+    )
+  end
 end

lib/pleroma/gun/connection_pool/worker.ex

@@ -93,12 +93,8 @@ def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    # DOWN message can receive right after `remove_client` call and cause worker to terminate
+
-    state =
+    Process.demonitor(ref, [:flush])
-      if is_nil(ref) do
-        state
-      else
-        Process.demonitor(ref)
 
     timer =
       if used_by == [] do
@@ -108,10 +104,7 @@ def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
         nil
       end
 
-        %{state | timer: timer}
+    {:reply, :ok, %{state | timer: timer}, :hibernate}
-      end
-
-    {:reply, :ok, state, :hibernate}
   end
 
   @impl true

lib/pleroma/migration_helper/notification_backfill.ex

@@ -19,13 +19,13 @@ def fill_in_notification_types do
     query
     |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
-      type =
+      if notification.activity do
-        notification.activity
+        type = type_from_activity(notification.activity)
-        |> type_from_activity()
 
         notification
         |> Ecto.Changeset.change(%{type: type})
         |> Repo.update()
+      end
     end)
   end
 
@@ -72,8 +72,7 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do
         "pleroma:emoji_reaction"
 
       "Create" ->
-        activity
+        type_from_activity_object(activity)
-        |> type_from_activity_object()
 
       t ->
         raise "No notification type for activity type #{t}"

lib/pleroma/moderation_log.ex

@@ -320,6 +320,19 @@ def insert_log(%{
     |> insert_log_entry_with_message()
   end
 
+  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+          {:ok, ModerationLog} | {:error, any}
+  def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
+    %ModerationLog{
+      data: %{
+        "actor" => %{"nickname" => actor.nickname},
+        "action" => "chat_message_delete",
+        "subject_id" => subject_id
+      }
+    }
+    |> insert_log_entry_with_message()
+  end
+
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
@@ -627,6 +640,17 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "chat_message_delete",
+          "subject_id" => subject_id
+        }
+      }) do
+    "@#{actor_nickname} deleted chat message ##{subject_id}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/object/fetcher.ex

@@ -98,8 +98,8 @@ def fetch_object_from_id(id, options \\ []) do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:transmogrifier, {:error, {:reject, nil}}} ->
+      {:transmogrifier, {:error, {:reject, e}}} ->
-        {:reject, nil}
+        {:reject, e}
 
       {:transmogrifier, _} = e ->
         {:error, e}

lib/pleroma/repo.ex

@@ -49,7 +49,21 @@ def get_assoc(resource, association) do
     end
   end
 
-  def chunk_stream(query, chunk_size) do
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
     # have individual records be the elements of the stream, instead of
@@ -69,7 +83,12 @@ def chunk_stream(query, chunk_size) do
 
             records ->
               last_id = List.last(records).id
+
+              if returns_as == :one do
                 {records, last_id}
+              else
+                {[records], last_id}
+              end
           end
       end,
       fn _ -> :ok end

lib/pleroma/repo_streamer.ex

@@ -1,34 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.RepoStreamer do
-  alias Pleroma.Repo
-  import Ecto.Query
-
-  def chunk_stream(query, chunk_size) do
-    Stream.unfold(0, fn
-      :halt ->
-        {[], :halt}
-
-      last_id ->
-        query
-        |> order_by(asc: :id)
-        |> where([r], r.id > ^last_id)
-        |> limit(^chunk_size)
-        |> Repo.all()
-        |> case do
-          [] ->
-            {[], :halt}
-
-          records ->
-            last_id = List.last(records).id
-            {records, last_id}
-        end
-    end)
-    |> Stream.take_while(fn
-      [] -> false
-      _ -> true
-    end)
-  end
-end

lib/pleroma/user.ex

@@ -25,7 +25,6 @@ defmodule Pleroma.User do
   alias Pleroma.Object
   alias Pleroma.Registration
   alias Pleroma.Repo
-  alias Pleroma.RepoStreamer
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web
@@ -276,9 +275,9 @@ def binary_id(%User{} = user), do: binary_id(user.id)
   @spec account_status(User.t()) :: account_status()
   def account_status(%User{deactivated: true}), do: :deactivated
   def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
-  def account_status(%User{approval_pending: true}), do: :approval_pending
+  def account_status(%User{local: true, approval_pending: true}), do: :approval_pending
 
-  def account_status(%User{confirmation_pending: true}) do
+  def account_status(%User{local: true, confirmation_pending: true}) do
     if Config.get([:instance, :account_activation_required]) do
       :confirmation_pending
     else
@@ -1775,7 +1774,7 @@ def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
   def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
-    |> RepoStreamer.chunk_stream(50)
+    |> Repo.chunk_stream(50, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity -> delete_activity(activity, user) end)
     end)

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -84,7 +84,7 @@ defp increase_replies_count_if_reply(%{
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Event]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
@@ -154,8 +154,8 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
       {:remote_limit_pass, _} ->
         {:error, :remote_limit}
 
-      {:reject, reason} ->
+      {:reject, _} = e ->
-        {:error, reason}
+        {:error, e}
     end
   end
 

lib/pleroma/web/activity_pub/object_validator.ex

@@ -12,11 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
@@ -149,10 +151,20 @@ def validate(%{"type" => "Question"} = object, meta) do
     end
   end
 
-  def validate(%{"type" => "Audio"} = object, meta) do
+  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
     with {:ok, object} <-
            object
-           |> AudioValidator.cast_and_validate()
+           |> AudioVideoValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Article"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> ArticleNoteValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
@@ -198,7 +210,7 @@ def validate(
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Event] do
+      when objtype in ~w[Question Answer Audio Video Event Article] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -232,14 +244,18 @@ def cast_and_apply(%{"type" => "Answer"} = object) do
     AnswerValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => "Audio"} = object) do
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do
-    AudioValidator.cast_and_apply(object)
+    AudioVideoValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(%{"type" => "Event"} = object) do
     EventValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Article"} = object) do
+    ArticleNoteValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
@@ -262,7 +278,8 @@ def stringify_keys(object) when is_list(object) do
   def stringify_keys(object), do: object
 
   def fetch_actor(object) do
-    with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
+    with actor <- Containment.get_actor(object),
+         {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
       User.get_or_fetch_by_ap_id(actor)
     end
   end

lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
@@ -25,14 +25,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     # TODO: Write type
     field(:tag, {:array, :map}, default: [])
     field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
     field(:content, :string)
+
     field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
 
     # TODO: Remove actor on objects
     field(:actor, ObjectValidators.ObjectID)
 
     field(:attributedTo, ObjectValidators.ObjectID)
-    field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
     field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
@@ -40,13 +45,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, :string)
+    field(:inReplyTo, ObjectValidators.ObjectID)
     field(:url, ObjectValidators.Uri)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_apply(data) do
@@ -62,19 +65,14 @@ def cast_and_validate(data) do
   end
 
   def cast_data(data) do
+    data = fix(data)
+
     %__MODULE__{}
     |> changeset(data)
   end
 
-  defp fix_url(%{"url" => url} = data) when is_list(url) do
+  defp fix_url(%{"url" => url} = data) when is_map(url) do
-    attachment =
+    Map.put(data, "url", url["href"])
-      Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end)
-
-    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
-
-    data
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
   end
 
   defp fix_url(data), do: data
@@ -83,8 +81,9 @@ defp fix(data) do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
-    |> Transmogrifier.fix_emoji()
+    |> CommonFixes.fix_actor()
     |> fix_url()
+    |> Transmogrifier.fix_emoji()
   end
 
   def changeset(struct, data) do
@@ -97,8 +96,8 @@ def changeset(struct, data) do
 
   def validate_data(data_cng) do
     data_cng
-    |> validate_inclusion(:type, ["Audio"])
+    |> validate_inclusion(:type, ["Article", "Note"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
@@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
 
-    embeds_many(:url, UrlObjectValidator)
+    embeds_many :url, UrlObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:href, ObjectValidators.Uri)
+      field(:mediaType, :string, default: "application/octet-stream")
+    end
   end
 
   def cast_and_validate(data) do
@@ -37,7 +42,18 @@ def changeset(struct, data) do
 
     struct
     |> cast(data, [:type, :mediaType, :name])
-    |> cast_embed(:url, required: true)
+    |> cast_embed(:url, with: &url_changeset/2)
+    |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
+    |> validate_required([:type, :mediaType, :url])
+  end
+
+  def url_changeset(struct, data) do
+    data = fix_media_type(data)
+
+    struct
+    |> cast(data, [:type, :href, :mediaType])
+    |> validate_inclusion(:type, ["Link"])
+    |> validate_required([:type, :href, :mediaType])
   end
 
   def fix_media_type(data) do
@@ -75,6 +91,7 @@ defp fix_url(data) do
 
   def validate_data(cng) do
     cng
+    |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])
   end
 end

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -0,0 +1,134 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EarmarkRenderer
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    field(:emoji, ObjectValidators.Emoji, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_url(%{"url" => url} = data) when is_list(url) do
+    attachment =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"])
+      end)
+
+    link_element =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and mime_type == "text/html"
+      end)
+
+    data
+    |> Map.put("attachment", [attachment])
+    |> Map.put("url", link_element["href"])
+  end
+
+  defp fix_url(data), do: data
+
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
+       when is_binary(content) do
+    content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.put(data, "content", content)
+  end
+
+  defp fix_content(data), do: data
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+    |> CommonFixes.fix_actor()
+    |> Transmogrifier.fix_emoji()
+    |> fix_url()
+    |> fix_content()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Audio", "Video"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.Utils
 
   # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
@@ -19,4 +20,12 @@ def fix_attribution(data) do
     data
     |> Map.put_new("actor", data["attributedTo"])
   end
+
+  def fix_actor(data) do
+    actor = Containment.get_actor(data)
+
+    data
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
+  end
 end

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   import Ecto.Changeset
-  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   @primary_key false
 
@@ -75,14 +76,15 @@ defp fix(data, meta) do
     data
     |> fix_context(meta)
     |> fix_addressing(meta)
+    |> CommonFixes.fix_actor()
   end
 
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_presence()
+    |> CommonValidations.validate_actor_presence()
-    |> validate_any_presence([:to, :cc])
+    |> CommonValidations.validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_context_match(meta)
     |> validate_object_nonexistence()

lib/pleroma/web/activity_pub/object_validators/note_validator.ex

@@ -1,73 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
-    field(:tag, {:array, :map}, default: [])
-    field(:type, :string)
-
-    field(:name, :string)
-    field(:summary, :string)
-    field(:content, :string)
-
-    field(:context, :string)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
-
-    field(:actor, ObjectValidators.ObjectID)
-    field(:attributedTo, ObjectValidators.ObjectID)
-    field(:published, ObjectValidators.DateTime)
-    field(:emoji, ObjectValidators.Emoji, default: %{})
-    field(:sensitive, :boolean, default: false)
-    # TODO: Write type
-    field(:attachment, {:array, :map}, default: [])
-    field(:replies_count, :integer, default: 0)
-    field(:like_count, :integer, default: 0)
-    field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:url, ObjectValidators.Uri)
-
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
-  end
-
-  def cast_and_validate(data) do
-    data
-    |> cast_data()
-    |> validate_data()
-  end
-
-  defp fix(data) do
-    data
-    |> Transmogrifier.fix_emoji()
-  end
-
-  def cast_data(data) do
-    data = fix(data)
-
-    %__MODULE__{}
-    |> cast(data, __schema__(:fields))
-  end
-
-  def validate_data(data_cng) do
-    data_cng
-    |> validate_inclusion(:type, ["Note"])
-    |> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
-  end
-end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
 
     field(:closed, ObjectValidators.DateTime)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])

lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

@@ -1,24 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-
-  import Ecto.Changeset
-  @primary_key false
-
-  embedded_schema do
-    field(:type, :string)
-    field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string, default: "application/octet-stream")
-  end
-
-  def changeset(struct, data) do
-    struct
-    |> cast(data, __schema__(:fields))
-    |> validate_required([:type, :href, :mediaType])
-  end
-end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -336,7 +336,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Question Event] do
+      when objtype in ~w[Audio Video Question Event Article] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
-  alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Maps
   alias Pleroma.Object
@@ -45,7 +44,6 @@ def fix_object(object, options \\ []) do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
-    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -274,24 +272,7 @@ def fix_url(%{"url" => url} = object) when is_map(url) do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
+  def fix_url(%{"url" => url} = object) when is_list(url) do
-    attachment =
-      Enum.find(url, fn x ->
-        media_type = x["mediaType"] || x["mimeType"] || ""
-
-        is_map(x) and String.starts_with?(media_type, "video/")
-      end)
-
-    link_element =
-      Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end)
-
-    object
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
-  end
-
-  def fix_url(%{"type" => object_type, "url" => url} = object)
-      when object_type != "Video" and is_list(url) do
     first_element = Enum.at(url, 0)
 
     url_string =
@@ -371,18 +352,6 @@ def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
 
   def fix_type(object, _), do: object
 
-  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
-       when is_binary(content) do
-    html_content =
-      content
-      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
-      |> Pleroma.HTML.filter_tags()
-
-    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
-  end
-
-  defp fix_content(object), do: object
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -455,7 +424,7 @@ def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Note Video Page} do
+      when objtype in ~w{Note Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -549,7 +518,9 @@ def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Event} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
+    data = Map.put(data, "object", strip_internal_fields(data["object"]))
+
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -23,8 +23,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
-  require Logger
-
   @users_page_size 50
 
   plug(
@@ -68,6 +66,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [:list_user_statuses, :list_instance_statuses]
   )
 
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true}
+    when action in [:list_user_chats]
+  )
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
@@ -256,6 +260,20 @@ def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickna
     end
   end
 
+  def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do
+    with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      chats =
+        Pleroma.Chat.for_user_query(user_id)
+        |> Pleroma.Repo.all()
+
+      conn
+      |> put_view(AdminAPI.ChatView)
+      |> render("index.json", chats: chats)
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
   def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     user = User.get_cached_by_nickname(nickname)
 

lib/pleroma/web/admin_api/controllers/chat_controller.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.Chat
+  alias Pleroma.Chat.MessageReference
+  alias Pleroma.ModerationLog
+  alias Pleroma.Pagination
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true} when action in [:show, :messages]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:chats"], admin: true} when action in [:delete_message]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation
+
+  def delete_message(%{assigns: %{user: user}} = conn, %{
+        message_id: message_id,
+        id: chat_id
+      }) do
+    with %MessageReference{object: %{data: %{"id" => object_ap_id}}} = cm_ref <-
+           MessageReference.get_by_id(message_id),
+         ^chat_id <- to_string(cm_ref.chat_id),
+         %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object_ap_id),
+         {:ok, _} <- CommonAPI.delete(activity_id, user) do
+      ModerationLog.insert_log(%{
+        action: "chat_message_delete",
+        actor: user,
+        subject_id: message_id
+      })
+
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("show.json", chat_message_reference: cm_ref)
+    else
+      _e ->
+        {:error, :could_not_delete}
+    end
+  end
+
+  def messages(conn, %{id: id} = params) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      cm_refs =
+        chat
+        |> MessageReference.for_chat_query()
+        |> Pagination.fetch_paginated(params)
+
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("index.json", chat_message_references: cm_refs)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(%{error: "not found"})
+    end
+  end
+
+  def show(conn, %{id: id}) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      conn
+      |> put_view(AdminAPI.ChatView)
+      |> render("show.json", chat: chat)
+    end
+  end
+end

lib/pleroma/web/admin_api/views/chat_view.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Chat
+  alias Pleroma.User
+  alias Pleroma.Web.MastodonAPI
+  alias Pleroma.Web.PleromaAPI
+
+  def render("index.json", %{chats: chats} = opts) do
+    render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats))
+  end
+
+  def render("show.json", %{chat: %Chat{user_id: user_id}} = opts) do
+    user = User.get_by_id(user_id)
+    sender = MastodonAPI.AccountView.render("show.json", user: user, skip_visibility_check: true)
+
+    serialized_chat = PleromaAPI.ChatView.render("show.json", opts)
+
+    serialized_chat
+    |> Map.put(:sender, sender)
+    |> Map.put(:receiver, serialized_chat[:account])
+    |> Map.delete(:account)
+  end
+
+  def render(view, opts), do: PleromaAPI.ChatView.render(view, opts)
+end

lib/pleroma/web/admin_api/views/status_view.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   require Pleroma.Constants
 
   alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI
 
   defdelegate merge_account_views(user), to: AdminAPI.AccountView
@@ -17,7 +18,7 @@ def render("index.json", opts) do
   end
 
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
-    user = MastodonAPI.StatusView.get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
 
     MastodonAPI.StatusView.render("show.json", opts)
     |> Map.merge(%{account: merge_account_views(user)})

lib/pleroma/web/api_spec/operations/admin/chat_operation.ex

@@ -0,0 +1,96 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
+  alias OpenApiSpex.Operation
+  alias Pleroma.Web.ApiSpec.Schemas.Chat
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def delete_message_operation do
+    %Operation{
+      tags: ["admin", "chat"],
+      summary: "Delete an individual chat message",
+      operationId: "AdminAPI.ChatController.delete_message",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:message_id, :path, :string, "The ID of the message")
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The deleted ChatMessage",
+            "application/json",
+            ChatMessage
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write:chats"]
+        }
+      ]
+    }
+  end
+
+  def messages_operation do
+    %Operation{
+      tags: ["admin", "chat"],
+      summary: "Get the most recent messages of the chat",
+      operationId: "AdminAPI.ChatController.messages",
+      parameters:
+        [Operation.parameter(:id, :path, :string, "The ID of the Chat")] ++
+          pagination_params(),
+      responses: %{
+        200 =>
+          Operation.response(
+            "The messages in the chat",
+            "application/json",
+            Pleroma.Web.ApiSpec.ChatOperation.chat_messages_response()
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read:chats"]
+        }
+      ]
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Create a chat",
+      operationId: "AdminAPI.ChatController.show",
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "The id of the chat",
+          required: true,
+          example: "1234"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The existing chat",
+            "application/json",
+            Chat
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
+    }
+  end
+end

lib/pleroma/web/common_api/common_api.ex

@@ -550,4 +550,21 @@ def hide_reblogs(%User{} = user, %User{} = target) do
   def show_reblogs(%User{} = user, %User{} = target) do
     UserRelationship.delete_reblog_mute(user, target)
   end
+
+  def get_user(ap_id, fake_record_fallback \\ true) do
+    cond do
+      user = User.get_cached_by_ap_id(ap_id) ->
+        user
+
+      user = User.get_by_guessed_nickname(ap_id) ->
+        user
+
+      fake_record_fallback ->
+        # TODO: refactor (fake records is never a good idea)
+        User.error_user(ap_id)
+
+      true ->
+        nil
+    end
+  end
 end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -55,23 +55,6 @@ defp get_replied_to_activities(activities) do
     end)
   end
 
-  def get_user(ap_id, fake_record_fallback \\ true) do
-    cond do
-      user = User.get_cached_by_ap_id(ap_id) ->
-        user
-
-      user = User.get_by_guessed_nickname(ap_id) ->
-        user
-
-      fake_record_fallback ->
-        # TODO: refactor (fake records is never a good idea)
-        User.error_user(ap_id)
-
-      true ->
-        nil
-    end
-  end
-
   defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
     do: context_id
 
@@ -119,7 +102,7 @@ def render("index.json", opts) do
           # Note: unresolved users are filtered out
           actors =
             (activities ++ parent_activities)
-            |> Enum.map(&get_user(&1.data["actor"], false))
+            |> Enum.map(&CommonAPI.get_user(&1.data["actor"], false))
             |> Enum.filter(& &1)
 
           UserRelationship.view_relationships_option(reading_user, actors, subset: :source_mutes)
@@ -138,7 +121,7 @@ def render(
         "show.json",
         %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts
       ) do
-    user = get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
     activity_object = Object.normalize(activity)
 
@@ -211,7 +194,7 @@ def render(
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
     object = Object.normalize(activity)
 
-    user = get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     user_follower_address = user.follower_address
 
     like_count = object.data["like_count"] || 0
@@ -265,7 +248,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
 
     reply_to = get_reply_to(activity, opts)
 
-    reply_to_user = reply_to && get_user(reply_to.data["actor"])
+    reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"])
 
     content =
       object

lib/pleroma/web/pleroma_api/controllers/chat_controller.ex

@@ -146,11 +146,8 @@ def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
     blocked_ap_ids = User.blocked_users_ap_ids(user)
 
     chats =
-      from(c in Chat,
+      Chat.for_user_query(user_id)
-        where: c.user_id == ^user_id,
+      |> where([c], c.recipient not in ^blocked_ap_ids)
-        where: c.recipient not in ^blocked_ap_ids,
-        order_by: [desc: c.updated_at]
-      )
       |> Repo.all()
 
     conn

lib/pleroma/web/pleroma_api/views/scrobble_view.ex

@@ -10,14 +10,14 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
   alias Pleroma.Activity
   alias Pleroma.HTML
   alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.MastodonAPI.StatusView
 
   def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do
     object = Object.normalize(activity)
 
-    user = StatusView.get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
 
     %{

lib/pleroma/web/rich_media/parser.ex

@@ -20,35 +20,60 @@ def parse(url) do
       with {:ok, data} <- get_cached_or_parse(url),
            {:ok, _} <- set_ttl_based_on_image(data, url) do
         {:ok, data}
-      else
-        {:error, {:invalid_metadata, data}} = e ->
-          Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
-          e
-
-        error ->
-          Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end)
-          error
       end
     end
 
     defp get_cached_or_parse(url) do
-      case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
+      case Cachex.fetch(:rich_media_cache, url, fn ->
+             case parse_url(url) do
+               {:ok, _} = res ->
+                 {:commit, res}
+
+               {:error, reason} = e ->
+                 # Unfortunately we have to log errors here, instead of doing that
+                 # along with ttl setting at the bottom. Otherwise we can get log spam
+                 # if more than one process was waiting for the rich media card
+                 # while it was generated. Ideally we would set ttl here as well,
+                 # so we don't override it number_of_waiters_on_generation
+                 # times, but one, obviously, can't set ttl for not-yet-created entry
+                 # and Cachex doesn't support returning ttl from the fetch callback.
+                 log_error(url, reason)
+                 {:commit, e}
+             end
+           end) do
+        {action, res} when action in [:commit, :ok] ->
+          case res do
             {:ok, _data} = res ->
               res
 
-        {:error, :body_too_large} = e ->
+            {:error, reason} = e ->
+              if action == :commit, do: set_error_ttl(url, reason)
               e
+          end
 
-        {:error, {:content_type, _}} = e ->
+        {:error, e} ->
-          e
+          {:error, {:cachex_error, e}}
+      end
+    end
+
+    defp set_error_ttl(_url, :body_too_large), do: :ok
+    defp set_error_ttl(_url, {:content_type, _}), do: :ok
 
     # The TTL is not set for the errors above, since they are unlikely to change
     # with time
-        {:error, _} = e ->
+
+    defp set_error_ttl(url, _reason) do
       ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
       Cachex.expire(:rich_media_cache, url, ttl)
-          e
+      :ok
     end
+
+    defp log_error(url, {:invalid_metadata, data}) do
+      Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
+    end
+
+    defp log_error(url, reason) do
+      Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
     end
   end
 

lib/pleroma/web/router.ex

@@ -178,6 +178,7 @@ defmodule Pleroma.Web.Router do
     get("/users", AdminAPIController, :list_users)
     get("/users/:nickname", AdminAPIController, :user_show)
     get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses)
+    get("/users/:nickname/chats", AdminAPIController, :list_user_chats)
 
     get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses)
 
@@ -214,6 +215,10 @@ defmodule Pleroma.Web.Router do
     get("/media_proxy_caches", MediaProxyCacheController, :index)
     post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
     post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
+
+    get("/chats/:id", ChatController, :show)
+    get("/chats/:id/messages", ChatController, :messages)
+    delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
   end
 
   scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do

priv/repo/migrations/20200914105638_delete_notification_without_activity.exs

@@ -0,0 +1,30 @@
+defmodule Pleroma.Repo.Migrations.DeleteNotificationWithoutActivity do
+  use Ecto.Migration
+
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    from(
+      q in Pleroma.Notification,
+      left_join: c in assoc(q, :activity),
+      select: %{id: type(q.id, :integer)},
+      where: is_nil(c.id)
+    )
+    |> Repo.chunk_stream(1_000, :batches)
+    |> Stream.each(fn records ->
+      notification_ids = Enum.map(records, fn %{id: id} -> id end)
+
+      Repo.delete_all(
+        from(n in "notifications",
+          where: n.id in ^notification_ids
+        )
+      )
+    end)
+    |> Stream.run()
+  end
+
+  def down do
+    :ok
+  end
+end

priv/repo/migrations/20200914105800_add_notification_constraints.exs

@@ -0,0 +1,23 @@
+defmodule Pleroma.Repo.Migrations.AddNotificationConstraints do
+  use Ecto.Migration
+
+  def up do
+    drop(constraint(:notifications, "notifications_activity_id_fkey"))
+
+    alter table(:notifications) do
+      modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all),
+        null: false
+      )
+    end
+  end
+
+  def down do
+    drop(constraint(:notifications, "notifications_activity_id_fkey"))
+
+    alter table(:notifications) do
+      modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all),
+        null: true
+      )
+    end
+  end
+end

test/fixtures/tesla_mock/wedistribute-create-article.json

@@ -0,0 +1 @@
+{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Create","actor":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","object":{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Article","name":"The end is near: Mastodon plans to drop OStatus support","content":"<!-- wp:paragraph {\"dropCap\":true} -->\n<p class=\"has-drop-cap\">The days of OStatus are numbered. The venerable protocol has served as a glue between many different types of servers since the early days of the Fediverse, connecting StatusNet (now GNU Social) to Friendica, Hubzilla, Mastodon, and Pleroma.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Now that many fediverse platforms support ActivityPub as a successor protocol, Mastodon appears to be drawing a line in the sand. In <a href=\"https:\/\/www.patreon.com\/posts\/mastodon-2-9-and-28121681\">a Patreon update<\/a>, Eugen Rochko writes:<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:quote -->\n<blockquote class=\"wp-block-quote\"><p>...OStatus...has overstayed its welcome in the code...and now that most of the network uses ActivityPub, it's time for it to go. <\/p><cite>Eugen Rochko, Mastodon creator<\/cite><\/blockquote>\n<!-- \/wp:quote -->\n\n<!-- wp:paragraph -->\n<p>The <a href=\"https:\/\/github.com\/tootsuite\/mastodon\/pull\/11205\">pull request<\/a> to remove Pubsubhubbub and Salmon, two of the main components of OStatus, has already been merged into Mastodon's master branch.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Some projects will be left in the dark as a side effect of this. GNU Social and PostActiv, for example, both only communicate using OStatus. While <a href=\"https:\/\/mastodon.social\/@dansup\/102076573310057902\">some discussion<\/a> exists regarding adopting ActivityPub for GNU Social, and <a href=\"https:\/\/notabug.org\/diogo\/gnu-social\/src\/activitypub\/plugins\/ActivityPub\">a plugin is in development<\/a>, it hasn't been formally adopted yet. We just hope that the <a href=\"https:\/\/status.fsf.org\/main\/public\">Free Software Foundation's instance<\/a> gets updated in time!<\/p>\n<!-- \/wp:paragraph -->","summary":"One of the largest platforms in the federated social web is dropping the protocol that it started with.","attributedTo":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","url":"https:\/\/wedistribute.org\/2019\/07\/mastodon-drops-ostatus\/","to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810","likes":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/likes","shares":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/shares"},"to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85809"}

test/marker_test.exs

@@ -33,8 +33,8 @@ test "return empty multi" do
     test "returns user markers" do
       user = insert(:user)
       marker = insert(:marker, user: user)
-      insert(:notification, user: user)
+      insert(:notification, user: user, activity: insert(:note_activity))
-      insert(:notification, user: user)
+      insert(:notification, user: user, activity: insert(:note_activity))
       insert(:marker, timeline: "home", user: user)
 
       assert Marker.get_markers(

test/object/fetcher_test.exs

@@ -6,10 +6,12 @@ defmodule Pleroma.Object.FetcherTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
-  import Tesla.Mock
+
   import Mock
+  import Tesla.Mock
 
   setup do
     mock(fn
@@ -71,20 +73,20 @@ test "it works when fetching the OP actor errors out" do
     setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
 
     test "it returns thread depth exceeded error if thread depth is exceeded" do
-      Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0)
+      Config.put([:instance, :federation_incoming_replies_max_depth], 0)
 
       assert {:error, "Max thread distance exceeded."} =
                Fetcher.fetch_object_from_id(@ap_id, depth: 1)
     end
 
     test "it fetches object if max thread depth is restricted to 0 and depth is not specified" do
-      Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0)
+      Config.put([:instance, :federation_incoming_replies_max_depth], 0)
 
       assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id)
     end
 
     test "it fetches object if requested depth does not exceed max thread depth" do
-      Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 10)
+      Config.put([:instance, :federation_incoming_replies_max_depth], 10)
 
       assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id, depth: 10)
     end
@@ -120,6 +122,16 @@ test "it fetches an object" do
 
       assert object == object_again
     end
+
+    test "Return MRF reason when fetched status is rejected by one" do
+      clear_config([:mrf_keyword, :reject], ["yeah"])
+      clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
+
+      assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
+               Fetcher.fetch_object_from_id(
+                 "http://mastodon.example.org/@admin/99541947525187367"
+               )
+    end
   end
 
   describe "implementation quirks" do
@@ -212,7 +224,7 @@ test "it can refetch pruned objects" do
                    Pleroma.Signature,
                    [:passthrough],
                    [] do
-      Pleroma.Config.put([:activitypub, :sign_object_fetches], true)
+      Config.put([:activitypub, :sign_object_fetches], true)
 
       Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
 
@@ -223,7 +235,7 @@ test "it can refetch pruned objects" do
                    Pleroma.Signature,
                    [:passthrough],
                    [] do
-      Pleroma.Config.put([:activitypub, :sign_object_fetches], false)
+      Config.put([:activitypub, :sign_object_fetches], false)
 
       Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
 

test/repo_test.exs

@@ -37,7 +37,9 @@ test "get one-to-one assoc from repo" do
 
     test "get one-to-many assoc from repo" do
       user = insert(:user)
-      notification = refresh_record(insert(:notification, user: user))
+
+      notification =
+        refresh_record(insert(:notification, user: user, activity: insert(:note_activity)))
 
       assert Repo.get_assoc(user, :notifications) == {:ok, [notification]}
     end
@@ -47,4 +49,32 @@ test "return error if has not assoc " do
       assert Repo.get_assoc(token, :user) == {:error, :not_found}
     end
   end
+
+  describe "chunk_stream/3" do
+    test "fetch records one-by-one" do
+      users = insert_list(50, :user)
+
+      {fetch_users, 50} =
+        from(t in User)
+        |> Repo.chunk_stream(5)
+        |> Enum.reduce({[], 0}, fn %User{} = user, {acc, count} ->
+          {acc ++ [user], count + 1}
+        end)
+
+      assert users == fetch_users
+    end
+
+    test "fetch records in bulk" do
+      users = insert_list(50, :user)
+
+      {fetch_users, 10} =
+        from(t in User)
+        |> Repo.chunk_stream(5, :batches)
+        |> Enum.reduce({[], 0}, fn users, {acc, count} ->
+          {acc ++ users, count + 1}
+        end)
+
+      assert users == fetch_users
+    end
+  end
 end

test/user_test.exs

@@ -1676,7 +1676,7 @@ test "returns true when the account is itself" do
       assert User.visible_for(user, user) == :visible
     end
 
-    test "returns false when the account is unauthenticated and auth is required" do
+    test "returns false when the account is unconfirmed and confirmation is required" do
       Pleroma.Config.put([:instance, :account_activation_required], true)
 
       user = insert(:user, local: true, confirmation_pending: true)
@@ -1685,14 +1685,23 @@ test "returns false when the account is unauthenticated and auth is required" do
       refute User.visible_for(user, other_user) == :visible
     end
 
-    test "returns true when the account is unauthenticated and auth is not required" do
+    test "returns true when the account is unconfirmed and confirmation is required but the account is remote" do
+      Pleroma.Config.put([:instance, :account_activation_required], true)
+
+      user = insert(:user, local: false, confirmation_pending: true)
+      other_user = insert(:user, local: true)
+
+      assert User.visible_for(user, other_user) == :visible
+    end
+
+    test "returns true when the account is unconfirmed and confirmation is not required" do
       user = insert(:user, local: true, confirmation_pending: true)
       other_user = insert(:user, local: true)
 
       assert User.visible_for(user, other_user) == :visible
     end
 
-    test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do
+    test "returns true when the account is unconfirmed and being viewed by a privileged account (confirmation required)" do
       Pleroma.Config.put([:instance, :account_activation_required], true)
 
       user = insert(:user, local: true, confirmation_pending: true)

test/web/activity_pub/object_validators/article_note_validator_test.exs

@@ -2,10 +2,10 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do
   use Pleroma.DataCase
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
   alias Pleroma.Web.ActivityPub.Utils
 
   import Pleroma.Factory
@@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
     end
 
     test "a basic note validates", %{note: note} do
-      %{valid?: true} = NoteValidator.cast_and_validate(note)
+      %{valid?: true} = ArticleNoteValidator.cast_and_validate(note)
     end
   end
 end

test/web/activity_pub/transmogrifier/article_handling_test.exs

@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  test "Pterotype (Wordpress Plugin) Article" do
+    Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} ->
+      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")}
+    end)
+
+    data =
+      File.read!("test/fixtures/tesla_mock/wedistribute-create-article.json") |> Jason.decode!()
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+    object = Object.normalize(data["object"])
+
+    assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support"
+
+    assert object.data["summary"] ==
+             "One of the largest platforms in the federated social web is dropping the protocol that it started with."
+
+    assert object.data["url"] == "https://wedistribute.org/2019/07/mastodon-drops-ostatus/"
+  end
+
+  test "Plume Article" do
+    Tesla.Mock.mock(fn
+      %{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} ->
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
+        }
+
+      %{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} ->
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
+        }
+    end)
+
+    {:ok, object} =
+      Fetcher.fetch_object_from_id(
+        "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
+      )
+
+    assert object.data["name"] == "This Month in Plume: June 2018"
+
+    assert object.data["url"] ==
+             "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
+  end
+
+  test "Prismo Article" do
+    Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} ->
+      %Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
+      }
+    end)
+
+    data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!()
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+    object = Object.normalize(data["object"])
+
+    assert object.data["url"] == "https://prismo.news/posts/83"
+  end
+end

test/web/activity_pub/transmogrifier/video_handling_test.exs

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "skip converting the content when it is nil" do
+    data =
+      File.read!("test/fixtures/tesla_mock/framatube.org-video.json")
+      |> Jason.decode!()
+      |> Kernel.put_in(["object", "content"], nil)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["content"] == nil
+  end
+
+  test "it converts content of object to html" do
+    data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["content"] ==
+             "<p>Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?</p><p>Transcription par @aprilorg ici : <a href=\"https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft\">https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft</a></p>"
+  end
+
+  test "it remaps video URLs as attachments if necessary" do
+    {:ok, object} =
+      Fetcher.fetch_object_from_id(
+        "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
+      )
+
+    assert object.data["url"] ==
+             "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
+
+    assert object.data["attachment"] == [
+             %{
+               "type" => "Link",
+               "mediaType" => "video/mp4",
+               "name" => nil,
+               "url" => [
+                 %{
+                   "href" =>
+                     "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
+                   "mediaType" => "video/mp4",
+                   "type" => "Link"
+                 }
+               ]
+             }
+           ]
+
+    data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["attachment"] == [
+             %{
+               "type" => "Link",
+               "mediaType" => "video/mp4",
+               "name" => nil,
+               "url" => [
+                 %{
+                   "href" =>
+                     "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
+                   "mediaType" => "video/mp4",
+                   "type" => "Link"
+                 }
+               ]
+             }
+           ]
+
+    assert object.data["url"] ==
+             "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
 
   alias Pleroma.Activity
   alias Pleroma.Object
-  alias Pleroma.Object.Fetcher
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -45,15 +44,6 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
       assert "test" in object.data["tag"]
     end
 
-    test "it works for incoming notices with url not being a string (prismo)" do
-      data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
-
-      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
-
-      assert object.data["url"] == "https://prismo.news/posts/83"
-    end
-
     test "it cleans up incoming notices which are not really DMs" do
       user = insert(:user)
       other_user = insert(:user)
@@ -355,83 +345,6 @@ test "it works for incoming unfollows with an existing follow" do
       refute User.following?(User.get_cached_by_ap_id(data["actor"]), user)
     end
 
-    test "skip converting the content when it is nil" do
-      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
-
-      {:ok, object} = Fetcher.fetch_and_contain_remote_object_from_id(object_id)
-
-      result =
-        Pleroma.Web.ActivityPub.Transmogrifier.fix_object(Map.merge(object, %{"content" => nil}))
-
-      assert result["content"] == nil
-    end
-
-    test "it converts content of object to html" do
-      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
-
-      {:ok, %{"content" => content_markdown}} =
-        Fetcher.fetch_and_contain_remote_object_from_id(object_id)
-
-      {:ok, %Pleroma.Object{data: %{"content" => content}} = object} =
-        Fetcher.fetch_object_from_id(object_id)
-
-      assert content_markdown ==
-               "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've..."
-
-      assert content ==
-               "<p>Support this and our other Michigan!/usr/group videos and meetings. Learn more at <a href=\"http://mug.org/membership\">http://mug.org/membership</a></p><p>Twenty Years in Jail: FreeBSD’s Jails, Then and Now</p><p>Jails started as a limited virtualization system, but over the last two years they’ve…</p>"
-
-      assert object.data["mediaType"] == "text/html"
-    end
-
-    test "it remaps video URLs as attachments if necessary" do
-      {:ok, object} =
-        Fetcher.fetch_object_from_id(
-          "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-        )
-
-      assert object.data["url"] ==
-               "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-
-      assert object.data["attachment"] == [
-               %{
-                 "type" => "Link",
-                 "mediaType" => "video/mp4",
-                 "url" => [
-                   %{
-                     "href" =>
-                       "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                     "mediaType" => "video/mp4",
-                     "type" => "Link"
-                   }
-                 ]
-               }
-             ]
-
-      {:ok, object} =
-        Fetcher.fetch_object_from_id(
-          "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
-        )
-
-      assert object.data["attachment"] == [
-               %{
-                 "type" => "Link",
-                 "mediaType" => "video/mp4",
-                 "url" => [
-                   %{
-                     "href" =>
-                       "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
-                     "mediaType" => "video/mp4",
-                     "type" => "Link"
-                   }
-                 ]
-               }
-             ]
-
-      assert object.data["url"] ==
-               "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
-    end
-
     test "it accepts Flag activities" do
       user = insert(:user)
       other_user = insert(:user)
@@ -1133,75 +1046,7 @@ test "fixes data for object when url is map" do
              }
     end
 
-    test "fixes data for video object" do
+    test "returns non-modified object" do
-      object = %{
-        "type" => "Video",
-        "url" => [
-          %{
-            "type" => "Link",
-            "mimeType" => "video/mp4",
-            "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "video/mp4",
-            "href" => "https://peertube46fb-ad81-2d4c2d1630e3-240.mp4"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d1630e3"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d16377-42"
-          }
-        ]
-      }
-
-      assert Transmogrifier.fix_url(object) == %{
-               "attachment" => [
-                 %{
-                   "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                   "mimeType" => "video/mp4",
-                   "type" => "Link"
-                 }
-               ],
-               "type" => "Video",
-               "url" => "https://peertube.-2d4c2d1630e3"
-             }
-    end
-
-    test "fixes url for not Video object" do
-      object = %{
-        "type" => "Text",
-        "url" => [
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d1630e3"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d16377-42"
-          }
-        ]
-      }
-
-      assert Transmogrifier.fix_url(object) == %{
-               "type" => "Text",
-               "url" => "https://peertube.-2d4c2d1630e3"
-             }
-
-      assert Transmogrifier.fix_url(%{"type" => "Text", "url" => []}) == %{
-               "type" => "Text",
-               "url" => ""
-             }
-    end
-
-    test "retunrs not modified object" do
       assert Transmogrifier.fix_url(%{"type" => "Text"}) == %{"type" => "Text"}
     end
   end

test/web/admin_api/controllers/admin_api_controller_test.exs

@@ -1510,6 +1510,56 @@ test "excludes reblogs by default", %{conn: conn, user: user} do
     end
   end
 
+  describe "GET /api/pleroma/admin/users/:nickname/chats" do
+    setup do
+      user = insert(:user)
+      recipients = insert_list(3, :user)
+
+      Enum.each(recipients, fn recipient ->
+        CommonAPI.post_chat_message(user, recipient, "yo")
+      end)
+
+      %{user: user}
+    end
+
+    test "renders user's chats", %{conn: conn, user: user} do
+      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}/chats")
+
+      assert json_response(conn, 200) |> length() == 3
+    end
+  end
+
+  describe "GET /api/pleroma/admin/users/:nickname/chats unauthorized" do
+    setup do
+      user = insert(:user)
+      recipient = insert(:user)
+      CommonAPI.post_chat_message(user, recipient, "yo")
+      %{conn: conn} = oauth_access(["read:chats"])
+      %{conn: conn, user: user}
+    end
+
+    test "returns 403", %{conn: conn, user: user} do
+      conn
+      |> get("/api/pleroma/admin/users/#{user.nickname}/chats")
+      |> json_response(403)
+    end
+  end
+
+  describe "GET /api/pleroma/admin/users/:nickname/chats unauthenticated" do
+    setup do
+      user = insert(:user)
+      recipient = insert(:user)
+      CommonAPI.post_chat_message(user, recipient, "yo")
+      %{conn: build_conn(), user: user}
+    end
+
+    test "returns 403", %{conn: conn, user: user} do
+      conn
+      |> get("/api/pleroma/admin/users/#{user.nickname}/chats")
+      |> json_response(403)
+    end
+  end
+
   describe "GET /api/pleroma/admin/moderation_log" do
     setup do
       moderator = insert(:user, is_moderator: true)

test/web/admin_api/controllers/chat_controller_test.exs

@@ -0,0 +1,219 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatControllerTest do
+  use Pleroma.Web.ConnCase
+
+  import Pleroma.Factory
+
+  alias Pleroma.Chat
+  alias Pleroma.Chat.MessageReference
+  alias Pleroma.Config
+  alias Pleroma.ModerationLog
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Web.CommonAPI
+
+  defp admin_setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "DELETE /api/pleroma/admin/chats/:id/messages/:message_id" do
+    setup do: admin_setup()
+
+    test "it deletes a message from the chat", %{conn: conn, admin: admin} do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, message} =
+        CommonAPI.post_chat_message(user, recipient, "Hello darkness my old friend")
+
+      object = Object.normalize(message, false)
+
+      chat = Chat.get(user.id, recipient.ap_id)
+      recipient_chat = Chat.get(recipient.id, user.ap_id)
+
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
+      recipient_cm_ref = MessageReference.for_chat_and_object(recipient_chat, object)
+
+      result =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}")
+        |> json_response_and_validate_schema(200)
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} deleted chat message ##{cm_ref.id}"
+
+      assert result["id"] == cm_ref.id
+      refute MessageReference.get_by_id(cm_ref.id)
+      refute MessageReference.get_by_id(recipient_cm_ref.id)
+      assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id)
+    end
+  end
+
+  describe "GET /api/pleroma/admin/chats/:id/messages" do
+    setup do: admin_setup()
+
+    test "it paginates", %{conn: conn} do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      Enum.each(1..30, fn _ ->
+        {:ok, _} = CommonAPI.post_chat_message(user, recipient, "hey")
+      end)
+
+      chat = Chat.get(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/chats/#{chat.id}/messages")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 20
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/chats/#{chat.id}/messages?max_id=#{List.last(result)["id"]}")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 10
+    end
+
+    test "it returns the messages for a given chat", %{conn: conn} do
+      user = insert(:user)
+      other_user = insert(:user)
+      third_user = insert(:user)
+
+      {:ok, _} = CommonAPI.post_chat_message(user, other_user, "hey")
+      {:ok, _} = CommonAPI.post_chat_message(user, third_user, "hey")
+      {:ok, _} = CommonAPI.post_chat_message(user, other_user, "how are you?")
+      {:ok, _} = CommonAPI.post_chat_message(other_user, user, "fine, how about you?")
+
+      chat = Chat.get(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/chats/#{chat.id}/messages")
+        |> json_response_and_validate_schema(200)
+
+      result
+      |> Enum.each(fn message ->
+        assert message["chat_id"] == chat.id |> to_string()
+      end)
+
+      assert length(result) == 3
+    end
+  end
+
+  describe "GET /api/pleroma/admin/chats/:id" do
+    setup do: admin_setup()
+
+    test "it returns a chat", %{conn: conn} do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/chats/#{chat.id}")
+        |> json_response_and_validate_schema(200)
+
+      assert result["id"] == to_string(chat.id)
+      assert %{} = result["sender"]
+      assert %{} = result["receiver"]
+      refute result["account"]
+    end
+  end
+
+  describe "unauthorized chat moderation" do
+    setup do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo")
+      object = Object.normalize(message, false)
+      chat = Chat.get(user.id, recipient.ap_id)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
+
+      %{conn: conn} = oauth_access(["read:chats", "write:chats"])
+      %{conn: conn, chat: chat, cm_ref: cm_ref}
+    end
+
+    test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{
+      conn: conn,
+      chat: chat,
+      cm_ref: cm_ref
+    } do
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}")
+      |> json_response(403)
+
+      assert MessageReference.get_by_id(cm_ref.id) == cm_ref
+    end
+
+    test "GET /api/pleroma/admin/chats/:id/messages", %{conn: conn, chat: chat} do
+      conn
+      |> get("/api/pleroma/admin/chats/#{chat.id}/messages")
+      |> json_response(403)
+    end
+
+    test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do
+      conn
+      |> get("/api/pleroma/admin/chats/#{chat.id}")
+      |> json_response(403)
+    end
+  end
+
+  describe "unauthenticated chat moderation" do
+    setup do
+      user = insert(:user)
+      recipient = insert(:user)
+
+      {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo")
+      object = Object.normalize(message, false)
+      chat = Chat.get(user.id, recipient.ap_id)
+      cm_ref = MessageReference.for_chat_and_object(chat, object)
+
+      %{conn: build_conn(), chat: chat, cm_ref: cm_ref}
+    end
+
+    test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{
+      conn: conn,
+      chat: chat,
+      cm_ref: cm_ref
+    } do
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}")
+      |> json_response(403)
+
+      assert MessageReference.get_by_id(cm_ref.id) == cm_ref
+    end
+
+    test "GET /api/pleroma/admin/chats/:id/messages", %{conn: conn, chat: chat} do
+      conn
+      |> get("/api/pleroma/admin/chats/#{chat.id}/messages")
+      |> json_response(403)
+    end
+
+    test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do
+      conn
+      |> get("/api/pleroma/admin/chats/#{chat.id}")
+      |> json_response(403)
+    end
+  end
+end

test/web/common_api/common_api_test.exs

@@ -1193,4 +1193,24 @@ test "respects visibility=private" do
       assert Visibility.get_visibility(activity) == "private"
     end
   end
+
+  describe "get_user/1" do
+    test "gets user by ap_id" do
+      user = insert(:user)
+      assert CommonAPI.get_user(user.ap_id) == user
+    end
+
+    test "gets user by guessed nickname" do
+      user = insert(:user, ap_id: "", nickname: "mario@mushroom.kingdom")
+      assert CommonAPI.get_user("https://mushroom.kingdom/users/mario") == user
+    end
+
+    test "fallback" do
+      assert %User{
+               name: "",
+               ap_id: "",
+               nickname: "erroruser@example.com"
+             } = CommonAPI.get_user("")
+    end
+  end
 end

test/web/mastodon_api/controllers/account_controller_test.exs

@@ -1442,7 +1442,10 @@ test "returns lists to which the account belongs" do
   describe "verify_credentials" do
     test "verify_credentials" do
       %{user: user, conn: conn} = oauth_access(["read:accounts"])
-      [notification | _] = insert_list(7, :notification, user: user)
+
+      [notification | _] =
+        insert_list(7, :notification, user: user, activity: insert(:note_activity))
+
       Pleroma.Notification.set_read_up_to(user, notification.id)
       conn = get(conn, "/api/v1/accounts/verify_credentials")
 

test/web/mastodon_api/controllers/marker_controller_test.exs

@@ -11,7 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
     test "gets markers with correct scopes", %{conn: conn} do
       user = insert(:user)
       token = insert(:oauth_token, user: user, scopes: ["read:statuses"])
-      insert_list(7, :notification, user: user)
+      insert_list(7, :notification, user: user, activity: insert(:note_activity))
 
       {:ok, %{"notifications" => marker}} =
         Pleroma.Marker.upsert(

test/web/mastodon_api/views/account_view_test.exs

@@ -449,7 +449,7 @@ test "shows unread_conversation_count only to the account owner" do
 
     test "shows unread_count only to the account owner" do
       user = insert(:user)
-      insert_list(7, :notification, user: user)
+      insert_list(7, :notification, user: user, activity: insert(:note_activity))
       other_user = insert(:user)
 
       user = User.get_cached_by_ap_id(user.ap_id)