Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitlab-ci.yml

@@ -8,7 +8,9 @@ variables: &global_variables
   MIX_ENV: test
 
 cache: &global_cache_policy
-  key: ${CI_COMMIT_REF_SLUG}
+  key:
+    files:
+      - mix.lock
   paths:
     - deps
     - _build
@@ -22,6 +24,7 @@ stages:
   - docker
 
 before_script:
+  - rm -rf _build/*/lib/pleroma
   - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
@@ -29,6 +32,9 @@ before_script:
   - apt-get -qq update
   - apt-get install -y libmagic-dev
 
+after_script:
+  - rm -rf _build/*/lib/pleroma
+
 build:
   stage: build
   script:

CHANGELOG.md

@@ -6,13 +6,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+
 - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
+- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
+
+### Added
+
+- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
+- Return OAuth token `id` (primary key) in POST `/oauth/token`.
+
+### Fixed
+- Don't crash so hard when email settings are invalid.
 
 ## Unreleased (Patch)
 
 ### Fixed
 
 - Try to save exported ConfigDB settings (migrate_from_db) in the system temp directory if default location is not writable.
+- Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image
+- Applying ConcurrentLimiter settings via AdminAPI
+- User login failures if their `notification_settings` were in a NULL state.
+- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
+- Fixed some Markdown issues, including trailing slash in links.
 
 ## [2.3.0] - 2020-03-01
 

config/config.exs

@@ -412,6 +412,8 @@
   threshold: 604_800,
   actions: [:delist, :strip_followers]
 
+config :pleroma, :mrf_follow_bot, follower_nickname: nil
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],

config/description.exs

@@ -2942,6 +2942,23 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :mrf_follow_bot,
+    tab: :mrf,
+    related_policy: "Pleroma.Web.ActivityPub.MRF.FollowBotPolicy",
+    label: "MRF FollowBot Policy",
+    type: :group,
+    description: "Automatically follows newly discovered accounts.",
+    children: [
+      %{
+        key: :follower_nickname,
+        type: :string,
+        description: "The name of the bot account to use for following newly discovered users.",
+        suggestions: ["followbot"]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :modules,

docs/configuration/cheatsheet.md

@@ -124,6 +124,7 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
     * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
     * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
+    * `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
 
@@ -220,6 +221,11 @@ Notes:
 - The hashtags in the configuration do not have a leading `#`.
 - This MRF Policy is always enabled, if you want to disable it you have to set empty lists
 
+#### :mrf_follow_bot
+
+* `follower_nickname`: The name of the bot account to use for following newly discovered users. Using `followbot` or similar is strongly suggested.
+
+
 ### :activitypub
 * `unfollow_blocked`: Whether blocks result in people getting unfollowed
 * `outgoing_blocks`: Whether to federate blocks to other instances

docs/development/API/differences_in_mastoapi_responses.md

@@ -38,6 +38,7 @@ Has these additional fields under the `pleroma` object:
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
 - `parent_visible`: If the parent of this post is visible to the user or not.
+- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
 
 ## Scheduled statuses
 
@@ -255,9 +256,29 @@ This information is returned in the `/api/v1/accounts/verify_credentials` endpoi
 
 *Pleroma supports refreshing tokens.*
 
-`POST /oauth/token`
+### POST `/oauth/token`
 
-Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token.
+You can obtain access tokens for a user in a few additional ways.
+
+#### Refreshing a token
+
+To obtain a new access token from a refresh token, pass `grant_type=refresh_token` with the following extra parameters:
+
+- `refresh_token`: The refresh token.
+
+#### Getting a token with a password
+
+To obtain a token from a user's password, pass `grant_type=password` with the following extra parameters:
+
+- `username`: Username to authenticate.
+- `password`: The user's password.
+
+#### Response body
+
+Additional fields are returned in the response:
+
+- `id`: The primary key of this token in Pleroma's database.
+- `me` (user tokens only): The ActivityPub ID of the user who owns the token.
 
 ## Account Registration
 

docs/index.md

@@ -20,7 +20,7 @@ The default front-end used by Pleroma is Pleroma-FE. You can find more informati
 
 ### Mastodon interface
 If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
-Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
+Just add a "/web" after your instance url (e.g. <https://pleroma.soykaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
 The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
 
 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

docs/installation/otp_en.md

@@ -290,7 +290,7 @@ nginx -t
 
 ## Create your first user and set as admin
 ```sh
-cd /opt/pleroma/bin
+cd /opt/pleroma
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
 ```
 This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.

lib/pleroma/activity.ex

@@ -184,40 +184,48 @@ def get_by_ap_id_with_object(ap_id) do
     |> Repo.one()
   end
 
-  @spec get_by_id(String.t()) :: Activity.t() | nil
+  @doc """
-  def get_by_id(id) do
+  Gets activity by ID, doesn't load activities from deactivated actors by default.
-    case FlakeId.flake_id?(id) do
+  """
-      true ->
+  @spec get_by_id(String.t(), keyword()) :: t() | nil
-        Activity
+  def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts)
-        |> where([a], a.id == ^id)
-        |> restrict_deactivated_users()
-        |> Repo.one()
 
-      _ ->
+  @spec get_by_id_with_user_actor(String.t()) :: t() | nil
-        nil
+  def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor])
+
+  @spec get_by_id_with_object(String.t()) :: t() | nil
+  def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object])
+
+  defp get_by_id_with_opts(id, opts) do
+    if FlakeId.flake_id?(id) do
+      query = Queries.by_id(id)
+
+      with_filters_query =
+        if is_list(opts[:filter]) do
+          Enum.reduce(opts[:filter], query, fn
+            {:type, type}, acc -> Queries.by_type(acc, type)
+            :restrict_deactivated, acc -> restrict_deactivated_users(acc)
+            _, acc -> acc
+          end)
+        else
+          query
+        end
+
+      with_preloads_query =
+        if is_list(opts[:preload]) do
+          Enum.reduce(opts[:preload], with_filters_query, fn
+            :user_actor, acc -> with_preloaded_user_actor(acc)
+            :object, acc -> with_preloaded_object(acc)
+            _, acc -> acc
+          end)
+        else
+          with_filters_query
+        end
+
+      Repo.one(with_preloads_query)
     end
   end
 
-  def get_by_id_with_user_actor(id) do
-    case FlakeId.flake_id?(id) do
-      true ->
-        Activity
-        |> where([a], a.id == ^id)
-        |> with_preloaded_user_actor()
-        |> Repo.one()
-
-      _ ->
-        nil
-    end
-  end
-
-  def get_by_id_with_object(id) do
-    Activity
-    |> where(id: ^id)
-    |> with_preloaded_object()
-    |> Repo.one()
-  end
-
   def all_by_ids_with_object(ids) do
     Activity
     |> where([a], a.id in ^ids)
@@ -269,6 +277,11 @@ def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
 
   def get_create_by_object_ap_id_with_object(_), do: nil
 
+  @spec create_by_id_with_object(String.t()) :: t() | nil
+  def create_by_id_with_object(id) do
+    get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"])
+  end
+
   defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
     get_create_by_object_ap_id_with_object(ap_id)
   end
@@ -368,12 +381,6 @@ def direct_conversation_id(activity, for_user) do
     end
   end
 
-  @spec pinned_by_actor?(Activity.t()) :: boolean()
-  def pinned_by_actor?(%Activity{} = activity) do
-    actor = user_actor(activity)
-    activity.id in actor.pinned_activities
-  end
-
   @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
   def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
     ap_id
@@ -384,4 +391,13 @@ def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
   end
 
   def get_by_object_ap_id_with_object(_), do: nil
+
+  @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t()
+  def add_by_params_query(object_id, actor, target) do
+    object_id
+    |> Queries.by_object_id()
+    |> Queries.by_type("Add")
+    |> Queries.by_actor(actor)
+    |> where([a], fragment("?->>'target' = ?", a.data, ^target))
+  end
 end

lib/pleroma/activity/queries.ex

@@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do
   alias Pleroma.Activity
   alias Pleroma.User
 
+  @spec by_id(query(), String.t()) :: query()
+  def by_id(query \\ Activity, id) do
+    from(a in query, where: a.id == ^id)
+  end
+
   @spec by_ap_id(query, String.t()) :: query
   def by_ap_id(query \\ Activity, ap_id) do
     from(

lib/pleroma/application_requirements.ex

@@ -34,15 +34,16 @@ defp handle_result({:error, message}), do: raise(VerifyError, message: message)
   defp check_welcome_message_config!(:ok) do
     if Pleroma.Config.get([:welcome, :email, :enabled], false) and
          not Pleroma.Emails.Mailer.enabled?() do
-      Logger.error("""
+      Logger.warn("""
-      To send welcome email do you need to enable mail.
+      To send welcome emails, you need to enable the mailer.
-      \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true
+      Welcome emails will NOT be sent with the current config.
-      """)
 
-      {:error, "The mail disabled."}
+      Enable the mailer:
-    else
+        config :pleroma, Pleroma.Emails.Mailer, enabled: true
-      :ok
+      """)
     end
+
+    :ok
   end
 
   defp check_welcome_message_config!(result), do: result
@@ -51,18 +52,21 @@ defp check_welcome_message_config!(result), do: result
   #
   def check_confirmation_accounts!(:ok) do
     if Pleroma.Config.get([:instance, :account_activation_required]) &&
-         not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+         not Pleroma.Emails.Mailer.enabled?() do
-      Logger.error(
+      Logger.warn("""
-        "Account activation enabled, but no Mailer settings enabled.\n" <>
+      Account activation is required, but the mailer is disabled.
-          "Please set config :pleroma, :instance, account_activation_required: false\n" <>
+      Users will NOT be able to confirm their accounts with this config.
-          "Otherwise setup and enable Mailer."
+      Either disable account activation or enable the mailer.
-      )
 
-      {:error,
+      Disable account activation:
-       "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."}
+        config :pleroma, :instance, account_activation_required: false
-    else
+
-      :ok
+      Enable the mailer:
+        config :pleroma, Pleroma.Emails.Mailer, enabled: true
+      """)
     end
+
+    :ok
   end
 
   def check_confirmation_accounts!(result), do: result

lib/pleroma/config/release_runtime_provider.ex

@@ -1,6 +1,6 @@
 defmodule Pleroma.Config.ReleaseRuntimeProvider do
   @moduledoc """
-  Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases.
+  Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.
   """
   @behaviour Config.Provider
 
@@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
   def init(opts), do: opts
 
   @impl true
-  def load(config, _opts) do
+  def load(config, opts) do
     with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
 
-    config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
+    config_path =
+      opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 
     with_runtime_config =
       if File.exists?(config_path) do
@@ -24,7 +25,7 @@ def load(config, _opts) do
         warning = [
           IO.ANSI.red(),
           IO.ANSI.bright(),
-          "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
+          "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
           IO.ANSI.reset()
         ]
 
@@ -33,13 +34,14 @@ def load(config, _opts) do
       end
 
     exported_config_path =
-      config_path
+      opts[:exported_config_path] ||
-      |> Path.dirname()
+        config_path
-      |> Path.join("prod.exported_from_db.secret.exs")
+        |> Path.dirname()
+        |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")
 
     with_exported =
       if File.exists?(exported_config_path) do
-        exported_config = Config.Reader.read!(with_runtime_config)
+        exported_config = Config.Reader.read!(exported_config_path)
         Config.Reader.merge(with_runtime_config, exported_config)
       else
         with_runtime_config

lib/pleroma/config_db.ex

@@ -387,6 +387,6 @@ defp find_valid_delimiter([delimiter | others], pattern, regex_delimiter) do
   @spec module_name?(String.t()) :: boolean()
   def module_name?(string) do
     Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
-      string in ["Oban", "Ueberauth", "ExSyslogger"]
+      string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
   end
 end

lib/pleroma/earmark_renderer.ex

@@ -1,256 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-# This file is derived from Earmark, under the following copyright:
-# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
-# SPDX-License-Identifier: Apache-2.0
-# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
-defmodule Pleroma.EarmarkRenderer do
-  @moduledoc false
-
-  alias Earmark.Block
-  alias Earmark.Context
-  alias Earmark.HtmlRenderer
-  alias Earmark.Options
-
-  import Earmark.Inline, only: [convert: 3]
-  import Earmark.Helpers.HtmlHelpers
-  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
-  import Earmark.Context, only: [append: 2, set_value: 2]
-  import Earmark.Options, only: [get_mapper: 1]
-
-  @doc false
-  def render(blocks, %Context{options: %Options{}} = context) do
-    messages = get_messages(context)
-
-    {contexts, html} =
-      get_mapper(context.options).(
-        blocks,
-        &render_block(&1, put_in(context.options.messages, []))
-      )
-      |> Enum.unzip()
-
-    all_messages =
-      contexts
-      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
-
-    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
-  end
-
-  #############
-  # Paragraph #
-  #############
-  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
-    lines = convert(lines, lnb, context)
-    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
-  end
-
-  ########
-  # Html #
-  ########
-  defp render_block(%Block.Html{html: html}, context) do
-    {context, html}
-  end
-
-  defp render_block(%Block.HtmlComment{lines: lines}, context) do
-    {context, lines}
-  end
-
-  defp render_block(%Block.HtmlOneline{html: html}, context) do
-    {context, html}
-  end
-
-  #########
-  # Ruler #
-  #########
-  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
-    add_attrs(context, "<hr />", attrs, [], lnb)
-  end
-
-  ###########
-  # Heading #
-  ###########
-  defp render_block(
-         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
-         context
-       ) do
-    converted = convert(content, lnb, context)
-    html = "<h#{level}>#{converted.value}</h#{level}>"
-    add_attrs(converted, html, attrs, [], lnb)
-  end
-
-  ##############
-  # Blockquote #
-  ##############
-
-  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
-    {context1, body} = render(blocks, context)
-    html = "<blockquote>#{body}</blockquote>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  #########
-  # Table #
-  #########
-
-  defp render_block(
-         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
-         context
-       ) do
-    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
-    context2 = set_value(context1, html)
-
-    context3 =
-      if header do
-        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
-      else
-        # Maybe an error, needed append(context, html)
-        context2
-      end
-
-    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
-
-    {context4, [context4.value, "</table>"]}
-  end
-
-  ########
-  # Code #
-  ########
-
-  defp render_block(
-         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
-         %Context{options: options} = context
-       ) do
-    class =
-      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
-
-    tag = ~s[<pre><code#{class}>]
-    lines = options.render_code.(block)
-    html = ~s[#{tag}#{lines}</code></pre>]
-    add_attrs(context, html, attrs, [], lnb)
-  end
-
-  #########
-  # Lists #
-  #########
-
-  defp render_block(
-         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
-         context
-       ) do
-    {context1, content} = render(items, context)
-    html = "<#{type}#{start}>#{content}</#{type}>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  # format a single paragraph list item, and remove the para tags
-  defp render_block(
-         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
-         context
-       )
-       when length(blocks) == 1 do
-    {context1, content} = render(blocks, context)
-    content = Regex.replace(~r{</?p>}, content, "")
-    html = "<li>#{content}</li>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  # format a spaced list item
-  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
-    {context1, content} = render(blocks, context)
-    html = "<li>#{content}</li>"
-    add_attrs(context1, html, attrs, [], lnb)
-  end
-
-  ##################
-  # Footnote Block #
-  ##################
-
-  defp render_block(%Block.FnList{blocks: footnotes}, context) do
-    items =
-      Enum.map(footnotes, fn note ->
-        blocks = append_footnote_link(note)
-        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
-      end)
-
-    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
-    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
-  end
-
-  #######################################
-  # Isolated IALs are rendered as paras #
-  #######################################
-
-  defp render_block(%Block.Ial{verbatim: verbatim}, context) do
-    {context, "<p>{:#{verbatim}}</p>"}
-  end
-
-  ####################
-  # IDDef is ignored #
-  ####################
-
-  defp render_block(%Block.IdDef{}, context), do: {context, ""}
-
-  #####################################
-  # And here are the inline renderers #
-  #####################################
-
-  defdelegate br, to: HtmlRenderer
-  defdelegate codespan(text), to: HtmlRenderer
-  defdelegate em(text), to: HtmlRenderer
-  defdelegate strong(text), to: HtmlRenderer
-  defdelegate strikethrough(text), to: HtmlRenderer
-
-  defdelegate link(url, text), to: HtmlRenderer
-  defdelegate link(url, text, title), to: HtmlRenderer
-
-  defdelegate image(path, alt, title), to: HtmlRenderer
-
-  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
-
-  # Table rows
-  defp add_trs(context, rows, tag, aligns, lnb) do
-    numbered_rows =
-      rows
-      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
-
-    numbered_rows
-    |> Enum.reduce(context, fn {row, lnb}, ctx ->
-      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
-    end)
-  end
-
-  defp add_tds(context, row, tag, aligns, lnb) do
-    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
-  end
-
-  defp add_td_fn(row, tag, aligns, lnb) do
-    fn n, ctx ->
-      style =
-        case Enum.at(aligns, n - 1, :default) do
-          :default -> ""
-          align -> " style=\"text-align: #{align}\""
-        end
-
-      col = Enum.at(row, n - 1)
-      converted = convert(col, lnb, set_messages(ctx, []))
-      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
-    end
-  end
-
-  ###############################
-  # Append Footnote Return Link #
-  ###############################
-
-  defdelegate append_footnote_link(note), to: HtmlRenderer
-  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
-
-  defdelegate render_code(lines), to: HtmlRenderer
-
-  defp code_classes(language, prefix) do
-    ["" | String.split(prefix || "")]
-    |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
-    |> Enum.join(" ")
-  end
-end

lib/pleroma/formatter.ex

@@ -121,6 +121,10 @@ def mentions_escape(text, options \\ []) do
     end
   end
 
+  def markdown_to_html(text) do
+    Earmark.as_html!(text, %Earmark.Options{compact_output: true})
+  end
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end

lib/pleroma/object/containment.ex

@@ -71,6 +71,14 @@ def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(oth
     compare_uris(id_uri, other_uri)
   end
 
+  # Mastodon pin activities don't have an id, so we check the object field, which will be pinned.
+  def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do
+    id_uri = URI.parse(id)
+    object_uri = URI.parse(object)
+
+    compare_uris(id_uri, object_uri)
+  end
+
   def contain_origin_from_id(_id, _data), do: :error
 
   def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),

lib/pleroma/user.ex

@@ -99,6 +99,7 @@ defmodule Pleroma.User do
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
     field(:following_address, :string)
+    field(:featured_address, :string)
     field(:search_rank, :float, virtual: true)
     field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
@@ -130,7 +131,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
     field(:emoji, :map, default: %{})
@@ -148,6 +148,7 @@ defmodule Pleroma.User do
     field(:accepts_chat_messages, :boolean, default: nil)
     field(:last_active_at, :naive_datetime)
     field(:disclose_client, :boolean, default: true)
+    field(:pinned_objects, :map, default: %{})
 
     embeds_one(
       :notification_settings,
@@ -372,8 +373,10 @@ def banner_url(user, options \\ []) do
   end
 
   # Should probably be renamed or removed
+  @spec ap_id(User.t()) :: String.t()
   def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
 
+  @spec ap_followers(User.t()) :: String.t()
   def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
   def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
@@ -381,6 +384,11 @@ def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
   def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
   def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
 
+  @spec ap_featured_collection(User.t()) :: String.t()
+  def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa
+
+  def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured"
+
   defp truncate_fields_param(params) do
     if Map.has_key?(params, :fields) do
       Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
@@ -443,6 +451,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :uri,
         :follower_address,
         :following_address,
+        :featured_address,
         :hide_followers,
         :hide_follows,
         :hide_followers_count,
@@ -454,7 +463,8 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :invisible,
         :actor_type,
         :also_known_as,
-        :accepts_chat_messages
+        :accepts_chat_messages,
+        :pinned_objects
       ]
     )
     |> cast(params, [:name], empty_values: [])
@@ -686,7 +696,7 @@ def register_changeset_ldap(struct, params = %{password: password})
     |> validate_format(:nickname, local_nickname_regex())
     |> put_ap_id()
     |> unique_constraint(:ap_id)
-    |> put_following_and_follower_address()
+    |> put_following_and_follower_and_featured_address()
   end
 
   def register_changeset(struct, params \\ %{}, opts \\ []) do
@@ -747,7 +757,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
     |> put_password_hash
     |> put_ap_id()
     |> unique_constraint(:ap_id)
-    |> put_following_and_follower_address()
+    |> put_following_and_follower_and_featured_address()
   end
 
   def maybe_validate_required_email(changeset, true), do: changeset
@@ -765,11 +775,16 @@ defp put_ap_id(changeset) do
     put_change(changeset, :ap_id, ap_id)
   end
 
-  defp put_following_and_follower_address(changeset) do
+  defp put_following_and_follower_and_featured_address(changeset) do
-    followers = ap_followers(%User{nickname: get_field(changeset, :nickname)})
+    user = %User{nickname: get_field(changeset, :nickname)}
+    followers = ap_followers(user)
+    following = ap_following(user)
+    featured = ap_featured_collection(user)
 
     changeset
     |> put_change(:follower_address, followers)
+    |> put_change(:following_address, following)
+    |> put_change(:featured_address, featured)
   end
 
   defp autofollow_users(user) do
@@ -2343,45 +2358,35 @@ def approval_changeset(user, set_approval: approved?) do
     cast(user, %{is_approved: approved?}, [:is_approved])
   end
 
-  def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do
+  @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()}
-    if id not in user.pinned_activities do
+  def add_pinned_object_id(%User{} = user, object_id) do
-      max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
+    if !user.pinned_objects[object_id] do
-      params = %{pinned_activities: user.pinned_activities ++ [id]}
+      params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())}
-
-      # if pinned activity was scheduled for deletion, we remove job
-      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do
-        Oban.cancel_job(expiration.id)
-      end
 
       user
-      |> cast(params, [:pinned_activities])
+      |> cast(params, [:pinned_objects])
-      |> validate_length(:pinned_activities,
+      |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects ->
-        max: max_pinned_statuses,
+        max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
-        message: "You have already pinned the maximum number of statuses"
+
-      )
+        if Enum.count(pinned_objects) <= max_pinned_statuses do
+          []
+        else
+          [pinned_objects: "You have already pinned the maximum number of statuses"]
+        end
+      end)
     else
       change(user)
     end
     |> update_and_set_cache()
   end
 
-  def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do
+  @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()}
-    params = %{pinned_activities: List.delete(user.pinned_activities, id)}
+  def remove_pinned_object_id(%User{} = user, object_id) do
-
-    # if pinned activity was scheduled for deletion, we reschedule it for deletion
-    if data["expires_at"] do
-      # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
-      {:ok, expires_at} =
-        data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
-
-      Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
-        activity_id: id,
-        expires_at: expires_at
-      })
-    end
-
     user
-    |> cast(params, [:pinned_activities])
+    |> cast(
+      %{pinned_objects: Map.delete(user.pinned_objects, object_id)},
+      [:pinned_objects]
+    )
     |> update_and_set_cache()
   end
 

lib/pleroma/utils.ex

@@ -11,6 +11,8 @@ defmodule Pleroma.Utils do
     eperm epipe erange erofs espipe esrch estale etxtbsy exdev
   )a
 
+  @repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000)
+
   def compile_dir(dir) when is_binary(dir) do
     dir
     |> File.ls!()
@@ -63,4 +65,21 @@ def posix_error_message(code) when code in @posix_error_codes do
   end
 
   def posix_error_message(_), do: ""
+
+  @doc """
+  Returns [timeout: integer] suitable for passing as an option to Repo functions.
+
+  This function detects if the execution was triggered from IEx shell, Mix task, or
+  ./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value.
+  """
+  @spec query_timeout() :: [timeout: integer]
+  def query_timeout do
+    {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2)
+
+    cond do
+      parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity]
+      parent == :erl_eval -> [timeout: :infinity]
+      true -> [timeout: @repo_timeout]
+    end
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -630,7 +630,7 @@ defp fetch_activities_for_user(user, reading_user, params) do
       |> Map.put(:type, ["Create", "Announce"])
       |> Map.put(:user, reading_user)
       |> Map.put(:actor_id, user.ap_id)
-      |> Map.put(:pinned_activity_ids, user.pinned_activities)
+      |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))
 
     params =
       if User.blocks?(reading_user, user) do
@@ -1075,8 +1075,18 @@ defp restrict_unlisted(query, %{restrict_unlisted: true}) do
 
   defp restrict_unlisted(query, _), do: query
 
-  defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do
+  defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do
-    from(activity in query, where: activity.id in ^ids)
+    from(
+      [activity, object: o] in query,
+      where:
+        fragment(
+          "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
+          activity.data,
+          activity.data,
+          activity.data,
+          ^ids
+        )
+    )
   end
 
   defp restrict_pinned(query, _), do: query
@@ -1419,6 +1429,9 @@ defp object_to_user_data(data) do
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
+    featured_address = data["featured"]
+    {:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address)
+
     public_key =
       if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
         data["publicKey"]["publicKeyPem"]
@@ -1447,13 +1460,15 @@ defp object_to_user_data(data) do
       name: data["name"],
       follower_address: data["followers"],
       following_address: data["following"],
+      featured_address: featured_address,
       bio: data["summary"] || "",
       actor_type: actor_type,
       also_known_as: Map.get(data, "alsoKnownAs", []),
       public_key: public_key,
       inbox: data["inbox"],
       shared_inbox: shared_inbox,
-      accepts_chat_messages: accepts_chat_messages
+      accepts_chat_messages: accepts_chat_messages,
+      pinned_objects: pinned_objects
     }
 
     # nickname can be nil because of virtual actors
@@ -1591,6 +1606,41 @@ def maybe_handle_clashing_nickname(data) do
     end
   end
 
+  def pin_data_from_featured_collection(%{
+        "type" => type,
+        "orderedItems" => objects
+      })
+      when type in ["OrderedCollection", "Collection"] do
+    Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
+  end
+
+  def fetch_and_prepare_featured_from_ap_id(nil) do
+    {:ok, %{}}
+  end
+
+  def fetch_and_prepare_featured_from_ap_id(ap_id) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
+      {:ok, pin_data_from_featured_collection(data)}
+    else
+      e ->
+        Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}")
+        {:ok, %{}}
+    end
+  end
+
+  def pinned_fetch_task(nil), do: nil
+
+  def pinned_fetch_task(%{pinned_objects: pins}) do
+    if Enum.all?(pins, fn {ap_id, _} ->
+         Object.get_cached_by_ap_id(ap_id) ||
+           match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
+       end) do
+      :ok
+    else
+      :error
+    end
+  end
+
   def make_user_from_ap_id(ap_id) do
     user = User.get_cached_by_ap_id(ap_id)
 
@@ -1598,6 +1648,8 @@ def make_user_from_ap_id(ap_id) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
       with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
+        {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
+
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -543,4 +543,12 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} =
       |> json(object.data)
     end
   end
+
+  def pinned(conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(UserView.render("featured.json", %{user: user}))
+    end
+  end
 end

lib/pleroma/web/activity_pub/builder.ex

@@ -273,4 +273,36 @@ defp object_action(actor, object) do
        "context" => object.data["context"]
      }, []}
   end
+
+  @spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()}
+  def pin(%User{} = user, object) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "target" => pinned_url(user.nickname),
+       "object" => object.data["id"],
+       "actor" => user.ap_id,
+       "type" => "Add",
+       "to" => [Pleroma.Constants.as_public()],
+       "cc" => [user.follower_address]
+     }, []}
+  end
+
+  @spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()}
+  def unpin(%User{} = user, object) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "target" => pinned_url(user.nickname),
+       "object" => object.data["id"],
+       "actor" => user.ap_id,
+       "type" => "Remove",
+       "to" => [Pleroma.Constants.as_public()],
+       "cc" => [user.follower_address]
+     }, []}
+  end
+
+  defp pinned_url(nickname) when is_binary(nickname) do
+    Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname)
+  end
 end

lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex

@@ -0,0 +1,59 @@
+defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+  alias Pleroma.Config
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  require Logger
+
+  @impl true
+  def filter(message) do
+    with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
+         %User{actor_type: "Service"} = follower <-
+           User.get_cached_by_nickname(follower_nickname),
+         %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
+      try_follow(follower, message)
+    else
+      nil ->
+        Logger.warn(
+          "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname
+            account does not exist, or the account is not correctly configured as a bot."
+        )
+
+        {:ok, message}
+
+      _ ->
+        {:ok, message}
+    end
+  end
+
+  defp try_follow(follower, message) do
+    to = Map.get(message, "to", [])
+    cc = Map.get(message, "cc", [])
+    actor = [message["actor"]]
+
+    Enum.concat([to, cc, actor])
+    |> List.flatten()
+    |> Enum.uniq()
+    |> User.get_all_by_ap_id()
+    |> Enum.each(fn user ->
+      with false <- user.local,
+           false <- User.following?(follower, user),
+           false <- User.locked?(user),
+           false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do
+        Logger.debug(
+          "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
+        )
+
+        CommonAPI.follow(follower, user)
+      end
+    end)
+
+    {:ok, message}
+  end
+
+  @impl true
+  def describe do
+    {:ok, %{}}
+  end
+end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
@@ -37,37 +38,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   @impl true
   def validate(object, meta)
 
-  def validate(%{"type" => type} = object, meta)
-      when type in ~w[Accept Reject] do
-    with {:ok, object} <-
-           object
-           |> AcceptRejectValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "Event"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> EventValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "Follow"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> FollowValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
   def validate(%{"type" => "Block"} = block_activity, meta) do
     with {:ok, block_activity} <-
            block_activity
@@ -87,16 +57,6 @@ def validate(%{"type" => "Block"} = block_activity, meta) do
     end
   end
 
-  def validate(%{"type" => "Update"} = update_activity, meta) do
-    with {:ok, update_activity} <-
-           update_activity
-           |> UpdateValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      update_activity = stringify_keys(update_activity)
-      {:ok, update_activity, meta}
-    end
-  end
-
   def validate(%{"type" => "Undo"} = object, meta) do
     with {:ok, object} <-
            object
@@ -123,76 +83,6 @@ def validate(%{"type" => "Delete"} = object, meta) do
     end
   end
 
-  def validate(%{"type" => "Like"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> LikeValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "ChatMessage"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> ChatMessageValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "Question"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> QuestionValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
-    with {:ok, object} <-
-           object
-           |> AudioVideoValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "Article"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> ArticleNoteValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "Answer"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> AnswerValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
-  def validate(%{"type" => "EmojiReact"} = object, meta) do
-    with {:ok, object} <-
-           object
-           |> EmojiReactValidator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
-      {:ok, object, meta}
-    end
-  end
-
   def validate(
         %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,
         meta
@@ -224,10 +114,60 @@ def validate(
     end
   end
 
-  def validate(%{"type" => "Announce"} = object, meta) do
+  def validate(%{"type" => type} = object, meta)
+      when type in ~w[Event Question Audio Video Article] do
+    validator =
+      case type do
+        "Event" -> EventValidator
+        "Question" -> QuestionValidator
+        "Audio" -> AudioVideoValidator
+        "Video" -> AudioVideoValidator
+        "Article" -> ArticleNoteValidator
+      end
+
     with {:ok, object} <-
            object
-           |> AnnounceValidator.cast_and_validate()
+           |> validator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+
+      # Insert copy of hashtags as strings for the non-hashtag table indexing
+      tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
+      object = Map.put(object, "tag", tag)
+
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => type} = object, meta)
+      when type in ~w[Accept Reject Follow Update Like EmojiReact Announce
+      ChatMessage Answer] do
+    validator =
+      case type do
+        "Accept" -> AcceptRejectValidator
+        "Reject" -> AcceptRejectValidator
+        "Follow" -> FollowValidator
+        "Update" -> UpdateValidator
+        "Like" -> LikeValidator
+        "EmojiReact" -> EmojiReactValidator
+        "Announce" -> AnnounceValidator
+        "ChatMessage" -> ChatMessageValidator
+        "Answer" -> AnswerValidator
+      end
+
+    with {:ok, object} <-
+           object
+           |> validator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do
+    with {:ok, object} <-
+           object
+           |> AddRemoveValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
@@ -260,7 +200,7 @@ def cast_and_apply(%{"type" => "Article"} = object) do
 
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
-  # is_struct/1 isn't present in Elixir 1.8.x
+  # is_struct/1 appears in Elixir 1.11
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()

lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex

@@ -27,7 +27,7 @@ def cast_data(data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Accept", "Reject"])

lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  require Pleroma.Constants
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.User
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:target)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:type)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+  end
+
+  def cast_and_validate(data) do
+    {:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"])
+
+    {:ok, actor} = maybe_refetch_user(actor)
+
+    data
+    |> maybe_fix_data_for_mastodon(actor)
+    |> cast_data()
+    |> validate_data(actor)
+  end
+
+  defp maybe_fix_data_for_mastodon(data, actor) do
+    # Mastodon sends pin/unpin objects without id, to, cc fields
+    data
+    |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id())
+    |> Map.put_new("to", [Pleroma.Constants.as_public()])
+    |> Map.put_new("cc", [actor.follower_address])
+  end
+
+  defp cast_data(data) do
+    cast(%__MODULE__{}, data, __schema__(:fields))
+  end
+
+  defp validate_data(changeset, actor) do
+    changeset
+    |> validate_required([:id, :target, :object, :actor, :type, :to, :cc])
+    |> validate_inclusion(:type, ~w(Add Remove))
+    |> validate_actor_presence()
+    |> validate_collection_belongs_to_actor(actor)
+    |> validate_object_presence()
+  end
+
+  defp validate_collection_belongs_to_actor(changeset, actor) do
+    validate_change(changeset, :target, fn :target, target ->
+      if target == actor.featured_address do
+        []
+      else
+        [target: "collection doesn't belong to actor"]
+      end
+    end)
+  end
+
+  defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do
+    {:ok, user}
+  end
+
+  defp maybe_refetch_user(%User{ap_id: ap_id}) do
+    Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id)
+  end
+end

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -50,7 +50,7 @@ def fix_after_cast(cng) do
     cng
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Announce"])
     |> validate_required([:id, :type, :object, :actor, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/answer_validator.ex

@@ -50,7 +50,7 @@ def changeset(struct, data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Answer"])
     |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])

lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -90,11 +90,12 @@ def changeset(struct, data) do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Article", "Note"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
 
@@ -90,7 +89,7 @@ defp fix_url(data) do
     end
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -5,11 +5,11 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
   use Ecto.Schema
 
-  alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -23,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -110,7 +109,7 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
        when is_binary(content) do
     content =
       content
-      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.Formatter.markdown_to_html()
       |> Pleroma.HTML.filter_tags()
 
     Map.put(data, "content", content)
@@ -132,11 +131,12 @@ def changeset(struct, data) do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Audio", "Video"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])

lib/pleroma/web/activity_pub/object_validators/block_validator.ex

@@ -26,7 +26,7 @@ def cast_data(data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Block"])

lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex

@@ -67,7 +67,7 @@ def changeset(struct, data) do
     |> cast_embed(:attachment)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["ChatMessage"])
     |> validate_required([:id, :actor, :to, :type, :published])

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   alias Pleroma.Object
   alias Pleroma.User
 
+  @spec validate_any_presence(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()
   def validate_any_presence(cng, fields) do
     non_empty =
       fields
@@ -29,6 +30,7 @@ def validate_any_presence(cng, fields) do
     end
   end
 
+  @spec validate_actor_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()
   def validate_actor_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :actor)
 
@@ -47,6 +49,7 @@ def validate_actor_presence(cng, options \\ []) do
     end)
   end
 
+  @spec validate_object_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()
   def validate_object_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :object)
     allowed_types = Keyword.get(options, :allowed_types, false)
@@ -68,6 +71,7 @@ def validate_object_presence(cng, options \\ []) do
     end)
   end
 
+  @spec validate_object_or_user_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t()
   def validate_object_or_user_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :object)
     options = Keyword.put(options, :field_name, field_name)
@@ -83,6 +87,7 @@ def validate_object_or_user_presence(cng, options \\ []) do
     if actor_cng.valid?, do: actor_cng, else: object_cng
   end
 
+  @spec validate_host_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()
   def validate_host_match(cng, fields \\ [:id, :actor]) do
     if same_domain?(cng, fields) do
       cng
@@ -95,6 +100,7 @@ def validate_host_match(cng, fields \\ [:id, :actor]) do
     end
   end
 
+  @spec validate_fields_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t()
   def validate_fields_match(cng, fields) do
     if map_unique?(cng, fields) do
       cng
@@ -122,12 +128,14 @@ defp map_unique?(cng, fields, func \\ & &1) do
     end)
   end
 
+  @spec same_domain?(Ecto.Changeset.t(), [atom()]) :: boolean()
   def same_domain?(cng, fields \\ [:actor, :object]) do
     map_unique?(cng, fields, fn value -> URI.parse(value).host end)
   end
 
   # This figures out if a user is able to create, delete or modify something
   # based on the domain and superuser status
+  @spec validate_modification_rights(Ecto.Changeset.t()) :: Ecto.Changeset.t()
   def validate_modification_rights(cng) do
     actor = User.get_cached_by_ap_id(get_field(cng, :actor))
 

lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex

@@ -39,7 +39,7 @@ def cast_and_validate(data, meta \\ []) do
     |> validate_data(meta)
   end
 
-  def validate_data(cng, meta \\ []) do
+  defp validate_data(cng, meta) do
     cng
     |> validate_required([:id, :actor, :to, :type, :object])
     |> validate_inclusion(:type, ["Create"])

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -79,7 +79,7 @@ defp fix(data, meta) do
     |> CommonFixes.fix_actor()
   end
 
-  def validate_data(cng, meta \\ []) do
+  defp validate_data(cng, meta) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -53,7 +53,7 @@ def add_deleted_activity_id(cng) do
     Tombstone
     Video
   }
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -70,7 +70,7 @@ def validate_emoji(cng) do
     end
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["EmojiReact"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content])

lib/pleroma/web/activity_pub/object_validators/event_validator.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
 
     field(:name, :string)
@@ -81,11 +81,12 @@ def changeset(struct, data) do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast(data, __schema__(:fields) -- [:attachment, :tag])
     |> cast_embed(:attachment)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Event"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/follow_validator.ex

@@ -27,7 +27,7 @@ def cast_data(data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Follow"])

lib/pleroma/web/activity_pub/object_validators/like_validator.ex

@@ -76,7 +76,7 @@ def fix_recipients(cng) do
     end
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Like"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
@@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:cc, ObjectValidators.Recipients, default: [])
     field(:bto, ObjectValidators.Recipients, default: [])
     field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
+    embeds_many(:tag, TagValidator)
-    field(:tag, {:array, :map}, default: [])
     field(:type, :string)
     field(:content, :string)
     field(:context, :string)
@@ -93,13 +93,14 @@ def changeset(struct, data) do
     data = fix(data)
 
     struct
-    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag])
     |> cast_embed(:attachment)
     |> cast_embed(:anyOf)
     |> cast_embed(:oneOf)
+    |> cast_embed(:tag)
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
     |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])

lib/pleroma/web/activity_pub/object_validators/tag_validator.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+
+  import Ecto.Changeset
+
+  @primary_key false
+  embedded_schema do
+    # Common
+    field(:type, :string)
+    field(:name, :string)
+
+    # Mention, Hashtag
+    field(:href, ObjectValidators.Uri)
+
+    # Emoji
+    embeds_one :icon, IconObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:url, ObjectValidators.Uri)
+    end
+
+    field(:updated, ObjectValidators.DateTime)
+    field(:id, ObjectValidators.Uri)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, %{"type" => "Mention"} = data) do
+    struct
+    |> cast(data, [:type, :name, :href])
+    |> validate_required([:type, :href])
+  end
+
+  def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do
+    name =
+      cond do
+        "#" <> name -> name
+        name -> name
+      end
+      |> String.downcase()
+
+    data = Map.put(data, "name", name)
+
+    struct
+    |> cast(data, [:type, :name, :href])
+    |> validate_required([:type, :name])
+  end
+
+  def changeset(struct, %{"type" => "Emoji"} = data) do
+    data = Map.put(data, "name", String.trim(data["name"], ":"))
+
+    struct
+    |> cast(data, [:type, :name, :updated, :id])
+    |> cast_embed(:icon, with: &icon_changeset/2)
+    |> validate_required([:type, :name, :icon])
+  end
+
+  def icon_changeset(struct, data) do
+    struct
+    |> cast(data, [:type, :url])
+    |> validate_inclusion(:type, ~w[Image])
+    |> validate_required([:type, :url])
+  end
+end

lib/pleroma/web/activity_pub/object_validators/undo_validator.ex

@@ -38,7 +38,7 @@ def changeset(struct, data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(data_cng) do
+  defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Undo"])
     |> validate_required([:id, :type, :object, :actor, :to, :cc])

lib/pleroma/web/activity_pub/object_validators/update_validator.ex

@@ -28,7 +28,7 @@ def cast_data(data) do
     |> cast(data, __schema__(:fields))
   end
 
-  def validate_data(cng) do
+  defp validate_data(cng) do
     cng
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Update"])

lib/pleroma/web/activity_pub/pipeline.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.Utils
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
@@ -24,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
   @spec common_pipeline(map(), keyword()) ::
           {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()}
   def common_pipeline(object, meta) do
-    case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do
+    case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do
       {:ok, {:ok, activity, meta}} ->
         @side_effects.handle_after_transaction(meta)
         {:ok, activity, meta}
@@ -40,19 +41,17 @@ def common_pipeline(object, meta) do
     end
   end
 
-  def do_common_pipeline(object, meta) do
+  def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct}
-    with {_, {:ok, validated_object, meta}} <-
+
-           {:validate_object, @object_validator.validate(object, meta)},
+  def do_common_pipeline(message, meta) do
-         {_, {:ok, mrfd_object, meta}} <-
+    with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)},
-           {:mrf_object, @mrf.pipeline_filter(validated_object, meta)},
+         {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)},
-         {_, {:ok, activity, meta}} <-
+         {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)},
-           {:persist_object, @activity_pub.persist(mrfd_object, meta)},
+         {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)},
-         {_, {:ok, activity, meta}} <-
+         {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do
-           {:execute_side_effects, @side_effects.handle(activity, meta)},
+      {:ok, message, meta}
-         {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
-      {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, message, _}} -> {:reject, message}
+      {:mrf, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -276,10 +276,10 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
     result =
       case deleted_object do
         %Object{} ->
-          with {:ok, deleted_object, activity} <- Object.delete(deleted_object),
+          with {:ok, deleted_object, _activity} <- Object.delete(deleted_object),
                {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]},
                %User{} = user <- User.get_cached_by_ap_id(actor) do
-            User.remove_pinnned_activity(user, activity)
+            User.remove_pinned_object_id(user, deleted_object.data["id"])
 
             {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object)
 
@@ -312,6 +312,63 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
     end
   end
 
+  # Tasks this handles:
+  # - adds pin to user
+  # - removes expiration job for pinned activity, if was set for expiration
+  @impl true
+  def handle(%{data: %{"type" => "Add"} = data} = object, meta) do
+    with %User{} = user <- User.get_cached_by_ap_id(data["actor"]),
+         {:ok, _user} <- User.add_pinned_object_id(user, data["object"]) do
+      # if pinned activity was scheduled for deletion, we remove job
+      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(meta[:activity_id]) do
+        Oban.cancel_job(expiration.id)
+      end
+
+      {:ok, object, meta}
+    else
+      nil ->
+        {:error, :user_not_found}
+
+      {:error, changeset} ->
+        if changeset.errors[:pinned_objects] do
+          {:error, :pinned_statuses_limit_reached}
+        else
+          changeset.errors
+        end
+    end
+  end
+
+  # Tasks this handles:
+  # - removes pin from user
+  # - removes corresponding Add activity
+  # - if activity had expiration, recreates activity expiration job
+  @impl true
+  def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do
+    with %User{} = user <- User.get_cached_by_ap_id(data["actor"]),
+         {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do
+      data["object"]
+      |> Activity.add_by_params_query(user.ap_id, user.featured_address)
+      |> Repo.delete_all()
+
+      # if pinned activity was scheduled for deletion, we reschedule it for deletion
+      if meta[:expires_at] do
+        # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
+        {:ok, expires_at} =
+          Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at])
+
+        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+          activity_id: meta[:activity_id],
+          expires_at: expires_at
+        })
+      end
+
+      {:ok, object, meta}
+    else
+      nil -> {:error, :user_not_found}
+      error -> error
+    end
+  end
+
   # Nothing to do
   @impl true
   def handle(object, meta) do

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -534,7 +534,7 @@ def handle_incoming(
   end
 
   def handle_incoming(%{"type" => type} = data, _options)
-      when type in ~w{Like EmojiReact Announce} do
+      when type in ~w{Like EmojiReact Announce Add Remove} do
     with :ok <- ObjectValidator.fetch_actor_and_object(data),
          {:ok, activity, _meta} <-
            Pipeline.common_pipeline(data, local: false) do
@@ -564,7 +564,7 @@ def handle_incoming(
            Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     else
-      {:error, {:validate_object, _}} = e ->
+      {:error, {:validate, _}} = e ->
         # Check if we have a create activity for this
         with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
              %Activity{data: %{"actor" => actor}} <-
@@ -1000,6 +1000,7 @@ def upgrade_user_from_ap_id(ap_id) do
     with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
          {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
          {:ok, user} <- update_user(user, data) do
+      {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end)
       TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
       {:ok, user}
     else

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
 
   alias Pleroma.Keys
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Endpoint
@@ -97,6 +99,7 @@ def render("user.json", %{user: user}) do
       "followers" => "#{user.ap_id}/followers",
       "inbox" => "#{user.ap_id}/inbox",
       "outbox" => "#{user.ap_id}/outbox",
+      "featured" => "#{user.ap_id}/collections/featured",
       "preferredUsername" => user.nickname,
       "name" => user.name,
       "summary" => user.bio,
@@ -245,6 +248,24 @@ def render("activity_collection_page.json", %{
     |> Map.merge(pagination)
   end
 
+  def render("featured.json", %{
+        user: %{featured_address: featured_address, pinned_objects: pinned_objects}
+      }) do
+    objects =
+      pinned_objects
+      |> Enum.sort_by(fn {_, pinned_at} -> pinned_at end, &>=/2)
+      |> Enum.map(fn {id, _} ->
+        ObjectView.render("object.json", %{object: Object.get_cached_by_ap_id(id)})
+      end)
+
+    %{
+      "id" => featured_address,
+      "type" => "OrderedCollection",
+      "orderedItems" => objects
+    }
+    |> Map.merge(Utils.make_json_ld_header())
+  end
+
   defp maybe_put_total_items(map, false, _total), do: map
 
   defp maybe_put_total_items(map, true, total) do

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -182,7 +182,34 @@ def pin_operation do
       parameters: [id_param()],
       responses: %{
         200 => status_response(),
-        400 => Operation.response("Error", "application/json", ApiError)
+        400 =>
+          Operation.response("Bad Request", "application/json", %Schema{
+            allOf: [ApiError],
+            title: "Unprocessable Entity",
+            example: %{
+              "error" => "You have already pinned the maximum number of statuses"
+            }
+          }),
+        404 =>
+          Operation.response("Not found", "application/json", %Schema{
+            allOf: [ApiError],
+            title: "Unprocessable Entity",
+            example: %{
+              "error" => "Record not found"
+            }
+          }),
+        422 =>
+          Operation.response(
+            "Unprocessable Entity",
+            "application/json",
+            %Schema{
+              allOf: [ApiError],
+              title: "Unprocessable Entity",
+              example: %{
+                "error" => "Someone else's status cannot be pinned"
+              }
+            }
+          )
       }
     }
   end
@@ -197,7 +224,22 @@ def unpin_operation do
       parameters: [id_param()],
       responses: %{
         200 => status_response(),
-        400 => Operation.response("Error", "application/json", ApiError)
+        400 =>
+          Operation.response("Bad Request", "application/json", %Schema{
+            allOf: [ApiError],
+            title: "Unprocessable Entity",
+            example: %{
+              "error" => "You have already pinned the maximum number of statuses"
+            }
+          }),
+        404 =>
+          Operation.response("Not found", "application/json", %Schema{
+            allOf: [ApiError],
+            title: "Unprocessable Entity",
+            example: %{
+              "error" => "Record not found"
+            }
+          })
       }
     }
   end

lib/pleroma/web/api_spec/schemas/status.ex

@@ -194,6 +194,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
           parent_visible: %Schema{
             type: :boolean,
             description: "`true` if the parent post is visible to the user"
+          },
+          pinned_at: %Schema{
+            type: :string,
+            format: "date-time",
+            nullable: true,
+            description:
+              "A datetime (ISO 8601) that states when the post was pinned or `null` if the post is not pinned"
           }
         }
       },

lib/pleroma/web/common_api.ex

@@ -228,17 +228,7 @@ def favorite_helper(user, id) do
       {:find_object, _} ->
         {:error, :not_found}
 
-      {:common_pipeline,
+      {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e ->
-       {
-         :error,
-         {
-           :validate_object,
-           {
-             :error,
-             changeset
-           }
-         }
-       }} = e ->
         if {:object, {"already liked by this actor", []}} in changeset.errors do
           {:ok, :already_liked}
         else
@@ -411,29 +401,58 @@ def post(user, %{status: _} = data) do
     end
   end
 
-  def pin(id, %{ap_id: user_ap_id} = user) do
+  @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()}
-    with %Activity{
+  def pin(id, %User{} = user) do
-           actor: ^user_ap_id,
+    with %Activity{} = activity <- create_activity_by_id(id),
-           data: %{"type" => "Create"},
+         true <- activity_belongs_to_actor(activity, user.ap_id),
-           object: %Object{data: %{"type" => object_type}}
+         true <- object_type_is_allowed_for_pin(activity.object),
-         } = activity <- Activity.get_by_id_with_object(id),
+         true <- activity_is_public(activity),
-         true <- object_type in ["Note", "Article", "Question"],
+         {:ok, pin_data, _} <- Builder.pin(user, activity.object),
-         true <- Visibility.is_public?(activity),
+         {:ok, _pin, _} <-
-         {:ok, _user} <- User.add_pinnned_activity(user, activity) do
+           Pipeline.common_pipeline(pin_data,
+             local: true,
+             activity_id: id
+           ) do
       {:ok, activity}
     else
-      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err}
+      {:error, {:side_effects, error}} -> error
-      _ -> {:error, dgettext("errors", "Could not pin")}
+      error -> error
     end
   end
 
+  defp create_activity_by_id(id) do
+    with nil <- Activity.create_by_id_with_object(id) do
+      {:error, :not_found}
+    end
+  end
+
+  defp activity_belongs_to_actor(%{actor: actor}, actor), do: true
+  defp activity_belongs_to_actor(_, _), do: {:error, :ownership_error}
+
+  defp object_type_is_allowed_for_pin(%{data: %{"type" => type}}) do
+    with false <- type in ["Note", "Article", "Question"] do
+      {:error, :not_allowed}
+    end
+  end
+
+  defp activity_is_public(activity) do
+    with false <- Visibility.is_public?(activity) do
+      {:error, :visibility_error}
+    end
+  end
+
+  @spec unpin(String.t(), User.t()) :: {:ok, User.t()} | {:error, term()}
   def unpin(id, user) do
-    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
+    with %Activity{} = activity <- create_activity_by_id(id),
-         {:ok, _user} <- User.remove_pinnned_activity(user, activity) do
+         {:ok, unpin_data, _} <- Builder.unpin(user, activity.object),
+         {:ok, _unpin, _} <-
+           Pipeline.common_pipeline(unpin_data,
+             local: true,
+             activity_id: activity.id,
+             expires_at: activity.data["expires_at"],
+             featured_address: user.featured_address
+           ) do
       {:ok, activity}
-    else
-      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err}
-      _ -> {:error, dgettext("errors", "Could not unpin")}
     end
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -286,7 +286,7 @@ def format_input(text, "text/html", options) do
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
-    |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+    |> Formatter.markdown_to_html()
     |> Formatter.linkify(options)
     |> Formatter.html_escape("text/html")
   end

lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex

@@ -30,6 +30,12 @@ def call(conn, {:error, error_message}) do
     |> json(%{error: error_message})
   end
 
+  def call(conn, {:error, status, message}) do
+    conn
+    |> put_status(status)
+    |> json(%{error: message})
+  end
+
   def call(conn, _) do
     conn
     |> put_status(:internal_server_error)

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -260,6 +260,18 @@ def unfavourite(%{assigns: %{user: user}} = conn, %{id: activity_id}) do
   def pin(%{assigns: %{user: user}} = conn, %{id: ap_id_or_id}) do
     with {:ok, activity} <- CommonAPI.pin(ap_id_or_id, user) do
       try_render(conn, "show.json", activity: activity, for: user, as: :activity)
+    else
+      {:error, :pinned_statuses_limit_reached} ->
+        {:error, "You have already pinned the maximum number of statuses"}
+
+      {:error, :ownership_error} ->
+        {:error, :unprocessable_entity, "Someone else's status cannot be pinned"}
+
+      {:error, :visibility_error} ->
+        {:error, :unprocessable_entity, "Non-public status cannot be pinned"}
+
+      error ->
+        error
     end
   end
 

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -23,7 +23,8 @@ def render("show.json", _) do
         streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Pleroma.Stats.get_stats(),
-      thumbnail: Pleroma.Web.base_url() <> Keyword.get(instance, :instance_thumbnail),
+      thumbnail:
+        URI.merge(Pleroma.Web.base_url(), Keyword.get(instance, :instance_thumbnail)) |> to_string,
       languages: ["en"],
       registrations: Keyword.get(instance, :registrations_open),
       approval_required: Keyword.get(instance, :account_approval_required),

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -152,6 +152,8 @@ def render(
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)
 
+    {pinned?, pinned_at} = pin_data(object, user)
+
     %{
       id: to_string(activity.id),
       uri: object.data["id"],
@@ -173,7 +175,7 @@ def render(
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
       muted: false,
-      pinned: pinned?(activity, user),
+      pinned: pinned?,
       sensitive: false,
       spoiler_text: "",
       visibility: get_visibility(activity),
@@ -184,7 +186,8 @@ def render(
       language: nil,
       emojis: [],
       pleroma: %{
-        local: activity.local
+        local: activity.local,
+        pinned_at: pinned_at
       }
     }
   end
@@ -316,6 +319,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
           fn for_user, user -> User.mutes?(for_user, user) end
         )
 
+    {pinned?, pinned_at} = pin_data(object, user)
+
     %{
       id: to_string(activity.id),
       uri: object.data["id"],
@@ -339,7 +344,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
       muted: muted,
-      pinned: pinned?(activity, user),
+      pinned: pinned?,
       sensitive: sensitive,
       spoiler_text: summary,
       visibility: get_visibility(object),
@@ -360,7 +365,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
         direct_conversation_id: direct_conversation_id,
         thread_muted: thread_muted?,
         emoji_reactions: emoji_reactions,
-        parent_visible: visible_for_user?(reply_to, opts[:for])
+        parent_visible: visible_for_user?(reply_to, opts[:for]),
+        pinned_at: pinned_at
       }
     }
   end
@@ -529,8 +535,13 @@ defp present?(nil), do: false
   defp present?(false), do: false
   defp present?(_), do: true
 
-  defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}),
+  defp pin_data(%Object{data: %{"id" => object_id}}, %User{pinned_objects: pinned_objects}) do
-    do: id in pinned_activities
+    if pinned_at = pinned_objects[object_id] do
+      {true, Utils.to_masto_date(pinned_at)}
+    else
+      {false, nil}
+    end
+  end
 
   defp build_emoji_map(emoji, users, current_user) do
     %{

lib/pleroma/web/o_auth/o_auth_view.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.OAuth.OAuthView do
 
   def render("token.json", %{token: token} = opts) do
     response = %{
+      id: token.id,
       token_type: "Bearer",
       access_token: token.token,
       refresh_token: token.refresh_token,

lib/pleroma/web/plugs/http_security_plug.ex

@@ -48,7 +48,8 @@ def headers do
       {"x-content-type-options", "nosniff"},
       {"referrer-policy", referrer_policy},
       {"x-download-options", "noopen"},
-      {"content-security-policy", csp_string()}
+      {"content-security-policy", csp_string()},
+      {"permissions-policy", "interest-cohort=()"}
     ]
 
     headers =

lib/pleroma/web/router.ex

@@ -704,6 +704,7 @@ defmodule Pleroma.Web.Router do
     # The following two are S2S as well, see `ActivityPub.fetch_follow_information_for_user/1`:
     get("/users/:nickname/followers", ActivityPubController, :followers)
     get("/users/:nickname/following", ActivityPubController, :following)
+    get("/users/:nickname/collections/featured", ActivityPubController, :pinned)
   end
 
   scope "/", Pleroma.Web.ActivityPub do

mix.exs

@@ -38,7 +38,7 @@ def project do
           include_executables_for: [:unix],
           applications: [ex_syslogger: :load, syslog: :load, eldap: :transient],
           steps: [:assemble, &put_otp_version/1, &copy_files/1, &copy_nginx_config/1],
-          config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, nil}]
+          config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, []}]
         ]
       ]
     ]
@@ -144,7 +144,7 @@ defp deps do
       {:ex_aws, "~> 2.1.6"},
       {:ex_aws_s3, "~> 2.0"},
       {:sweet_xml, "~> 0.6.6"},
-      {:earmark, "1.4.3"},
+      {:earmark, "1.4.15"},
       {:bbcode_pleroma, "~> 0.2.0"},
       {:crypt,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/crypt.git",

mix.lock

@@ -27,8 +27,8 @@
   "db_connection": {:hex, :db_connection, "2.3.1", "4c9f3ed1ef37471cbdd2762d6655be11e38193904d9c5c1c9389f1b891a3088e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "abaab61780dde30301d840417890bd9f74131041afd02174cf4e10635b3a63f5"},
   "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
-  "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"},
+  "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"},
-  "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
   "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_explain": {:hex, :ecto_explain, "0.1.2", "a9d504cbd4adc809911f796d5ef7ebb17a576a6d32286c3d464c015bd39d5541", [:mix], [], "hexpm", "1d0e7798ae30ecf4ce34e912e5354a0c1c832b7ebceba39298270b9a9f316330"},

priv/repo/migrations/20210202110641_add_pinned_objects_to_users.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.AddPinnedObjectsToUsers do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add(:pinned_objects, :map)
+    end
+  end
+end

priv/repo/migrations/20210203141144_add_featured_address_to_users.exs

@@ -0,0 +1,23 @@
+defmodule Pleroma.Repo.Migrations.AddFeaturedAddressToUsers do
+  use Ecto.Migration
+
+  def up do
+    alter table(:users) do
+      add(:featured_address, :string)
+    end
+
+    create(index(:users, [:featured_address]))
+
+    execute("""
+
+    update users set featured_address = concat(ap_id, '/collections/featured') where local = true and featured_address is null;
+
+    """)
+  end
+
+  def down do
+    alter table(:users) do
+      remove(:featured_address)
+    end
+  end
+end

priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs

@@ -0,0 +1,28 @@
+defmodule Pleroma.Repo.Migrations.MovePinnedActivitiesIntoPinnedObjects do
+  use Ecto.Migration
+
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  def up do
+    from(u in User)
+    |> select([u], {u.id, fragment("?.pinned_activities", u)})
+    |> Repo.stream()
+    |> Stream.each(fn {user_id, pinned_activities_ids} ->
+      pinned_activities = Pleroma.Activity.all_by_ids_with_object(pinned_activities_ids)
+
+      pins =
+        Map.new(pinned_activities, fn %{object: %{data: %{"id" => object_id}}} ->
+          {object_id, NaiveDateTime.utc_now()}
+        end)
+
+      from(u in User, where: u.id == ^user_id)
+      |> Repo.update_all(set: [pinned_objects: pins])
+    end)
+    |> Stream.run()
+  end
+
+  def down, do: :noop
+end

priv/repo/migrations/20210206045221_remove_pinned_activities_from_users.exs

@@ -0,0 +1,15 @@
+defmodule Pleroma.Repo.Migrations.RemovePinnedActivitiesFromUsers do
+  use Ecto.Migration
+
+  def up do
+    alter table(:users) do
+      remove(:pinned_activities)
+    end
+  end
+
+  def down do
+    alter table(:users) do
+      add(:pinned_activities, {:array, :string}, default: [])
+    end
+  end
+end

priv/repo/migrations/20210401143153_user_notification_settings_fix.exs

@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.UserNotificationSettingsFix do
+  use Ecto.Migration
+
+  def up do
+    execute(~s(UPDATE users
+    SET 
+      notification_settings = '{"followers": true, "follows": true, "non_follows": true, "non_followers": true}'::jsonb WHERE notification_settings IS NULL
+))
+
+    execute("ALTER TABLE users
+    ALTER COLUMN notification_settings SET NOT NULL")
+  end
+
+  def down do
+    :ok
+  end
+end

priv/scrubbers/default.ex

@@ -39,6 +39,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_these_attributes(:code, [])
   Meta.allow_tag_with_these_attributes(:del, [])
   Meta.allow_tag_with_these_attributes(:em, [])
+  Meta.allow_tag_with_these_attributes(:hr, [])
   Meta.allow_tag_with_these_attributes(:i, [])
   Meta.allow_tag_with_these_attributes(:li, [])
   Meta.allow_tag_with_these_attributes(:ol, [])
@@ -58,6 +59,8 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"])
   Meta.allow_tag_with_these_attributes(:span, [])
 
+  Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"])
+
   @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images])
 
   if @allow_inline_images do

test/fixtures/config/temp.exported_from_db.secret.exs

@@ -0,0 +1,5 @@
+use Mix.Config
+
+config :pleroma, exported_config_merged: true
+
+config :pleroma, :first_setting, key: "new value"

test/fixtures/mastodon/collections/featured.json

@@ -0,0 +1,39 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://{{domain}}/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "id": "https://{{domain}}/users/{{nickname}}/collections/featured",
+  "orderedItems": [
+    {
+      "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://{{domain}}/schemas/litepub-0.1.jsonld",
+        {
+          "@language": "und"
+        }
+      ],
+      "actor": "https://{{domain}}/users/{{nickname}}",
+      "attachment": [],
+      "attributedTo": "https://{{domain}}/users/{{nickname}}",
+      "cc": [
+        "https://{{domain}}/users/{{nickname}}/followers"
+      ],
+      "content": "",
+      "id": "https://{{domain}}/objects/{{object_id}}",
+      "published": "2021-02-12T15:13:43.915429Z",
+      "sensitive": false,
+      "source": "",
+      "summary": "",
+      "tag": [],
+      "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+      ],
+      "type": "Note"
+    }
+  ],
+  "type": "OrderedCollection"
+}

test/fixtures/statuses/masto-note.json

@@ -0,0 +1,47 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "https://example.com/users/{{nickname}}/statuses/{{status_id}}",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": null,
+  "published": "2021-02-24T12:40:49Z",
+  "url": "https://example.com/@{{nickname}}/{{status_id}}",
+  "attributedTo": "https://example.com/users/{{nickname}}",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "https://example.com/users/{{nickname}}/followers"
+  ],
+  "sensitive": false,
+  "atomUri": "https://example.com/users/{{nickname}}/statuses/{{status_id}}",
+  "inReplyToAtomUri": null,
+  "conversation": "tag:example.com,2021-02-24:objectId=15:objectType=Conversation",
+  "content": "<p></p>",
+  "contentMap": {
+    "en": "<p></p>"
+  },
+  "attachment": [],
+  "tag": [],
+  "replies": {
+    "id": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies?only_other_accounts=true&page=true",
+      "partOf": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies",
+      "items": []
+    }
+  }
+}

test/fixtures/statuses/note.json

@@ -0,0 +1,27 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://example.com/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "actor": "https://example.com/users/{{nickname}}",
+  "attachment": [],
+  "attributedTo": "https://example.com/users/{{nickname}}",
+  "cc": [
+    "https://example.com/users/{{nickname}}/followers"
+  ],
+  "content": "Content",
+  "context": "https://example.com/contexts/e4b180e1-7403-477f-aeb4-de57e7a3fe7f",
+  "conversation": "https://example.com/contexts/e4b180e1-7403-477f-aeb4-de57e7a3fe7f",
+  "id": "https://example.com/objects/{{object_id}}",
+  "published": "2019-12-15T22:00:05.279583Z",
+  "sensitive": false,
+  "summary": "",
+  "tag": [],
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "type": "Note"
+}

test/fixtures/users_mock/masto_featured.json

@@ -0,0 +1,18 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount"
+    }
+  ],
+  "id": "https://{{domain}}/users/{{nickname}}/collections/featured",
+  "type": "OrderedCollection",
+  "totalItems": 0,
+  "orderedItems": []
+}

test/fixtures/users_mock/user.json

@@ -0,0 +1,42 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://example.com/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "attachment": [],
+  "endpoints": {
+    "oauthAuthorizationEndpoint": "https://example.com/oauth/authorize",
+    "oauthRegistrationEndpoint": "https://example.com/api/v1/apps",
+    "oauthTokenEndpoint": "https://example.com/oauth/token",
+    "sharedInbox": "https://example.com/inbox"
+  },
+  "followers": "https://example.com/users/{{nickname}}/followers",
+  "following": "https://example.com/users/{{nickname}}/following",
+  "icon": {
+    "type": "Image",
+    "url": "https://example.com/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"
+  },
+  "id": "https://example.com/users/{{nickname}}",
+  "image": {
+    "type": "Image",
+    "url": "https://example.com/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"
+  },
+  "inbox": "https://example.com/users/{{nickname}}/inbox",
+  "manuallyApprovesFollowers": false,
+  "name": "{{nickname}}",
+  "outbox": "https://example.com/users/{{nickname}}/outbox",
+  "preferredUsername": "{{nickname}}",
+  "publicKey": {
+    "id": "https://example.com/users/{{nickname}}#main-key",
+    "owner": "https://example.com/users/{{nickname}}",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+  },
+  "featured": "https://example.com/users/{{nickname}}/collections/featured",
+  "summary": "your friendly neighborhood pleroma developer<br>I like cute things and distributed systems, and really hate delete and redrafts",
+  "tag": [],
+  "type": "Person",
+  "url": "https://example.com/users/{{nickname}}"
+}

test/pleroma/activity_test.exs

@@ -254,4 +254,26 @@ test "get_by_object_ap_id_with_object/1" do
 
     assert %{id: ^id} = Activity.get_by_object_ap_id_with_object(obj_id)
   end
+
+  test "add_by_params_query/3" do
+    user = insert(:user)
+
+    note = insert(:note_activity, user: user)
+
+    insert(:add_activity, user: user, note: note)
+    insert(:add_activity, user: user, note: note)
+    insert(:add_activity, user: user)
+
+    assert Repo.aggregate(Activity, :count, :id) == 4
+
+    add_query =
+      Activity.add_by_params_query(note.data["object"], user.ap_id, user.featured_address)
+
+    assert Repo.aggregate(add_query, :count, :id) == 2
+
+    Repo.delete_all(add_query)
+    assert Repo.aggregate(add_query, :count, :id) == 0
+
+    assert Repo.aggregate(Activity, :count, :id) == 2
+  end
 end

test/pleroma/application_requirements_test.exs

@@ -35,13 +35,13 @@ test "doesn't raise if the pool size is unexpected but the respective flag is se
     setup do: clear_config([:welcome])
     setup do: clear_config([Pleroma.Emails.Mailer])
 
-    test "raises if welcome email enabled but mail disabled" do
+    test "warns if welcome email enabled but mail disabled" do
       clear_config([:welcome, :email, :enabled], true)
       clear_config([Pleroma.Emails.Mailer, :enabled], false)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError, "The mail disabled.", fn ->
+      assert capture_log(fn ->
-        capture_log(&Pleroma.ApplicationRequirements.verify!/0)
+               assert Pleroma.ApplicationRequirements.verify!() == :ok
-      end
+             end) =~ "Welcome emails will NOT be sent"
     end
   end
 
@@ -57,15 +57,13 @@ test "raises if welcome email enabled but mail disabled" do
 
     setup do: clear_config([:instance, :account_activation_required])
 
-    test "raises if account confirmation is required but mailer isn't enable" do
+    test "warns if account confirmation is required but mailer isn't enabled" do
       clear_config([:instance, :account_activation_required], true)
       clear_config([Pleroma.Emails.Mailer, :enabled], false)
 
-      assert_raise Pleroma.ApplicationRequirements.VerifyError,
+      assert capture_log(fn ->
-                   "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails.",
+               assert Pleroma.ApplicationRequirements.verify!() == :ok
-                   fn ->
+             end) =~ "Users will NOT be able to confirm their accounts"
-                     capture_log(&Pleroma.ApplicationRequirements.verify!/0)
-                   end
     end
 
     test "doesn't do anything if account confirmation is disabled" do

test/pleroma/config/release_runtime_provider_test.exs

@@ -0,0 +1,45 @@
+defmodule Pleroma.Config.ReleaseRuntimeProviderTest do
+  use ExUnit.Case, async: true
+
+  alias Pleroma.Config.ReleaseRuntimeProvider
+
+  describe "load/2" do
+    test "loads release defaults config and warns about non-existent runtime config" do
+      ExUnit.CaptureIO.capture_io(fn ->
+        merged = ReleaseRuntimeProvider.load([], [])
+        assert merged == Pleroma.Config.Holder.release_defaults()
+      end) =~
+        "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file"
+    end
+
+    test "merged runtime config" do
+      merged =
+        ReleaseRuntimeProvider.load([], config_path: "test/fixtures/config/temp.secret.exs")
+
+      assert merged[:pleroma][:first_setting] == [key: "value", key2: [Pleroma.Repo]]
+      assert merged[:pleroma][:second_setting] == [key: "value2", key2: ["Activity"]]
+    end
+
+    test "merged exported config" do
+      ExUnit.CaptureIO.capture_io(fn ->
+        merged =
+          ReleaseRuntimeProvider.load([],
+            exported_config_path: "test/fixtures/config/temp.exported_from_db.secret.exs"
+          )
+
+        assert merged[:pleroma][:exported_config_merged]
+      end) =~
+        "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file"
+    end
+
+    test "runtime config is merged with exported config" do
+      merged =
+        ReleaseRuntimeProvider.load([],
+          config_path: "test/fixtures/config/temp.secret.exs",
+          exported_config_path: "test/fixtures/config/temp.exported_from_db.secret.exs"
+        )
+
+      assert merged[:pleroma][:first_setting] == [key2: [Pleroma.Repo], key: "new value"]
+    end
+  end
+end

test/pleroma/earmark_renderer_test.exs

@@ -1,79 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.EarmarkRendererTest do
-  use Pleroma.DataCase, async: true
-
-  test "Paragraph" do
-    code = ~s[Hello\n\nWorld!]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<p>Hello</p><p>World!</p>"
-  end
-
-  test "raw HTML" do
-    code = ~s[<a href="http://example.org/">OwO</a><!-- what's this?-->]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<p>#{code}</p>"
-  end
-
-  test "rulers" do
-    code = ~s[before\n\n-----\n\nafter]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<p>before</p><hr /><p>after</p>"
-  end
-
-  test "headings" do
-    code = ~s[# h1\n## h2\n### h3\n]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<h1>h1</h1><h2>h2</h2><h3>h3</h3>]
-  end
-
-  test "blockquote" do
-    code = ~s[> whoms't are you quoting?]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<blockquote><p>whoms’t are you quoting?</p></blockquote>"
-  end
-
-  test "code" do
-    code = ~s[`mix`]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<p><code class="inline">mix</code></p>]
-
-    code = ~s[``mix``]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<p><code class="inline">mix</code></p>]
-
-    code = ~s[```\nputs "Hello World"\n```]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<pre><code class="">puts &quot;Hello World&quot;</code></pre>]
-  end
-
-  test "lists" do
-    code = ~s[- one\n- two\n- three\n- four]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>"
-
-    code = ~s[1. one\n2. two\n3. three\n4. four\n]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<ol><li>one</li><li>two</li><li>three</li><li>four</li></ol>"
-  end
-
-  test "delegated renderers" do
-    code = ~s[a<br/>b]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == "<p>#{code}</p>"
-
-    code = ~s[*aaaa~*]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<p><em>aaaa~</em></p>]
-
-    code = ~s[**aaaa~**]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<p><strong>aaaa~</strong></p>]
-
-    # strikethrought
-    code = ~s[<del>aaaa~</del>]
-    result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
-    assert result == ~s[<p><del>aaaa~</del></p>]
-  end
-end

test/pleroma/user_test.exs

@@ -572,6 +572,24 @@ test "it sends a registration confirmed email if no others will be sent" do
       )
     end
 
+    test "it fails gracefully with invalid email config" do
+      cng = User.register_changeset(%User{}, @full_user_data)
+
+      # Disable the mailer but enable all the things that want to send emails
+      clear_config([Pleroma.Emails.Mailer, :enabled], false)
+      clear_config([:instance, :account_activation_required], true)
+      clear_config([:instance, :account_approval_required], true)
+      clear_config([:welcome, :email, :enabled], true)
+      clear_config([:welcome, :email, :sender], "lain@lain.com")
+
+      # The user is still created
+      assert {:ok, %User{nickname: "nick"}} = User.register(cng)
+
+      # No emails are sent
+      ObanHelpers.perform_all()
+      refute_email_sent()
+    end
+
     test "it requires an email, name, nickname and password, bio is optional when account_activation_required is enabled" do
       clear_config([:instance, :account_activation_required], true)
 
@@ -2338,4 +2356,49 @@ test "active_user_count/1" do
     assert User.active_user_count(6) == 3
     assert User.active_user_count(1) == 1
   end
+
+  describe "pins" do
+    setup do
+      user = insert(:user)
+
+      [user: user, object_id: object_id_from_created_activity(user)]
+    end
+
+    test "unique pins", %{user: user, object_id: object_id} do
+      assert {:ok, %{pinned_objects: %{^object_id => pinned_at1} = pins} = updated_user} =
+               User.add_pinned_object_id(user, object_id)
+
+      assert Enum.count(pins) == 1
+
+      assert {:ok, %{pinned_objects: %{^object_id => pinned_at2} = pins}} =
+               User.add_pinned_object_id(updated_user, object_id)
+
+      assert pinned_at1 == pinned_at2
+
+      assert Enum.count(pins) == 1
+    end
+
+    test "respects max_pinned_statuses limit", %{user: user, object_id: object_id} do
+      clear_config([:instance, :max_pinned_statuses], 1)
+      {:ok, updated} = User.add_pinned_object_id(user, object_id)
+
+      object_id2 = object_id_from_created_activity(user)
+
+      {:error, %{errors: errors}} = User.add_pinned_object_id(updated, object_id2)
+      assert Keyword.has_key?(errors, :pinned_objects)
+    end
+
+    test "remove_pinned_object_id/2", %{user: user, object_id: object_id} do
+      assert {:ok, updated} = User.add_pinned_object_id(user, object_id)
+
+      {:ok, after_remove} = User.remove_pinned_object_id(updated, object_id)
+      assert after_remove.pinned_objects == %{}
+    end
+  end
+
+  defp object_id_from_created_activity(user) do
+    %{id: id} = insert(:note_activity, user: user)
+    %{object: %{data: %{"id" => object_id}}} = Activity.get_by_id_with_object(id)
+    object_id
+  end
 end

test/pleroma/web/activity_pub/activity_pub_controller_test.exs

@@ -636,6 +636,186 @@ test "without valid signature, " <>
       |> post("/inbox", non_create_data)
       |> json_response(400)
     end
+
+    test "accepts Add/Remove activities", %{conn: conn} do
+      object_id = "c61d6733-e256-4fe1-ab13-1e369789423f"
+
+      status =
+        File.read!("test/fixtures/statuses/note.json")
+        |> String.replace("{{nickname}}", "lain")
+        |> String.replace("{{object_id}}", object_id)
+
+      object_url = "https://example.com/objects/#{object_id}"
+
+      user =
+        File.read!("test/fixtures/users_mock/user.json")
+        |> String.replace("{{nickname}}", "lain")
+
+      actor = "https://example.com/users/lain"
+
+      Tesla.Mock.mock(fn
+        %{
+          method: :get,
+          url: ^object_url
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: status,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+
+        %{
+          method: :get,
+          url: ^actor
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: user,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+
+        %{method: :get, url: "https://example.com/users/lain/collections/featured"} ->
+          %Tesla.Env{
+            status: 200,
+            body:
+              "test/fixtures/users_mock/masto_featured.json"
+              |> File.read!()
+              |> String.replace("{{domain}}", "example.com")
+              |> String.replace("{{nickname}}", "lain"),
+            headers: [{"content-type", "application/activity+json"}]
+          }
+      end)
+
+      data = %{
+        "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f",
+        "actor" => actor,
+        "object" => object_url,
+        "target" => "https://example.com/users/lain/collections/featured",
+        "type" => "Add",
+        "to" => [Pleroma.Constants.as_public()]
+      }
+
+      assert "ok" ==
+               conn
+               |> assign(:valid_signature, true)
+               |> put_req_header("content-type", "application/activity+json")
+               |> post("/inbox", data)
+               |> json_response(200)
+
+      ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
+      assert Activity.get_by_ap_id(data["id"])
+      user = User.get_cached_by_ap_id(data["actor"])
+      assert user.pinned_objects[data["object"]]
+
+      data = %{
+        "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423d",
+        "actor" => actor,
+        "object" => object_url,
+        "target" => "https://example.com/users/lain/collections/featured",
+        "type" => "Remove",
+        "to" => [Pleroma.Constants.as_public()]
+      }
+
+      assert "ok" ==
+               conn
+               |> assign(:valid_signature, true)
+               |> put_req_header("content-type", "application/activity+json")
+               |> post("/inbox", data)
+               |> json_response(200)
+
+      ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
+      user = refresh_record(user)
+      refute user.pinned_objects[data["object"]]
+    end
+
+    test "mastodon pin/unpin", %{conn: conn} do
+      status_id = "105786274556060421"
+
+      status =
+        File.read!("test/fixtures/statuses/masto-note.json")
+        |> String.replace("{{nickname}}", "lain")
+        |> String.replace("{{status_id}}", status_id)
+
+      status_url = "https://example.com/users/lain/statuses/#{status_id}"
+
+      user =
+        File.read!("test/fixtures/users_mock/user.json")
+        |> String.replace("{{nickname}}", "lain")
+
+      actor = "https://example.com/users/lain"
+
+      Tesla.Mock.mock(fn
+        %{
+          method: :get,
+          url: ^status_url
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: status,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+
+        %{
+          method: :get,
+          url: ^actor
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: user,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+
+        %{method: :get, url: "https://example.com/users/lain/collections/featured"} ->
+          %Tesla.Env{
+            status: 200,
+            body:
+              "test/fixtures/users_mock/masto_featured.json"
+              |> File.read!()
+              |> String.replace("{{domain}}", "example.com")
+              |> String.replace("{{nickname}}", "lain"),
+            headers: [{"content-type", "application/activity+json"}]
+          }
+      end)
+
+      data = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "actor" => actor,
+        "object" => status_url,
+        "target" => "https://example.com/users/lain/collections/featured",
+        "type" => "Add"
+      }
+
+      assert "ok" ==
+               conn
+               |> assign(:valid_signature, true)
+               |> put_req_header("content-type", "application/activity+json")
+               |> post("/inbox", data)
+               |> json_response(200)
+
+      ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
+      assert Activity.get_by_object_ap_id_with_object(data["object"])
+      user = User.get_cached_by_ap_id(data["actor"])
+      assert user.pinned_objects[data["object"]]
+
+      data = %{
+        "actor" => actor,
+        "object" => status_url,
+        "target" => "https://example.com/users/lain/collections/featured",
+        "type" => "Remove"
+      }
+
+      assert "ok" ==
+               conn
+               |> assign(:valid_signature, true)
+               |> put_req_header("content-type", "application/activity+json")
+               |> post("/inbox", data)
+               |> json_response(200)
+
+      ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
+      assert Activity.get_by_object_ap_id_with_object(data["object"])
+      user = refresh_record(user)
+      refute user.pinned_objects[data["object"]]
+    end
   end
 
   describe "/users/:nickname/inbox" do
@@ -1772,4 +1952,29 @@ test "POST /api/ap/upload_media", %{conn: conn} do
       |> json_response(403)
     end
   end
+
+  test "pinned collection", %{conn: conn} do
+    clear_config([:instance, :max_pinned_statuses], 2)
+    user = insert(:user)
+    objects = insert_list(2, :note, user: user)
+
+    Enum.reduce(objects, user, fn %{data: %{"id" => object_id}}, user ->
+      {:ok, updated} = User.add_pinned_object_id(user, object_id)
+      updated
+    end)
+
+    %{nickname: nickname, featured_address: featured_address, pinned_objects: pinned_objects} =
+      refresh_record(user)
+
+    %{"id" => ^featured_address, "orderedItems" => items} =
+      conn
+      |> get("/users/#{nickname}/collections/featured")
+      |> json_response(200)
+
+    object_ids = Enum.map(items, & &1["id"])
+
+    assert Enum.all?(pinned_objects, fn {obj_id, _} ->
+             obj_id in object_ids
+           end)
+  end
 end

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -235,6 +235,83 @@ test "works for bridgy actors" do
                "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}]
              }
     end
+
+    test "fetches user featured collection" do
+      ap_id = "https://example.com/users/lain"
+
+      featured_url = "https://example.com/users/lain/collections/featured"
+
+      user_data =
+        "test/fixtures/users_mock/user.json"
+        |> File.read!()
+        |> String.replace("{{nickname}}", "lain")
+        |> Jason.decode!()
+        |> Map.put("featured", featured_url)
+        |> Jason.encode!()
+
+      object_id = Ecto.UUID.generate()
+
+      featured_data =
+        "test/fixtures/mastodon/collections/featured.json"
+        |> File.read!()
+        |> String.replace("{{domain}}", "example.com")
+        |> String.replace("{{nickname}}", "lain")
+        |> String.replace("{{object_id}}", object_id)
+
+      object_url = "https://example.com/objects/#{object_id}"
+
+      object_data =
+        "test/fixtures/statuses/note.json"
+        |> File.read!()
+        |> String.replace("{{object_id}}", object_id)
+        |> String.replace("{{nickname}}", "lain")
+
+      Tesla.Mock.mock(fn
+        %{
+          method: :get,
+          url: ^ap_id
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: user_data,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+
+        %{
+          method: :get,
+          url: ^featured_url
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: featured_data,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+      end)
+
+      Tesla.Mock.mock_global(fn
+        %{
+          method: :get,
+          url: ^object_url
+        } ->
+          %Tesla.Env{
+            status: 200,
+            body: object_data,
+            headers: [{"content-type", "application/activity+json"}]
+          }
+      end)
+
+      {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id)
+      Process.sleep(50)
+
+      assert user.featured_address == featured_url
+      assert Map.has_key?(user.pinned_objects, object_url)
+
+      in_db = Pleroma.User.get_by_ap_id(ap_id)
+      assert in_db.featured_address == featured_url
+      assert Map.has_key?(user.pinned_objects, object_url)
+
+      assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url)
+    end
   end
 
   test "it fetches the appropriate tag-restricted posts" do

test/pleroma/web/activity_pub/mrf/follow_bot_policy_test.exs

@@ -0,0 +1,126 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicyTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.MRF.FollowBotPolicy
+
+  import Pleroma.Factory
+
+  describe "FollowBotPolicy" do
+    test "follows remote users" do
+      bot = insert(:user, actor_type: "Service")
+      remote_user = insert(:user, local: false)
+      clear_config([:mrf_follow_bot, :follower_nickname], bot.nickname)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => [remote_user.follower_address],
+        "cc" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Create",
+        "object" => %{
+          "content" => "Test post",
+          "type" => "Note",
+          "attributedTo" => remote_user.ap_id,
+          "inReplyTo" => nil
+        },
+        "actor" => remote_user.ap_id
+      }
+
+      refute User.following?(bot, remote_user)
+
+      assert User.get_follow_requests(remote_user) |> length == 0
+
+      FollowBotPolicy.filter(message)
+
+      assert User.get_follow_requests(remote_user) |> length == 1
+    end
+
+    test "does not follow users with #nobot in bio" do
+      bot = insert(:user, actor_type: "Service")
+      remote_user = insert(:user, %{local: false, bio: "go away bots! #nobot"})
+      clear_config([:mrf_follow_bot, :follower_nickname], bot.nickname)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => [remote_user.follower_address],
+        "cc" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Create",
+        "object" => %{
+          "content" => "I don't like follow bots",
+          "type" => "Note",
+          "attributedTo" => remote_user.ap_id,
+          "inReplyTo" => nil
+        },
+        "actor" => remote_user.ap_id
+      }
+
+      refute User.following?(bot, remote_user)
+
+      assert User.get_follow_requests(remote_user) |> length == 0
+
+      FollowBotPolicy.filter(message)
+
+      assert User.get_follow_requests(remote_user) |> length == 0
+    end
+
+    test "does not follow local users" do
+      bot = insert(:user, actor_type: "Service")
+      local_user = insert(:user, local: true)
+      clear_config([:mrf_follow_bot, :follower_nickname], bot.nickname)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => [local_user.follower_address],
+        "cc" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Create",
+        "object" => %{
+          "content" => "Hi I'm a local user",
+          "type" => "Note",
+          "attributedTo" => local_user.ap_id,
+          "inReplyTo" => nil
+        },
+        "actor" => local_user.ap_id
+      }
+
+      refute User.following?(bot, local_user)
+
+      assert User.get_follow_requests(local_user) |> length == 0
+
+      FollowBotPolicy.filter(message)
+
+      assert User.get_follow_requests(local_user) |> length == 0
+    end
+
+    test "does not follow users requiring follower approval" do
+      bot = insert(:user, actor_type: "Service")
+      remote_user = insert(:user, %{local: false, is_locked: true})
+      clear_config([:mrf_follow_bot, :follower_nickname], bot.nickname)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "to" => [remote_user.follower_address],
+        "cc" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Create",
+        "object" => %{
+          "content" => "I don't like randos following me",
+          "type" => "Note",
+          "attributedTo" => remote_user.ap_id,
+          "inReplyTo" => nil
+        },
+        "actor" => remote_user.ap_id
+      }
+
+      refute User.following?(bot, remote_user)
+
+      assert User.get_follow_requests(remote_user) |> length == 0
+
+      FollowBotPolicy.filter(message)
+
+      assert User.get_follow_requests(remote_user) |> length == 0
+    end
+  end
+end

test/pleroma/web/activity_pub/pipeline_test.exs

@@ -25,9 +25,6 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
       MRFMock
       |> expect(:pipeline_filter, fn o, m -> {:ok, o, m} end)
 
-      ActivityPubMock
-      |> expect(:persist, fn o, m -> {:ok, o, m} end)
-
       SideEffectsMock
       |> expect(:handle, fn o, m -> {:ok, o, m} end)
       |> expect(:handle_after_transaction, fn m -> m end)
@@ -42,6 +39,9 @@ test "when given an `object_data` in meta, Federation will receive a the origina
 
       activity_with_object = %{activity | data: Map.put(activity.data, "object", object)}
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       FederatorMock
       |> expect(:publish, fn ^activity_with_object -> :ok end)
 
@@ -50,7 +50,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
 
       assert {:ok, ^activity, ^meta} =
                Pleroma.Web.ActivityPub.Pipeline.common_pipeline(
-                 activity,
+                 activity.data,
                  meta
                )
     end
@@ -59,6 +59,9 @@ test "it goes through validation, filtering, persisting, side effects and federa
       activity = insert(:note_activity)
       meta = [local: true]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       FederatorMock
       |> expect(:publish, fn ^activity -> :ok end)
 
@@ -66,29 +69,35 @@ test "it goes through validation, filtering, persisting, side effects and federa
       |> expect(:get, fn [:instance, :federating] -> true end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
 
     test "it goes through validation, filtering, persisting, side effects without federation for remote activities" do
       activity = insert(:note_activity)
       meta = [local: false]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       ConfigMock
       |> expect(:get, fn [:instance, :federating] -> true end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
 
     test "it goes through validation, filtering, persisting, side effects without federation for local activities if federation is deactivated" do
       activity = insert(:note_activity)
       meta = [local: true]
 
+      ActivityPubMock
+      |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
       ConfigMock
       |> expect(:get, fn [:instance, :federating] -> false end)
 
       assert {:ok, ^activity, ^meta} =
-               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+               Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
     end
   end
 end

test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs

@@ -0,0 +1,172 @@
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.AddRemoveHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase, async: true
+
+  require Pleroma.Constants
+
+  import Pleroma.Factory
+
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  test "it accepts Add/Remove activities" do
+    user =
+      "test/fixtures/users_mock/user.json"
+      |> File.read!()
+      |> String.replace("{{nickname}}", "lain")
+
+    object_id = "c61d6733-e256-4fe1-ab13-1e369789423f"
+
+    object =
+      "test/fixtures/statuses/note.json"
+      |> File.read!()
+      |> String.replace("{{nickname}}", "lain")
+      |> String.replace("{{object_id}}", object_id)
+
+    object_url = "https://example.com/objects/#{object_id}"
+
+    actor = "https://example.com/users/lain"
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: ^actor
+      } ->
+        %Tesla.Env{
+          status: 200,
+          body: user,
+          headers: [{"content-type", "application/activity+json"}]
+        }
+
+      %{
+        method: :get,
+        url: ^object_url
+      } ->
+        %Tesla.Env{
+          status: 200,
+          body: object,
+          headers: [{"content-type", "application/activity+json"}]
+        }
+
+      %{method: :get, url: "https://example.com/users/lain/collections/featured"} ->
+        %Tesla.Env{
+          status: 200,
+          body:
+            "test/fixtures/users_mock/masto_featured.json"
+            |> File.read!()
+            |> String.replace("{{domain}}", "example.com")
+            |> String.replace("{{nickname}}", "lain"),
+          headers: [{"content-type", "application/activity+json"}]
+        }
+    end)
+
+    message = %{
+      "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f",
+      "actor" => actor,
+      "object" => object_url,
+      "target" => "https://example.com/users/lain/collections/featured",
+      "type" => "Add",
+      "to" => [Pleroma.Constants.as_public()],
+      "cc" => ["https://example.com/users/lain/followers"]
+    }
+
+    assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+    assert activity.data == message
+    user = User.get_cached_by_ap_id(actor)
+    assert user.pinned_objects[object_url]
+
+    remove = %{
+      "id" => "http://localhost:400/objects/d61d6733-e256-4fe1-ab13-1e369789423d",
+      "actor" => actor,
+      "object" => object_url,
+      "target" => "https://example.com/users/lain/collections/featured",
+      "type" => "Remove",
+      "to" => [Pleroma.Constants.as_public()],
+      "cc" => ["https://example.com/users/lain/followers"]
+    }
+
+    assert {:ok, activity} = Transmogrifier.handle_incoming(remove)
+    assert activity.data == remove
+
+    user = refresh_record(user)
+    refute user.pinned_objects[object_url]
+  end
+
+  test "Add/Remove activities for remote users without featured address" do
+    user = insert(:user, local: false, domain: "example.com")
+
+    user =
+      user
+      |> Ecto.Changeset.change(featured_address: nil)
+      |> Repo.update!()
+
+    %{host: host} = URI.parse(user.ap_id)
+
+    user_data =
+      "test/fixtures/users_mock/user.json"
+      |> File.read!()
+      |> String.replace("{{nickname}}", user.nickname)
+
+    object_id = "c61d6733-e256-4fe1-ab13-1e369789423f"
+
+    object =
+      "test/fixtures/statuses/note.json"
+      |> File.read!()
+      |> String.replace("{{nickname}}", user.nickname)
+      |> String.replace("{{object_id}}", object_id)
+
+    object_url = "https://#{host}/objects/#{object_id}"
+
+    actor = "https://#{host}/users/#{user.nickname}"
+
+    featured = "https://#{host}/users/#{user.nickname}/collections/featured"
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: ^actor
+      } ->
+        %Tesla.Env{
+          status: 200,
+          body: user_data,
+          headers: [{"content-type", "application/activity+json"}]
+        }
+
+      %{
+        method: :get,
+        url: ^object_url
+      } ->
+        %Tesla.Env{
+          status: 200,
+          body: object,
+          headers: [{"content-type", "application/activity+json"}]
+        }
+
+      %{method: :get, url: ^featured} ->
+        %Tesla.Env{
+          status: 200,
+          body:
+            "test/fixtures/users_mock/masto_featured.json"
+            |> File.read!()
+            |> String.replace("{{domain}}", "#{host}")
+            |> String.replace("{{nickname}}", user.nickname),
+          headers: [{"content-type", "application/activity+json"}]
+        }
+    end)
+
+    message = %{
+      "id" => "https://#{host}/objects/d61d6733-e256-4fe1-ab13-1e369789423f",
+      "actor" => actor,
+      "object" => object_url,
+      "target" => "https://#{host}/users/#{user.nickname}/collections/featured",
+      "type" => "Add",
+      "to" => [Pleroma.Constants.as_public()],
+      "cc" => ["https://#{host}/users/#{user.nickname}/followers"]
+    }
+
+    assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+    assert activity.data == message
+    user = User.get_cached_by_ap_id(actor)
+    assert user.pinned_objects[object_url]
+  end
+end

test/pleroma/web/admin_api/controllers/config_controller_test.exs

@@ -1410,6 +1410,82 @@ test "enables the welcome messages", %{conn: conn} do
                "need_reboot" => false
              }
     end
+
+    test "custom instance thumbnail", %{conn: conn} do
+      clear_config([:instance])
+
+      params = %{
+        "group" => ":pleroma",
+        "key" => ":instance",
+        "value" => [
+          %{
+            "tuple" => [
+              ":instance_thumbnail",
+              "https://example.com/media/new_thumbnail.jpg"
+            ]
+          }
+        ]
+      }
+
+      res =
+        assert conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/admin/config", %{"configs" => [params]})
+               |> json_response_and_validate_schema(200)
+
+      assert res == %{
+               "configs" => [
+                 %{
+                   "db" => [":instance_thumbnail"],
+                   "group" => ":pleroma",
+                   "key" => ":instance",
+                   "value" => params["value"]
+                 }
+               ],
+               "need_reboot" => false
+             }
+
+      _res =
+        assert conn
+               |> get("/api/v1/instance")
+               |> json_response_and_validate_schema(200)
+
+      assert res = %{"thumbnail" => "https://example.com/media/new_thumbnail.jpg"}
+    end
+
+    test "Concurrent Limiter", %{conn: conn} do
+      clear_config([ConcurrentLimiter])
+
+      params = %{
+        "group" => ":pleroma",
+        "key" => "ConcurrentLimiter",
+        "value" => [
+          %{
+            "tuple" => [
+              "Pleroma.Web.RichMedia.Helpers",
+              [
+                %{"tuple" => [":max_running", 6]},
+                %{"tuple" => [":max_waiting", 6]}
+              ]
+            ]
+          },
+          %{
+            "tuple" => [
+              "Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy",
+              [
+                %{"tuple" => [":max_running", 7]},
+                %{"tuple" => [":max_waiting", 7]}
+              ]
+            ]
+          }
+        ]
+      }
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/pleroma/admin/config", %{"configs" => [params]})
+             |> json_response_and_validate_schema(200)
+    end
   end
 
   describe "GET /api/pleroma/admin/config/descriptions" do

test/pleroma/web/common_api/utils_test.exs

@@ -168,6 +168,123 @@ test "works for text/markdown with mentions" do
     end
   end
 
+  describe "format_input/3 with markdown" do
+    test "Paragraph" do
+      code = ~s[Hello\n\nWorld!]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == "<p>Hello</p><p>World!</p>"
+    end
+
+    test "links" do
+      code = "https://en.wikipedia.org/wiki/Animal_Crossing_(video_game)"
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
+
+      code = "https://github.com/pragdave/earmark/"
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
+    end
+
+    test "link with local mention" do
+      insert(:user, %{nickname: "lain"})
+
+      code = "https://example.com/@lain"
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><a href="#{code}">#{code}</a></p>]
+    end
+
+    test "local mentions" do
+      mario = insert(:user, %{nickname: "mario"})
+      luigi = insert(:user, %{nickname: "luigi"})
+
+      code = "@mario @luigi yo what's up?"
+      {result, _, []} = Utils.format_input(code, "text/markdown")
+
+      assert result ==
+               ~s[<p><span class="h-card"><a class="u-url mention" data-user="#{mario.id}" href="#{
+                 mario.ap_id
+               }" rel="ugc">@<span>mario</span></a></span> <span class="h-card"><a class="u-url mention" data-user="#{
+                 luigi.id
+               }" href="#{luigi.ap_id}" rel="ugc">@<span>luigi</span></a></span> yo what’s up?</p>]
+    end
+
+    test "remote mentions" do
+      mario = insert(:user, %{nickname: "mario@mushroom.world", local: false})
+      luigi = insert(:user, %{nickname: "luigi@mushroom.world", local: false})
+
+      code = "@mario@mushroom.world @luigi@mushroom.world yo what's up?"
+      {result, _, []} = Utils.format_input(code, "text/markdown")
+
+      assert result ==
+               ~s[<p><span class="h-card"><a class="u-url mention" data-user="#{mario.id}" href="#{
+                 mario.ap_id
+               }" rel="ugc">@<span>mario</span></a></span> <span class="h-card"><a class="u-url mention" data-user="#{
+                 luigi.id
+               }" href="#{luigi.ap_id}" rel="ugc">@<span>luigi</span></a></span> yo what’s up?</p>]
+    end
+
+    test "raw HTML" do
+      code = ~s[<a href="http://example.org/">OwO</a><!-- what's this?-->]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<a href="http://example.org/">OwO</a>]
+    end
+
+    test "rulers" do
+      code = ~s[before\n\n-----\n\nafter]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == "<p>before</p><hr/><p>after</p>"
+    end
+
+    test "blockquote" do
+      code = ~s[> whoms't are you quoting?]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == "<blockquote><p>whoms’t are you quoting?</p></blockquote>"
+    end
+
+    test "code" do
+      code = ~s[`mix`]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><code class="inline">mix</code></p>]
+
+      code = ~s[``mix``]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><code class="inline">mix</code></p>]
+
+      code = ~s[```\nputs "Hello World"\n```]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<pre><code>puts &quot;Hello World&quot;</code></pre>]
+
+      code = ~s[    <div>\n    </div>]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<pre><code>&lt;div&gt;\n&lt;/div&gt;</code></pre>]
+    end
+
+    test "lists" do
+      code = ~s[- one\n- two\n- three\n- four]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == "<ul><li>one</li><li>two</li><li>three</li><li>four</li></ul>"
+
+      code = ~s[1. one\n2. two\n3. three\n4. four\n]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == "<ol><li>one</li><li>two</li><li>three</li><li>four</li></ol>"
+    end
+
+    test "delegated renderers" do
+      code = ~s[*aaaa~*]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><em>aaaa~</em></p>]
+
+      code = ~s[**aaaa~**]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><strong>aaaa~</strong></p>]
+
+      # strikethrough
+      code = ~s[~~aaaa~~~]
+      {result, [], []} = Utils.format_input(code, "text/markdown")
+      assert result == ~s[<p><del>aaaa</del>~</p>]
+    end
+  end
+
   describe "context_to_conversation_id" do
     test "creates a mapping object" do
       conversation_id = Utils.context_to_conversation_id("random context")

test/pleroma/web/common_api_test.exs

@@ -597,7 +597,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do
 
       object = Object.normalize(activity, fetch: false)
 
-      assert object.data["content"] == "<p><b>2hu</b></p>alert(&#39;xss&#39;)"
+      assert object.data["content"] == "<p><b>2hu</b></p>"
       assert object.data["source"] == post
     end
 
@@ -827,13 +827,17 @@ test "favoriting a status twice returns ok, but without the like activity" do
       [user: user, activity: activity]
     end
 
+    test "activity not found error", %{user: user} do
+      assert {:error, :not_found} = CommonAPI.pin("id", user)
+    end
+
     test "pin status", %{user: user, activity: activity} do
       assert {:ok, ^activity} = CommonAPI.pin(activity.id, user)
 
-      id = activity.id
+      %{data: %{"id" => object_id}} = Object.normalize(activity)
       user = refresh_record(user)
 
-      assert %User{pinned_activities: [^id]} = user
+      assert user.pinned_objects |> Map.keys() == [object_id]
     end
 
     test "pin poll", %{user: user} do
@@ -845,10 +849,11 @@ test "pin poll", %{user: user} do
 
       assert {:ok, ^activity} = CommonAPI.pin(activity.id, user)
 
-      id = activity.id
+      %{data: %{"id" => object_id}} = Object.normalize(activity)
+
       user = refresh_record(user)
 
-      assert %User{pinned_activities: [^id]} = user
+      assert user.pinned_objects |> Map.keys() == [object_id]
     end
 
     test "unlisted statuses can be pinned", %{user: user} do
@@ -859,7 +864,7 @@ test "unlisted statuses can be pinned", %{user: user} do
     test "only self-authored can be pinned", %{activity: activity} do
       user = insert(:user)
 
-      assert {:error, "Could not pin"} = CommonAPI.pin(activity.id, user)
+      assert {:error, :ownership_error} = CommonAPI.pin(activity.id, user)
     end
 
     test "max pinned statuses", %{user: user, activity: activity_one} do
@@ -869,8 +874,12 @@ test "max pinned statuses", %{user: user, activity: activity_one} do
 
       user = refresh_record(user)
 
-      assert {:error, "You have already pinned the maximum number of statuses"} =
+      assert {:error, :pinned_statuses_limit_reached} = CommonAPI.pin(activity_two.id, user)
-               CommonAPI.pin(activity_two.id, user)
+    end
+
+    test "only public can be pinned", %{user: user} do
+      {:ok, activity} = CommonAPI.post(user, %{status: "private status", visibility: "private"})
+      {:error, :visibility_error} = CommonAPI.pin(activity.id, user)
     end
 
     test "unpin status", %{user: user, activity: activity} do
@@ -884,7 +893,7 @@ test "unpin status", %{user: user, activity: activity} do
 
       user = refresh_record(user)
 
-      assert %User{pinned_activities: []} = user
+      assert user.pinned_objects == %{}
     end
 
     test "should unpin when deleting a status", %{user: user, activity: activity} do
@@ -896,7 +905,40 @@ test "should unpin when deleting a status", %{user: user, activity: activity} do
 
       user = refresh_record(user)
 
-      assert %User{pinned_activities: []} = user
+      assert user.pinned_objects == %{}
+    end
+
+    test "ephemeral activity won't be deleted if was pinned", %{user: user} do
+      {:ok, activity} = CommonAPI.post(user, %{status: "Hello!", expires_in: 601})
+
+      assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id)
+
+      {:ok, _activity} = CommonAPI.pin(activity.id, user)
+      refute Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id)
+
+      user = refresh_record(user)
+      {:ok, _} = CommonAPI.unpin(activity.id, user)
+
+      # recreates expiration job on unpin
+      assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id)
+    end
+
+    test "ephemeral activity deletion job won't be deleted on pinning error", %{
+      user: user,
+      activity: activity
+    } do
+      clear_config([:instance, :max_pinned_statuses], 1)
+
+      {:ok, _activity} = CommonAPI.pin(activity.id, user)
+
+      {:ok, activity2} = CommonAPI.post(user, %{status: "another status", expires_in: 601})
+
+      assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity2.id)
+
+      user = refresh_record(user)
+      {:error, :pinned_statuses_limit_reached} = CommonAPI.pin(activity2.id, user)
+
+      assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity2.id)
     end
   end
 

test/pleroma/web/mastodon_api/controllers/status_controller_test.exs

@@ -1209,20 +1209,27 @@ test "returns 404 error for a wrong id", %{conn: conn} do
     setup do: clear_config([:instance, :max_pinned_statuses], 1)
 
     test "pin status", %{conn: conn, user: user, activity: activity} do
-      id_str = to_string(activity.id)
+      id = activity.id
 
-      assert %{"id" => ^id_str, "pinned" => true} =
+      assert %{"id" => ^id, "pinned" => true} =
                conn
                |> put_req_header("content-type", "application/json")
                |> post("/api/v1/statuses/#{activity.id}/pin")
                |> json_response_and_validate_schema(200)
 
-      assert [%{"id" => ^id_str, "pinned" => true}] =
+      assert [%{"id" => ^id, "pinned" => true}] =
                conn
                |> get("/api/v1/accounts/#{user.id}/statuses?pinned=true")
                |> json_response_and_validate_schema(200)
     end
 
+    test "non authenticated user", %{activity: activity} do
+      assert build_conn()
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/v1/statuses/#{activity.id}/pin")
+             |> json_response(403) == %{"error" => "Invalid credentials."}
+    end
+
     test "/pin: returns 400 error when activity is not public", %{conn: conn, user: user} do
       {:ok, dm} = CommonAPI.post(user, %{status: "test", visibility: "direct"})
 
@@ -1231,7 +1238,18 @@ test "/pin: returns 400 error when activity is not public", %{conn: conn, user:
         |> put_req_header("content-type", "application/json")
         |> post("/api/v1/statuses/#{dm.id}/pin")
 
-      assert json_response_and_validate_schema(conn, 400) == %{"error" => "Could not pin"}
+      assert json_response_and_validate_schema(conn, 422) == %{
+               "error" => "Non-public status cannot be pinned"
+             }
+    end
+
+    test "pin by another user", %{activity: activity} do
+      %{conn: conn} = oauth_access(["write:accounts"])
+
+      assert conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/v1/statuses/#{activity.id}/pin")
+             |> json_response(422) == %{"error" => "Someone else's status cannot be pinned"}
     end
 
     test "unpin status", %{conn: conn, user: user, activity: activity} do
@@ -1252,13 +1270,11 @@ test "unpin status", %{conn: conn, user: user, activity: activity} do
                |> json_response_and_validate_schema(200)
     end
 
-    test "/unpin: returns 400 error when activity is not exist", %{conn: conn} do
+    test "/unpin: returns 404 error when activity doesn't exist", %{conn: conn} do
-      conn =
+      assert conn
-        conn
+             |> put_req_header("content-type", "application/json")
-        |> put_req_header("content-type", "application/json")
+             |> post("/api/v1/statuses/1/unpin")
-        |> post("/api/v1/statuses/1/unpin")
+             |> json_response_and_validate_schema(404) == %{"error" => "Record not found"}
-
-      assert json_response_and_validate_schema(conn, 400) == %{"error" => "Could not unpin"}
     end
 
     test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -286,7 +286,8 @@ test "a note activity" do
         direct_conversation_id: nil,
         thread_muted: false,
         emoji_reactions: [],
-        parent_visible: false
+        parent_visible: false,
+        pinned_at: nil
       }
     }
 

test/pleroma/web/o_auth/o_auth_controller_test.exs

@@ -805,10 +805,12 @@ test "issues a token for `password` grant_type with valid credentials, with full
           "client_secret" => app.client_secret
         })
 
-      assert %{"access_token" => token} = json_response(conn, 200)
+      assert %{"id" => id, "access_token" => access_token} = json_response(conn, 200)
 
-      token = Repo.get_by(Token, token: token)
+      token = Repo.get_by(Token, token: access_token)
       assert token
+      assert token.id == id
+      assert token.token == access_token
       assert token.scopes == app.scopes
     end
 

test/pleroma/web/twitter_api/remote_follow_controller_test.exs

@@ -27,6 +27,16 @@ test "adds status to pleroma instance if the `acct` is a status", %{conn: conn}
             body: File.read!("test/fixtures/tesla_mock/status.emelie.json")
           }
 
+        %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} ->
+          %Tesla.Env{
+            status: 200,
+            headers: [{"content-type", "application/activity+json"}],
+            body:
+              File.read!("test/fixtures/users_mock/masto_featured.json")
+              |> String.replace("{{domain}}", "mastodon.social")
+              |> String.replace("{{nickname}}", "emelie")
+          }
+
         %{method: :get, url: "https://mastodon.social/users/emelie"} ->
           %Tesla.Env{
             status: 200,
@@ -52,6 +62,16 @@ test "show follow account page if the `acct` is a account link", %{conn: conn} d
             headers: [{"content-type", "application/activity+json"}],
             body: File.read!("test/fixtures/tesla_mock/emelie.json")
           }
+
+        %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} ->
+          %Tesla.Env{
+            status: 200,
+            headers: [{"content-type", "application/activity+json"}],
+            body:
+              File.read!("test/fixtures/users_mock/masto_featured.json")
+              |> String.replace("{{domain}}", "mastodon.social")
+              |> String.replace("{{nickname}}", "emelie")
+          }
       end)
 
       response =
@@ -70,6 +90,16 @@ test "show follow page if the `acct` is a account link", %{conn: conn} do
             headers: [{"content-type", "application/activity+json"}],
             body: File.read!("test/fixtures/tesla_mock/emelie.json")
           }
+
+        %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} ->
+          %Tesla.Env{
+            status: 200,
+            headers: [{"content-type", "application/activity+json"}],
+            body:
+              File.read!("test/fixtures/users_mock/masto_featured.json")
+              |> String.replace("{{domain}}", "mastodon.social")
+              |> String.replace("{{nickname}}", "emelie")
+          }
       end)
 
       user = insert(:user)

test/support/factory.ex

@@ -4,6 +4,9 @@
 
 defmodule Pleroma.Factory do
   use ExMachina.Ecto, repo: Pleroma.Repo
+
+  require Pleroma.Constants
+
   alias Pleroma.Object
   alias Pleroma.User
 
@@ -41,23 +44,27 @@ def user_factory(attrs \\ %{}) do
 
     urls =
       if attrs[:local] == false do
-        base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"])
+        base_domain = attrs[:domain] || Enum.random(["domain1.com", "domain2.com", "domain3.com"])
 
         ap_id = "https://#{base_domain}/users/#{user.nickname}"
 
         %{
           ap_id: ap_id,
           follower_address: ap_id <> "/followers",
-          following_address: ap_id <> "/following"
+          following_address: ap_id <> "/following",
+          featured_address: ap_id <> "/collections/featured"
         }
       else
         %{
           ap_id: User.ap_id(user),
           follower_address: User.ap_followers(user),
-          following_address: User.ap_following(user)
+          following_address: User.ap_following(user),
+          featured_address: User.ap_featured_collection(user)
         }
       end
 
+    attrs = Map.delete(attrs, :domain)
+
     user
     |> Map.put(:raw_bio, user.bio)
     |> Map.merge(urls)
@@ -221,6 +228,45 @@ def direct_note_activity_factory do
     }
   end
 
+  def add_activity_factory(attrs \\ %{}) do
+    featured_collection_activity(attrs, "Add")
+  end
+
+  def remove_activity_factor(attrs \\ %{}) do
+    featured_collection_activity(attrs, "Remove")
+  end
+
+  defp featured_collection_activity(attrs, type) do
+    user = attrs[:user] || insert(:user)
+    note = attrs[:note] || insert(:note, user: user)
+
+    data_attrs =
+      attrs
+      |> Map.get(:data_attrs, %{})
+      |> Map.put(:type, type)
+
+    attrs = Map.drop(attrs, [:user, :note, :data_attrs])
+
+    data =
+      %{
+        "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(),
+        "target" => user.featured_address,
+        "object" => note.data["object"],
+        "actor" => note.data["actor"],
+        "type" => "Add",
+        "to" => [Pleroma.Constants.as_public()],
+        "cc" => [user.follower_address]
+      }
+      |> Map.merge(data_attrs)
+
+    %Pleroma.Activity{
+      data: data,
+      actor: data["actor"],
+      recipients: data["to"]
+    }
+    |> Map.merge(attrs)
+  end
+
   def note_activity_factory(attrs \\ %{}) do
     user = attrs[:user] || insert(:user)
     note = attrs[:note] || insert(:note, user: user)

test/support/http_request_mock.ex

@@ -89,6 +89,18 @@ def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do
      }}
   end
 
+  def get("https://mastodon.sdf.org/users/rinpatch/collections/featured", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         File.read!("test/fixtures/users_mock/masto_featured.json")
+         |> String.replace("{{domain}}", "mastodon.sdf.org")
+         |> String.replace("{{nickname}}", "rinpatch"),
+       headers: [{"content-type", "application/activity+json"}]
+     }}
+  end
+
   def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do
     {:ok,
      %Tesla.Env{
@@ -905,6 +917,18 @@ def get("https://mastodon.social/users/lambadalambda", _, _, _) do
      }}
   end
 
+  def get("https://mastodon.social/users/lambadalambda/collections/featured", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body:
+         File.read!("test/fixtures/users_mock/masto_featured.json")
+         |> String.replace("{{domain}}", "mastodon.social")
+         |> String.replace("{{nickname}}", "lambadalambda"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
   def get("https://apfed.club/channel/indio", _, _, _) do
     {:ok,
      %Tesla.Env{