Merge develop

.gitignore

@@ -10,6 +10,7 @@
 /test/tmp/
 /doc
 /instance
+/priv/ssh_keys
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*

.gitlab-ci.yml

@@ -48,6 +48,7 @@ unit-testing:
   - name: postgres:9.6.2
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
+    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix test --trace --preload-modules

CHANGELOG.md

@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/)
 - ActivityPub C2S: OAuth endpoints
 - Metadata RelMe provider
+- OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 
 ### Changed
@@ -61,10 +62,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Added support max_id & since_id for bookmark timeline endpoints.
 
 ### Fixed
+- Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
 - Followers counter not being updated when a follower is blocked
 - Deactivated users being able to request an access token
 - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
 - proper Twitter Card generation instead of a dummy
+- Deletions failing for users with a large number of posts
 - NodeInfo: Include admins in `staffAccounts`
 - ActivityPub: Crashing when requesting empty local user's outbox
 - Federation: Handling of objects without `summary` property

config/config.exs

@@ -416,7 +416,8 @@
   web_push: 50,
   mailer: 10,
   transmogrifier: 20,
-  scheduled_activities: 10
+  scheduled_activities: 10,
+  background: 5
 
 config :pleroma, :fetch_initial_posts,
   enabled: false,
@@ -443,6 +444,9 @@
   base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
   uid: System.get_env("LDAP_UID") || "cn"
 
+config :esshd,
+  enabled: false
+
 oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "")
 
 ueberauth_providers =
@@ -476,6 +480,10 @@
     inactivity_threshold: 7
   }
 
+config :pleroma, :oauth2,
+  token_expires_in: 600,
+  issue_new_refresh_token: true
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

docs/api/differences_in_mastoapi_responses.md

@@ -80,3 +80,10 @@ Additional parameters can be added to the JSON body/Form data:
 - `hide_favorites` - if true, user's favorites timeline will be hidden
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
 - `default_scope` - the scope returned under `privacy` key in Source subentity
+
+## Authentication
+
+*Pleroma supports refreshing tokens.
+
+`POST /oauth/token`
+Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.

docs/config.md

@@ -37,7 +37,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 An example for Sendgrid adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,
   api_key: "YOUR_API_KEY"
@@ -45,7 +45,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 
 An example for SMTP adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.SMTP,
   relay: "smtp.gmail.com",
@@ -109,7 +109,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
 
 An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
-```
+```elixir
 config :logger,
   backends: [{ExSyslogger, :ex_syslogger}]
 
@@ -118,7 +118,7 @@ config :logger, :ex_syslogger,
 ```
 
 Another example, keeping console output and adding the pid to syslog output:
-```
+```elixir
 config :logger,
   backends: [:console, {ExSyslogger, :ex_syslogger}]
 
@@ -130,7 +130,7 @@ config :logger, :ex_syslogger,
 See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 An example of logging info to local syslog, but warn to a Slack channel:
-```
+```elixir
 config :logger,
   backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ],
   level: :info
@@ -156,14 +156,30 @@ Frontends can access these settings at `/api/pleroma/frontend_configurations`
 
 To add your own configuration for PleromaFE, use it like this:
 
-`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}`
+```elixir
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "pleroma-dark",
+    # ... see /priv/static/static/config.json for the available keys.
+},
+  masto_fe: %{
+    showInstanceSpecificPanel: true
+  }
+```
 
-These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys.
+These settings **need to be complete**, they will override the defaults.
+
+NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below.
 
 ## :fe
 __THIS IS DEPRECATED__
 
-If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`.
+If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
+Please **set this option to false** in your config like this:
+
+```elixir
+config :pleroma, :fe, false
+```
 
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
@@ -274,7 +290,7 @@ their ActivityPub ID.
 
 An example:
 
-```exs
+```elixir
 config :pleroma, :mrf_user_allowlist,
   "example.org": ["https://example.org/users/admin"]
 ```
@@ -303,7 +319,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end
 
 Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example:
 
-```exs
+```elixir
 config :pleroma, :admin_token, "somerandomtoken"
 ```
 
@@ -387,7 +403,7 @@ Configuration for the `auto_linker` library:
 
 Example:
 
-```exs
+```elixir
 config :auto_linker,
   opts: [
     scheme: true,
@@ -428,8 +444,29 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+## BBS / SSH access
+
+To enable simple command line interface accessible over ssh, add a setting like this to your configuration file:
+
+```exs
+app_dir = File.cwd!
+priv_dir = Path.join([app_dir, "priv/ssh_keys"])
+
+config :esshd,
+  enabled: true,
+  priv_dir: priv_dir,
+  handler: "Pleroma.BBS.Handler",
+  port: 10_022,
+  password_authenticator: "Pleroma.BBS.Authenticator"
+```
+
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+
 ## :auth
 
+* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator
+* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication
+
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
@@ -448,7 +485,7 @@ Email notifications settings.
     - interval: Minimum interval between digest emails to one user
     - inactivity_threshold: Minimum user inactivity threshold
 
-# OAuth consumer mode
+## OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
 Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies).
@@ -472,7 +509,7 @@ Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you ha
 Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`,
 per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables:
 
-```
+```elixir
 # Twitter
 config :ueberauth, Ueberauth.Strategy.Twitter.OAuth,
   consumer_key: System.get_env("TWITTER_CONSUMER_KEY"),
@@ -501,6 +538,13 @@ config :ueberauth, Ueberauth,
   ]
 ```
 
+## OAuth 2.0 provider - :oauth2
+
+Configure OAuth 2 provider capabilities:
+
+* `token_expires_in` - The lifetime in seconds of the access token.
+* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`

installation/download-mastofe-build.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+project_id="74"
+project_branch="rebase/glitch-soc"
+static_dir="instance/static"
+# For bundling:
+# project_branch="pleroma"
+# static_dir="priv/static"
+
+if [[ ! -d "${static_dir}" ]]
+then
+	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
+	exit 1
+fi
+
+last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+
+echo "branch:${project_branch}"
+echo "Last-Modified:${last_modified}"
+
+artifact="mastofe.zip"
+
+if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+then
+	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	then
+		echo "MastoFE is up-to-date, exiting…"
+		exit 0
+	fi
+fi
+
+curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+
+# TODO: Update the emoji as well
+rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit
+unzip -q "${artifact}" || exit
+
+cp public/assets/sw.js "${static_dir}/sw.js" || exit
+cp -r public/packs "${static_dir}/packs" || exit
+
+echo "${last_modified}" > mastofe.timestamp
+rm -fr public
+rm -i "${artifact}"

lib/mix/tasks/benchmark.ex

@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.Benchmark do
+  use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
+
+  def run(["search"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "search" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+      end
+    })
+  end
+
+  def run(["tag"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "tag" => fn ->
+        %{"type" => "Create", "tag" => "cofe"}
+        |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
+      end
+    })
+  end
+end

lib/mix/tasks/pleroma/user.ex

@@ -126,7 +126,7 @@ def run(["new", nickname, email | rest]) do
 
     proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
-    unless not proceed? do
+    if proceed? do
       Common.start_pleroma()
 
       params = %{
@@ -163,7 +163,7 @@ def run(["rm", nickname]) do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete(user)
+      User.perform(:delete, user)
       Mix.shell().info("User #{nickname} deleted.")
     else
       _ ->
@@ -380,7 +380,7 @@ def run(["delete_activities", nickname]) do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete_user_activities(user)
+      {:ok, _} = User.delete_user_activities(user)
       Mix.shell().info("User #{nickname} statuses deleted.")
     else
       _ ->

lib/pleroma/activity.ex

@@ -6,14 +6,18 @@ defmodule Pleroma.Activity do
   use Ecto.Schema
 
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.User
 
   import Ecto.Changeset
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @type actor :: String.t()
+
   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@@ -33,6 +37,8 @@ defmodule Pleroma.Activity do
     field(:local, :boolean, default: true)
     field(:actor, :string)
     field(:recipients, {:array, :string}, default: [])
+    # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
+    has_one(:bookmark, Bookmark)
     has_many(:notifications, Notification, on_delete: :delete_all)
 
     # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
@@ -71,6 +77,16 @@ def with_preloaded_object(query) do
     |> preload([activity, object], object: object)
   end
 
+  def with_preloaded_bookmark(query, %User{} = user) do
+    from([a] in query,
+      left_join: b in Bookmark,
+      on: b.user_id == ^user.id and b.activity_id == a.id,
+      preload: [bookmark: b]
+    )
+  end
+
+  def with_preloaded_bookmark(query, _), do: query
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
@@ -80,6 +96,16 @@ def get_by_ap_id(ap_id) do
     )
   end
 
+  def get_bookmark(%Activity{} = activity, %User{} = user) do
+    if Ecto.assoc_loaded?(activity.bookmark) do
+      activity.bookmark
+    else
+      Bookmark.get(user.id, activity.id)
+    end
+  end
+
+  def get_bookmark(_, _), do: nil
+
   def change(struct, params \\ %{}) do
     struct
     |> cast(params, [:data])
@@ -260,4 +286,9 @@ def all_by_actor_and_id(actor, status_ids) do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  @spec query_by_actor(actor()) :: Ecto.Query.t()
+  def query_by_actor(actor) do
+    from(a in Activity, where: a.actor == ^actor)
+  end
 end

lib/pleroma/bbs/authenticator.ex

@@ -0,0 +1,16 @@
+defmodule Pleroma.BBS.Authenticator do
+  use Sshd.PasswordAuthenticator
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+
+  def authenticate(username, password) do
+    username = to_string(username)
+    password = to_string(password)
+
+    with %User{} = user <- User.get_by_nickname(username) do
+      Pbkdf2.checkpw(password, user.password_hash)
+    else
+      _e -> false
+    end
+  end
+end

lib/pleroma/bbs/handler.ex

@@ -0,0 +1,147 @@
+defmodule Pleroma.BBS.Handler do
+  use Sshd.ShellHandler
+  alias Pleroma.Activity
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
+
+  def on_shell(username, _pubkey, _ip, _port) do
+    :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
+    user = Pleroma.User.get_cached_by_nickname(to_string(username))
+    Logger.debug("#{inspect(user)}")
+    loop(run_state(user: user))
+  end
+
+  def on_connect(username, ip, port, method) do
+    Logger.debug(fn ->
+      """
+      Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{
+        inspect(port)
+      } using #{inspect(method)}
+      """
+    end)
+  end
+
+  def on_disconnect(username, ip, port) do
+    Logger.debug(fn ->
+      "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}"
+    end)
+  end
+
+  defp loop(state) do
+    self_pid = self()
+    counter = state.counter
+    prefix = state.prefix
+    user = state.user
+
+    input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end)
+    wait_input(state, input)
+  end
+
+  def puts_activity(activity) do
+    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
+    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts("")
+  end
+
+  def handle_command(state, "help") do
+    IO.puts("Available commands:")
+    IO.puts("help - This help")
+    IO.puts("home - Show the home timeline")
+    IO.puts("p <text> - Post the given text")
+    IO.puts("r <id> <text> - Reply to the post with the given id")
+    IO.puts("quit - Quit")
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "r " <> text) do
+    text = String.trim(text)
+    [activity_id, rest] = String.split(text, " ", parts: 2)
+
+    with %Activity{} <- Activity.get_by_id(activity_id),
+         {:ok, _activity} <-
+           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+      IO.puts("Replied!")
+    else
+      _e -> IO.puts("Could not reply...")
+    end
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "p " <> text) do
+    text = String.trim(text)
+
+    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+      IO.puts("Posted!")
+    else
+      _e -> IO.puts("Could not post...")
+    end
+
+    state
+  end
+
+  def handle_command(state, "home") do
+    user = state.user
+
+    params =
+      %{}
+      |> Map.put("type", ["Create"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("muting_user", user)
+      |> Map.put("user", user)
+
+    activities =
+      [user.ap_id | user.following]
+      |> ActivityPub.fetch_activities(params)
+      |> ActivityPub.contain_timeline(user)
+
+    Enum.each(activities, fn activity ->
+      puts_activity(activity)
+    end)
+
+    state
+  end
+
+  def handle_command(state, command) do
+    IO.puts("Unknown command '#{command}'")
+    state
+  end
+
+  defp wait_input(state, input) do
+    receive do
+      {:input, ^input, "quit\n"} ->
+        IO.puts("Exiting...")
+
+      {:input, ^input, code} when is_binary(code) ->
+        code = String.trim(code)
+
+        state = handle_command(state, code)
+
+        loop(%{state | counter: state.counter + 1})
+
+      {:error, :interrupted} ->
+        IO.puts("Caught Ctrl+C...")
+        loop(%{state | counter: state.counter + 1})
+
+      {:input, ^input, msg} ->
+        :ok = Logger.warn("received unknown message: #{inspect(msg)}")
+        loop(%{state | counter: state.counter + 1})
+    end
+  end
+
+  defp run_state(opts) do
+    %{prefix: "pleroma", counter: 1, user: opts[:user]}
+  end
+
+  defp io_get(pid, prefix, counter, username) do
+    prompt = prompt(prefix, counter, username)
+    send(pid, {:input, self(), IO.gets(:stdio, prompt)})
+  end
+
+  defp prompt(prefix, counter, username) do
+    prompt = "#{username}@#{prefix}:#{counter}>"
+    prompt <> " "
+  end
+end

lib/pleroma/conversation.ex

@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Conversation do
+  alias Pleroma.Conversation.Participation
+  alias Pleroma.Repo
+  alias Pleroma.User
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "conversations" do
+    # This is the context ap id.
+    field(:ap_id, :string)
+    has_many(:participations, Participation)
+    has_many(:users, through: [:participations, :user])
+
+    timestamps()
+  end
+
+  def creation_cng(struct, params) do
+    struct
+    |> cast(params, [:ap_id])
+    |> validate_required([:ap_id])
+    |> unique_constraint(:ap_id)
+  end
+
+  def create_for_ap_id(ap_id) do
+    %__MODULE__{}
+    |> creation_cng(%{ap_id: ap_id})
+    |> Repo.insert(
+      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
+      returning: true,
+      conflict_target: :ap_id
+    )
+  end
+
+  def get_for_ap_id(ap_id) do
+    Repo.get_by(__MODULE__, ap_id: ap_id)
+  end
+
+  @doc """
+  This will
+  1. Create a conversation if there isn't one already
+  2. Create a participation for all the people involved who don't have one already
+  3. Bump all relevant participations to 'unread'
+  """
+  def create_or_bump_for(activity) do
+    with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity),
+         object <- Pleroma.Object.normalize(activity),
+         "Create" <- activity.data["type"],
+         "Note" <- object.data["type"],
+         ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
+      {:ok, conversation} = create_for_ap_id(ap_id)
+
+      users = User.get_users_from_set(activity.recipients, false)
+
+      participations =
+        Enum.map(users, fn user ->
+          {:ok, participation} =
+            Participation.create_for_user_and_conversation(user, conversation)
+
+          participation
+        end)
+
+      {:ok,
+       %{
+         conversation
+         | participations: participations
+       }}
+    else
+      e -> {:error, e}
+    end
+  end
+end

lib/pleroma/conversation/participation.ex

@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Conversation.Participation do
+  use Ecto.Schema
+  alias Pleroma.Conversation
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "conversation_participations" do
+    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:conversation, Conversation)
+    field(:read, :boolean, default: false)
+    field(:last_activity_id, Pleroma.FlakeId, virtual: true)
+
+    timestamps()
+  end
+
+  def creation_cng(struct, params) do
+    struct
+    |> cast(params, [:user_id, :conversation_id])
+    |> validate_required([:user_id, :conversation_id])
+  end
+
+  def create_for_user_and_conversation(user, conversation) do
+    %__MODULE__{}
+    |> creation_cng(%{user_id: user.id, conversation_id: conversation.id})
+    |> Repo.insert(
+      on_conflict: [set: [read: false, updated_at: NaiveDateTime.utc_now()]],
+      returning: true,
+      conflict_target: [:user_id, :conversation_id]
+    )
+  end
+
+  def read_cng(struct, params) do
+    struct
+    |> cast(params, [:read])
+    |> validate_required([:read])
+  end
+
+  def mark_as_read(participation) do
+    participation
+    |> read_cng(%{read: true})
+    |> Repo.update()
+  end
+
+  def mark_as_unread(participation) do
+    participation
+    |> read_cng(%{read: false})
+    |> Repo.update()
+  end
+
+  def for_user(user, params \\ %{}) do
+    from(p in __MODULE__,
+      where: p.user_id == ^user.id,
+      order_by: [desc: p.updated_at]
+    )
+    |> Pleroma.Pagination.fetch_paginated(params)
+    |> Repo.preload(conversation: [:users])
+  end
+
+  def for_user_with_last_activity_id(user, params \\ %{}) do
+    for_user(user, params)
+    |> Enum.map(fn participation ->
+      activity_id =
+        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
+          "user" => user,
+          "blocking_user" => user
+        })
+
+      %{
+        participation
+        | last_activity_id: activity_id
+      }
+    end)
+  end
+end

lib/pleroma/formatter.ex

@@ -113,9 +113,7 @@ def emojify(text, emoji, strip \\ false) do
 
       html =
         if not strip do
-          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+          "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
-            MediaProxy.url(file)
-          }' />"
         else
           ""
         end
@@ -130,12 +128,23 @@ def demojify(text) do
 
   def demojify(text, nil), do: text
 
+  @doc "Outputs a list of the emoji-shortcodes in a text"
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []
 
+  @doc "Outputs a list of the emoji-Maps in a text"
+  def get_emoji_map(text) when is_binary(text) do
+    get_emoji(text)
+    |> Enum.reduce(%{}, fn {name, file, _group}, acc ->
+      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+    end)
+  end
+
+  def get_emoji_map(_), do: []
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end

lib/pleroma/html.ex

@@ -28,12 +28,18 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+  def get_cached_scrubbed_html_for_activity(
+        content,
+        scrubbers,
+        activity,
+        key \\ "",
+        callback \\ fn x -> x end
+      ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
-      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
+      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end
 
@@ -42,24 +48,27 @@ def get_cached_stripped_html_for_activity(content, activity, key) do
       content,
       HtmlSanitizeEx.Scrubber.StripTags,
       activity,
-      key
+      key,
+      &HtmlEntities.decode/1
     )
   end
 
   def ensure_scrubbed_html(
         content,
         scrubbers,
-        false = _fake
+        fake,
+        callback
       ) do
-    {:commit, filter_tags(content, scrubbers)}
+    content =
-  end
+      content
+      |> filter_tags(scrubbers)
+      |> callback.()
 
-  def ensure_scrubbed_html(
+    if fake do
-        content,
+      {:ignore, content}
-        scrubbers,
+    else
-        true = _fake
+      {:commit, content}
-      ) do
+    end
-    {:ignore, filter_tags(content, scrubbers)}
   end
 
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
@@ -142,6 +151,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])
@@ -212,6 +222,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])

lib/pleroma/object/containment.ex

@@ -1,7 +1,5 @@
 defmodule Pleroma.Object.Containment do
   @moduledoc """
-  # Object Containment
-
   This module contains some useful functions for containing objects to specific
   origins and determining those origins.  They previously lived in the
   ActivityPub `Transmogrifier` module.

lib/pleroma/plugs/http_security_plug.ex

@@ -35,7 +35,7 @@ defp headers do
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
-    websocket_url = String.replace(static_url, "http", "ws")
+    websocket_url = Pleroma.Web.Endpoint.websocket_url()
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 

lib/pleroma/plugs/oauth_plug.ex

@@ -16,6 +16,16 @@ def init(options), do: options
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
+  def call(%{params: %{"access_token" => access_token}} = conn, _) do
+    with {:ok, user, token_record} <- fetch_user_and_token(access_token) do
+      conn
+      |> assign(:token, token_record)
+      |> assign(:user, user)
+    else
+      _ -> conn
+    end
+  end
+
   def call(conn, _) do
     with {:ok, token_str} <- fetch_token_str(conn),
          {:ok, user, token_record} <- fetch_user_and_token(token_str) do

lib/pleroma/repo.ex

@@ -19,4 +19,32 @@ defmodule Instrumenter do
   def init(_, opts) do
     {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
   end
+
+  @doc "find resource based on prepared query"
+  @spec find_resource(Ecto.Query.t()) :: {:ok, struct()} | {:error, :not_found}
+  def find_resource(%Ecto.Query{} = query) do
+    case __MODULE__.one(query) do
+      nil -> {:error, :not_found}
+      resource -> {:ok, resource}
+    end
+  end
+
+  def find_resource(_query), do: {:error, :not_found}
+
+  @doc """
+  Gets association from cache or loads if need
+
+  ## Examples
+
+    iex> Repo.get_assoc(token, :user)
+    %User{}
+
+  """
+  @spec get_assoc(struct(), atom()) :: {:ok, struct()} | {:error, :not_found}
+  def get_assoc(resource, association) do
+    case __MODULE__.preload(resource, association) do
+      %{^association => assoc} when not is_nil(assoc) -> {:ok, assoc}
+      _ -> {:error, :not_found}
+    end
+  end
 end

lib/pleroma/upload.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Upload do
   @moduledoc """
-  # Upload
+  Manage user uploads
 
   Options:
   * `:type`: presets for activity type (defaults to Document) and size limits from app configuration

lib/pleroma/user.ex

@@ -10,8 +10,6 @@ defmodule Pleroma.User do
 
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
-  alias Pleroma.Bookmark
-  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -56,7 +54,6 @@ defmodule Pleroma.User do
     field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
     field(:last_digest_emailed_at, :naive_datetime)
-    has_many(:bookmarks, Bookmark)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
     embeds_one(:info, Pleroma.User.Info)
@@ -424,7 +421,7 @@ def follow_import(%User{} = follower, followed_identifiers)
     Enum.map(
       followed_identifiers,
       fn followed_identifier ->
-        with %User{} = followed <- get_or_fetch(followed_identifier),
+        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
              {:ok, follower} <- maybe_direct_follow(follower, followed),
              {:ok, _} <- ActivityPub.follow(follower, followed) do
           followed
@@ -508,7 +505,15 @@ def get_cached_by_id(id) do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
+
+    Cachex.fetch!(:user_cache, key, fn ->
+      user_result = get_or_fetch_by_nickname(nickname)
+
+      case user_result do
+        {:ok, user} -> {:commit, user}
+        {:error, _error} -> {:ignore, nil}
+      end
+    end)
   end
 
   def get_cached_by_nickname_or_id(nickname_or_id) do
@@ -544,7 +549,7 @@ def fetch_by_nickname(nickname) do
 
   def get_or_fetch_by_nickname(nickname) do
     with %User{} = user <- get_by_nickname(nickname) do
-      user
+      {:ok, user}
     else
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
@@ -554,9 +559,9 @@ def get_or_fetch_by_nickname(nickname) do
             {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
           end
 
-          user
+          {:ok, user}
         else
-          _e -> nil
+          _e -> {:error, "not found " <> nickname}
         end
     end
   end
@@ -903,7 +908,7 @@ def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_i
     Enum.map(
       blocked_identifiers,
       fn blocked_identifier ->
-        with %User{} = blocked <- get_or_fetch(blocked_identifier),
+        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
              {:ok, blocker} <- block(blocker, blocked),
              {:ok, _} <- ActivityPub.block(blocker, blocked) do
           blocked
@@ -1158,7 +1163,12 @@ def update_notification_settings(%User{} = user, settings \\ %{}) do
     |> update_and_set_cache()
   end
 
-  def delete(%User{} = user) do
+  @spec delete(User.t()) :: :ok
+  def delete(%User{} = user),
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user])
+
+  @spec perform(atom(), User.t()) :: {:ok, User.t()}
+  def perform(:delete, %User{} = user) do
     {:ok, user} = User.deactivate(user)
 
     # Remove all relationships
@@ -1174,22 +1184,23 @@ def delete(%User{} = user) do
   end
 
   def delete_user_activities(%User{ap_id: ap_id} = user) do
-    Activity
+    stream =
-    |> where(actor: ^ap_id)
+      ap_id
-    |> Activity.with_preloaded_object()
+      |> Activity.query_by_actor()
-    |> Repo.all()
+      |> Activity.with_preloaded_object()
-    |> Enum.each(fn
+      |> Repo.stream()
-      %{data: %{"type" => "Create"}} = activity ->
-        activity |> Object.normalize() |> ActivityPub.delete()
 
-      # TODO: Do something with likes, follows, repeats.
+    Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
-      _ ->
-        "Doing nothing"
-    end)
 
     {:ok, user}
   end
 
+  defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
+    Object.normalize(activity) |> ActivityPub.delete()
+  end
+
+  defp delete_activity(_activity), do: "Doing nothing"
+
   def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
@@ -1203,11 +1214,11 @@ def fetch_by_ap_id(ap_id) do
 
     case ap_try do
       {:ok, user} ->
-        user
+        {:ok, user}
 
       _ ->
         case OStatus.make_user(ap_id) do
-          {:ok, user} -> user
+          {:ok, user} -> {:ok, user}
           _ -> {:error, "Could not fetch by AP id"}
         end
     end
@@ -1217,20 +1228,20 @@ def get_or_fetch_by_ap_id(ap_id) do
     user = get_cached_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
-      user
+      {:ok, user}
     else
       # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
       should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
 
-      user = fetch_by_ap_id(ap_id)
+      resp = fetch_by_ap_id(ap_id)
 
       if should_fetch_initial do
-        with %User{} = user do
+        with {:ok, %User{} = user} = resp do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end
       end
 
-      user
+      resp
     end
   end
 
@@ -1272,7 +1283,7 @@ def public_key_from_info(%{magic_key: magic_key}) do
   end
 
   def get_public_key_for_ap_id(ap_id) do
-    with %User{} = user <- get_or_fetch_by_ap_id(ap_id),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key_from_info(user.info) do
       {:ok, public_key}
     else
@@ -1324,18 +1335,15 @@ def wait_and_refresh(timeout, %User{} = a, %User{} = b) do
     end
   end
 
-  def parse_bio(bio, user \\ %User{info: %{source_data: %{}}})
+  def parse_bio(bio) when is_binary(bio) and bio != "" do
-  def parse_bio(nil, _user), do: ""
+    bio
-  def parse_bio(bio, _user) when bio == "", do: bio
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
+  end
 
-  def parse_bio(bio, user) do
+  def parse_bio(_), do: ""
-    emoji =
-      (user.info.source_data["tag"] || [])
-      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
-        {String.trim(name, ":"), url}
-      end)
 
+  def parse_bio(bio, user) when is_binary(bio) and bio != "" do
     # TODO: get profile URLs other than user.ap_id
     profile_urls = [user.ap_id]
 
@@ -1345,9 +1353,10 @@ def parse_bio(bio, user) do
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
     )
     |> elem(0)
-    |> Formatter.emojify(emoji)
   end
 
+  def parse_bio(_, _), do: ""
+
   def tag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: tag(user_identifier, tags)

lib/pleroma/user/info.ex

@@ -44,6 +44,7 @@ defmodule Pleroma.User.Info do
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
     field(:email_notifications, :map, default: %{"digest" => false})
+    field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
       default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true}

lib/pleroma/user_invite_token.ex

@@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do
     timestamps()
   end
 
-  @spec create_invite(map()) :: UserInviteToken.t()
+  @spec create_invite(map()) :: {:ok, UserInviteToken.t()}
   def create_invite(params \\ %{}) do
     %UserInviteToken{}
     |> cast(params, [:max_use, :expires_at])

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
+  alias Pleroma.Conversation
   alias Pleroma.Instances
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -141,7 +142,14 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do
       end)
 
       Notification.create_notifications(activity)
+
+      participations =
+        activity
+        |> Conversation.create_or_bump_for()
+        |> get_participations()
+
       stream_out(activity)
+      stream_out_participations(participations)
       {:ok, activity}
     else
       %Activity{} = activity ->
@@ -164,11 +172,23 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do
     end
   end
 
+  defp get_participations({:ok, %{participations: participations}}), do: participations
+  defp get_participations(_), do: []
+
+  def stream_out_participations(participations) do
+    participations =
+      participations
+      |> Repo.preload(:user)
+
+    Enum.each(participations, fn participation ->
+      Pleroma.Web.Streamer.stream("participation", participation)
+    end)
+  end
+
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce", "Delete"] do
-      object = Object.normalize(activity)
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -180,6 +200,8 @@ def stream_out(activity) do
         end
 
         if activity.data["type"] in ["Create"] do
+          object = Object.normalize(activity)
+
           object.data
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)
@@ -194,6 +216,7 @@ def stream_out(activity) do
           end
         end
       else
+        # TODO: Write test, replace with visibility test
         if !Enum.member?(activity.data["cc"] || [], public) &&
              !Enum.member?(
                activity.data["to"],
@@ -456,35 +479,44 @@ def flag(
     end
   end
 
-  def fetch_activities_for_context(context, opts \\ %{}) do
+  defp fetch_activities_for_context_query(context, opts) do
     public = ["https://www.w3.org/ns/activitystreams#Public"]
 
     recipients =
       if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public
 
-    query = from(activity in Activity)
+    from(activity in Activity)
-
+    |> restrict_blocked(opts)
-    query =
+    |> restrict_recipients(recipients, opts["user"])
-      query
+    |> where(
-      |> restrict_blocked(opts)
+      [activity],
-      |> restrict_recipients(recipients, opts["user"])
+      fragment(
-
+        "?->>'type' = ? and ?->>'context' = ?",
-    query =
+        activity.data,
-      from(
+        "Create",
-        activity in query,
+        activity.data,
-        where:
+        ^context
-          fragment(
-            "?->>'type' = ? and ?->>'context' = ?",
-            activity.data,
-            "Create",
-            activity.data,
-            ^context
-          ),
-        order_by: [desc: :id]
       )
-      |> Activity.with_preloaded_object()
+    )
+    |> order_by([activity], desc: activity.id)
+  end
 
-    Repo.all(query)
+  @spec fetch_activities_for_context(String.t(), keyword() | map()) :: [Activity.t()]
+  def fetch_activities_for_context(context, opts \\ %{}) do
+    context
+    |> fetch_activities_for_context_query(opts)
+    |> Activity.with_preloaded_object()
+    |> Repo.all()
+  end
+
+  @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) ::
+          Pleroma.FlakeId.t() | nil
+  def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
+    context
+    |> fetch_activities_for_context_query(opts)
+    |> limit(1)
+    |> select([a], a.id)
+    |> Repo.one()
   end
 
   def fetch_public_activities(opts \\ %{}) do
@@ -783,11 +815,32 @@ defp maybe_preload_objects(query, _) do
     |> Activity.with_preloaded_object()
   end
 
+  defp maybe_preload_bookmarks(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_preload_bookmarks(query, opts) do
+    query
+    |> Activity.with_preloaded_bookmark(opts["user"])
+  end
+
+  defp maybe_order(query, %{order: :desc}) do
+    query
+    |> order_by(desc: :id)
+  end
+
+  defp maybe_order(query, %{order: :asc}) do
+    query
+    |> order_by(asc: :id)
+  end
+
+  defp maybe_order(query, _), do: query
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query = from(activity in Activity)
 
     base_query
     |> maybe_preload_objects(opts)
+    |> maybe_preload_bookmarks(opts)
+    |> maybe_order(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -155,7 +155,7 @@ def outbox(conn, %{"nickname" => nickname} = params) do
 
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = recipient <- User.get_cached_by_nickname(nickname),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(params["actor"]),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]),
          true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
   alias Pleroma.User
 
+  @moduledoc "Prevent followbots from following with a bit of heuristic"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   # XXX: this should become User.normalize_by_ap_id() or similar, really.

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
   require Logger
+  @moduledoc "Drop and log everything received"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
   alias Pleroma.Object
 
+  @moduledoc "Ensure a re: is prepended on replies to a post with a Subject"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -4,6 +4,8 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
   alias Pleroma.User
+  @moduledoc "Block messages with too much mentions (configurable)"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp delist_message(message, threshold) when threshold > 0 do

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
+  @moduledoc "Reject or Word-Replace messages with a keyword or regex"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
   defp string_matches?(string, _) when not is_binary(string) do
     false

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
+  @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/noop_policy.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do
+  @moduledoc "Does nothing (lets the messages go through unmodified)"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
+  @moduledoc "Scrub configured hypertext markup"
   alias Pleroma.HTML
 
   @behaviour Pleroma.Web.ActivityPub.MRF

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do
   alias Pleroma.User
+  @moduledoc "Rejects non-public (followers-only, direct) activities"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   alias Pleroma.User
+  @moduledoc "Filter activities depending on their origin instance"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp check_accept(%{host: actor_host} = _actor_info, object) do

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -5,6 +5,19 @@
 defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
   alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
+  @moduledoc """
+     Apply policies based on user tags
+
+     This policy applies policies on a user activities depending on their tags
+     on your instance.
+
+     - `mrf_tag:media-force-nsfw`: Mark as sensitive on presence of attachments
+     - `mrf_tag:media-strip`: Remove attachments
+     - `mrf_tag:force-unlisted`: Mark as unlisted (removes from the federated timeline)
+     - `mrf_tag:sandbox`: Remove from public (local and federated) timelines
+     - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests
+     - `mrf_tag:disable-any-subscription`: Reject any follow requests
+  """
 
   defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
   defp get_tags(_), do: []

lib/pleroma/web/activity_pub/mrf/user_allowlist.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
   alias Pleroma.Config
 
+  @moduledoc "Accept-list of users from specified instances"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp filter_by_list(object, []), do: {:ok, object}

lib/pleroma/web/activity_pub/relay.ex

@@ -15,7 +15,7 @@ def get_actor do
 
   def follow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.follow(local_user, target_user) do
       Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}")
       {:ok, activity}
@@ -28,7 +28,7 @@ def follow(target_instance) do
 
   def unfollow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
       {:ok, activity}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -126,7 +126,7 @@ def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collec
   def fix_implicit_addressing(object, _), do: object
 
   def fix_addressing(object) do
-    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
     followers_collection = User.ap_followers(user)
 
     object
@@ -407,7 +407,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
       |> fix_addressing
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
       params = %{
@@ -436,7 +436,7 @@ def handle_incoming(
         %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
       with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]),
            {:user_blocked, false} <-
@@ -485,7 +485,7 @@ def handle_incoming(
         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -511,7 +511,7 @@ def handle_incoming(
         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -535,7 +535,7 @@ def handle_incoming(
         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -548,7 +548,7 @@ def handle_incoming(
         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
@@ -603,7 +603,7 @@ def handle_incoming(
     object_id = Utils.get_ap_id(object_id)
 
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
@@ -622,7 +622,7 @@ def handle_incoming(
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
@@ -640,7 +640,7 @@ def handle_incoming(
         } = _data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.unfollow(follower, followed, id, false) do
       User.unfollow(follower, followed)
       {:ok, activity}
@@ -659,7 +659,7 @@ def handle_incoming(
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked),
-         %User{} = blocker <- User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do
       User.unblock(blocker, blocked)
       {:ok, activity}
@@ -673,7 +673,7 @@ def handle_incoming(
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
       User.unfollow(blocker, blocked)
       User.block(blocker, blocked)
@@ -692,7 +692,7 @@ def handle_incoming(
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
@@ -856,10 +856,16 @@ def add_mention_tags(object) do
     |> Map.put("tag", tags ++ mentions)
   end
 
+  def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do
+    user_info = add_emoji_tags(user_info)
+
+    object
+    |> Map.put(:info, user_info)
+  end
+
   # TODO: we should probably send mtime instead of unix epoch time for updated
-  def add_emoji_tags(object) do
+  def add_emoji_tags(%{"emoji" => emoji} = object) do
     tags = object["tag"] || []
-    emoji = object["emoji"] || []
 
     out =
       emoji
@@ -877,6 +883,10 @@ def add_emoji_tags(object) do
     |> Map.put("tag", tags ++ out)
   end
 
+  def add_emoji_tags(object) do
+    object
+  end
+
   def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -69,6 +69,11 @@ def render("user.json", %{user: user}) do
 
     endpoints = render("endpoints.json", %{user: user})
 
+    user_tags =
+      user
+      |> Transmogrifier.add_emoji_tags()
+      |> Map.get("tag", [])
+
     %{
       "id" => user.ap_id,
       "type" => "Person",
@@ -87,7 +92,7 @@ def render("user.json", %{user: user}) do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "tag" => user.info.source_data["tag"] || []
+      "tag" => (user.info.source_data["tag"] || []) ++ user_tags
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

lib/pleroma/web/auth/authenticator.ex

@@ -42,4 +42,30 @@ def oauth_consumer_template do
     implementation().oauth_consumer_template() ||
       Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html")
   end
+
+  @doc "Gets user by nickname or email for auth."
+  @spec fetch_user(String.t()) :: User.t() | nil
+  def fetch_user(name) do
+    User.get_by_nickname_or_email(name)
+  end
+
+  # Gets name and password from conn
+  #
+  @spec fetch_credentials(Plug.Conn.t() | map()) ::
+          {:ok, {name :: any, password :: any}} | {:error, :invalid_credentials}
+  def fetch_credentials(%Plug.Conn{params: params} = _),
+    do: fetch_credentials(params)
+
+  def fetch_credentials(params) do
+    case params do
+      %{"authorization" => %{"name" => name, "password" => password}} ->
+        {:ok, {name, password}}
+
+      %{"grant_type" => "password", "username" => name, "password" => password} ->
+        {:ok, {name, password}}
+
+      _ ->
+        {:error, :invalid_credentials}
+    end
+  end
 end

lib/pleroma/web/auth/ldap_authenticator.ex

@@ -7,6 +7,9 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
   require Logger
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
   @base Pleroma.Web.Auth.PleromaAuthenticator
 
@@ -20,30 +23,20 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
   defdelegate oauth_consumer_template, to: @base
 
   def get_user(%Plug.Conn{} = conn) do
-    if Pleroma.Config.get([:ldap, :enabled]) do
+    with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])},
-      {name, password} =
+         {:ok, {name, password}} <- fetch_credentials(conn),
-        case conn.params do
+         %User{} = user <- ldap_user(name, password) do
-          %{"authorization" => %{"name" => name, "password" => password}} ->
+      {:ok, user}
-            {name, password}
-
-          %{"grant_type" => "password", "username" => name, "password" => password} ->
-            {name, password}
-        end
-
-      case ldap_user(name, password) do
-        %User{} = user ->
-          {:ok, user}
-
-        {:error, {:ldap_connection_error, _}} ->
-          # When LDAP is unavailable, try default authenticator
-          @base.get_user(conn)
-
-        error ->
-          error
-      end
     else
-      # Fall back to default authenticator
+      {:error, {:ldap_connection_error, _}} ->
-      @base.get_user(conn)
+        # When LDAP is unavailable, try default authenticator
+        @base.get_user(conn)
+
+      {:ldap, _} ->
+        @base.get_user(conn)
+
+      error ->
+        error
     end
   end
 
@@ -94,7 +87,7 @@ defp bind_user(connection, ldap, name, password) do
 
     case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do
       :ok ->
-        case User.get_by_nickname_or_email(name) do
+        case fetch_user(name) do
           %User{} = user ->
             user
 

lib/pleroma/web/auth/pleroma_authenticator.ex

@@ -8,19 +8,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
   alias Pleroma.Repo
   alias Pleroma.User
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
-    {name, password} =
+    with {:ok, {name, password}} <- fetch_credentials(conn),
-      case conn.params do
+         {_, %User{} = user} <- {:user, fetch_user(name)},
-        %{"authorization" => %{"name" => name, "password" => password}} ->
-          {name, password}
-
-        %{"grant_type" => "password", "username" => name, "password" => password} ->
-          {name, password}
-      end
-
-    with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
          {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do
       {:ok, user}
     else

lib/pleroma/web/common_api/common_api.ex

@@ -151,8 +151,8 @@ def post(user, %{"status" => status} = data) do
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
-         cw <- data["spoiler_text"],
+         cw <- data["spoiler_text"] || "",
-         full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
+         full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
            make_note_data(
@@ -170,10 +170,7 @@ def post(user, %{"status" => status} = data) do
            Map.put(
              object,
              "emoji",
-             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
+             Formatter.get_emoji_map(full_payload)
-             |> Enum.reduce(%{}, fn {name, file, _}, acc ->
-               Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
-             end)
            ) do
       res =
         ActivityPub.create(

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -8,7 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Activity
   alias Pleroma.Bookmark
   alias Pleroma.Config
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -23,6 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.AppView
+  alias Pleroma.Web.MastodonAPI.ConversationView
   alias Pleroma.Web.MastodonAPI.FilterView
   alias Pleroma.Web.MastodonAPI.ListView
   alias Pleroma.Web.MastodonAPI.MastodonAPI
@@ -86,7 +89,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
     user_params =
       %{}
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end)
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
       |> add_if_present(params, "avatar", :avatar, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
@@ -96,6 +99,12 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
         end
       end)
 
+    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
+
+    user_info_emojis =
+      ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+      |> Enum.dedup()
+
     info_params =
       [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role]
       |> Enum.reduce(%{}, fn key, acc ->
@@ -112,6 +121,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
           _ -> :error
         end
       end)
+      |> Map.put(:emoji, user_info_emojis)
 
     info_cng = User.Info.profile_update(user.info, info_params)
 
@@ -157,7 +167,7 @@ def user(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do
     end
   end
 
-  @mastodon_api_level "2.5.0"
+  @mastodon_api_level "2.6.5"
 
   def masto_instance(conn, _params) do
     instance = Config.get(:instance)
@@ -285,8 +295,6 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:home_timeline, activities)
     |> put_view(StatusView)
@@ -305,8 +313,6 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.fetch_public_activities()
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:public_timeline, activities, false, %{"local" => local_only})
     |> put_view(StatusView)
@@ -314,8 +320,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
   end
 
   def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
-    with %User{} = user <- User.get_cached_by_id(params["id"]),
+    with %User{} = user <- User.get_cached_by_id(params["id"]) do
-         reading_user <- Repo.preload(reading_user, :bookmarks) do
       activities = ActivityPub.fetch_user_activities(user, reading_user, params)
 
       conn
@@ -342,8 +347,6 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.fetch_activities_query(params)
       |> Pagination.fetch_paginated(params)
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:dm_timeline, activities)
     |> put_view(StatusView)
@@ -353,8 +356,6 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
   def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.visible_for_user?(activity, user) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user})
@@ -504,8 +505,6 @@ def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, announce, _activity} <- CommonAPI.repeat(ap_id_or_id, user),
          %Activity{} = announce <- Activity.normalize(announce.data) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: announce, for: user, as: :activity})
@@ -515,8 +514,6 @@ def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -567,8 +564,6 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
          {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -580,8 +575,6 @@ def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
          {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -704,7 +697,7 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     end
   end
 
-  def favourited_by(conn, %{"id" => id}) do
+  def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
@@ -712,13 +705,13 @@ def favourited_by(conn, %{"id" => id}) do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{users: users, as: :user})
+      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
   end
 
-  def reblogged_by(conn, %{"id" => id}) do
+  def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
@@ -726,7 +719,7 @@ def reblogged_by(conn, %{"id" => id}) do
 
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
@@ -783,7 +776,7 @@ def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
       conn
       |> add_link_headers(:followers, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -800,7 +793,7 @@ def following(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
       conn
       |> add_link_headers(:following, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -808,7 +801,7 @@ def follow_requests(%{assigns: %{user: followed}} = conn, _params) do
     with {:ok, follow_requests} <- User.get_follow_requests(followed) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: follow_requests, as: :user})
+      |> render("accounts.json", %{for: followed, users: follow_requests, as: :user})
     end
   end
 
@@ -1102,8 +1095,6 @@ def favourites(%{assigns: %{user: user}} = conn, params) do
       ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:favourites, activities)
     |> put_view(StatusView)
@@ -1149,7 +1140,6 @@ def user_favourites(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params
 
   def bookmarks(%{assigns: %{user: user}} = conn, params) do
     user = User.get_cached_by_id(user.id)
-    user = Repo.preload(user, bookmarks: :activity)
 
     bookmarks =
       Bookmark.for_user_query(user.id)
@@ -1157,7 +1147,7 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do
 
     activities =
       bookmarks
-      |> Enum.map(fn b -> b.activity end)
+      |> Enum.map(fn b -> Map.put(b.activity, :bookmark, Map.delete(b, :activity)) end)
 
     conn
     |> add_link_headers(:bookmarks, bookmarks)
@@ -1235,7 +1225,7 @@ def list_accounts(%{assigns: %{user: user}} = conn, %{"id" => id}) do
          {:ok, users} = Pleroma.List.get_following(list) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     end
   end
 
@@ -1266,8 +1256,6 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params)
         |> ActivityPub.fetch_activities_bounded(following, params)
         |> Enum.reverse()
 
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: user, as: :activity})
@@ -1295,8 +1283,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
       initial_state =
         %{
           meta: %{
-            streaming_api_base_url:
+            streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(),
-              String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
             access_token: token,
             locale: "en",
             domain: Pleroma.Web.Endpoint.host(),
@@ -1653,7 +1640,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
               x,
               "id",
               case User.get_or_fetch(x["acct"]) do
-                %{id: id} -> id
+                {:ok, %User{id: id}} -> id
                 _ -> 0
               end
             )
@@ -1705,6 +1692,31 @@ def reports(%{assigns: %{user: user}} = conn, params) do
     end
   end
 
+  def conversations(%{assigns: %{user: user}} = conn, params) do
+    participations = Participation.for_user_with_last_activity_id(user, params)
+
+    conversations =
+      Enum.map(participations, fn participation ->
+        ConversationView.render("participation.json", %{participation: participation, user: user})
+      end)
+
+    conn
+    |> add_link_headers(:conversations, participations)
+    |> json(conversations)
+  end
+
+  def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do
+    with %Participation{} = participation <-
+           Repo.get_by(Participation, id: participation_id, user_id: user.id),
+         {:ok, participation} <- Participation.mark_as_read(participation) do
+      participation_view =
+        ConversationView.render("participation.json", %{participation: participation, user: user})
+
+      conn
+      |> json(participation_view)
+    end
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     res = render(conn, target, params)

lib/pleroma/web/mastodon_api/views/conversation_view.ex

@@ -0,0 +1,38 @@
+defmodule Pleroma.Web.MastodonAPI.ConversationView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Activity
+  alias Pleroma.Repo
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  def render("participation.json", %{participation: participation, user: user}) do
+    participation = Repo.preload(participation, conversation: :users)
+
+    last_activity_id =
+      with nil <- participation.last_activity_id do
+        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
+          "user" => user,
+          "blocking_user" => user
+        })
+      end
+
+    activity = Activity.get_by_id_with_object(last_activity_id)
+
+    last_status = StatusView.render("status.json", %{activity: activity, for: user})
+
+    accounts =
+      AccountView.render("accounts.json", %{
+        users: participation.conversation.users,
+        as: :user
+      })
+
+    %{
+      id: participation.id |> to_string(),
+      accounts: accounts,
+      unread: !participation.read,
+      last_status: last_status
+    }
+  end
+end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -75,18 +75,22 @@ def render("index.json", opts) do
 
   def render(
         "status.json",
-        %{activity: %{data: %{"type" => "Announce", "object" => object}} = activity} = opts
+        %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts
       ) do
     user = get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
+    activity_object = Object.normalize(activity)
+
+    reblogged_activity =
+      Activity.create_by_object_ap_id(activity_object.data["id"])
+      |> Activity.with_preloaded_bookmark(opts[:for])
+      |> Repo.one()
 
-    reblogged_activity = Activity.get_create_by_object_ap_id(object)
     reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity))
 
-    activity_object = Object.normalize(activity)
     favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || [])
 
-    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], reblogged_activity)
+    bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil
 
     mentions =
       activity.recipients
@@ -96,8 +100,8 @@ def render(
 
     %{
       id: to_string(activity.id),
-      uri: object,
+      uri: activity_object.data["id"],
-      url: object,
+      url: activity_object.data["id"],
       account: AccountView.render("account.json", %{user: user}),
       in_reply_to_id: nil,
       in_reply_to_account_id: nil,
@@ -149,7 +153,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
 
     favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 
-    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], activity)
+    bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil
 
     attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)

lib/pleroma/web/oauth/app.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.App do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @type t :: %__MODULE__{}
   schema "apps" do
     field(:client_name, :string)
     field(:redirect_uris, :string)

lib/pleroma/web/oauth/authorization.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
   import Ecto.Changeset
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
@@ -63,4 +64,11 @@ def delete_user_authorizations(%User{id: user_id}) do
     )
     |> Repo.delete_all()
   end
+
+  @doc "gets auth for app by token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
 end

lib/pleroma/web/oauth/oauth_controller.ex

@@ -13,11 +13,15 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken
 
   import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
 
   if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth)
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+
   plug(:fetch_session)
   plug(:fetch_flash)
 
@@ -138,25 +142,33 @@ defp handle_create_authorization_error(conn, error, %{"authorization" => _}) do
     Authenticator.handle_error(conn, error)
   end
 
+  @doc "Renew access_token with refresh_token"
+  def token_exchange(
+        conn,
+        %{"grant_type" => "refresh_token", "refresh_token" => token} = params
+      ) do
+    with %App{} = app <- get_app_from_request(conn, params),
+         {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token),
+         {:ok, token} <- RefreshToken.grant(token) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
+
+      json(conn, response_token(user, token, response_attrs))
+    else
+      _error ->
+        put_status(conn, 400)
+        |> json(%{error: "Invalid credentials"})
+    end
+  end
+
   def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         fixed_token = fix_padding(params["code"]),
+         fixed_token = Token.Utils.fix_padding(params["code"]),
-         %Authorization{} = auth <-
+         {:ok, auth} <- Authorization.get_by_token(app, fixed_token),
-           Repo.get_by(Authorization, token: fixed_token, app_id: app.id),
          %User{} = user <- User.get_cached_by_id(auth.user_id),
-         {:ok, token} <- Token.exchange_token(app, auth),
+         {:ok, token} <- Token.exchange_token(app, auth) do
-         {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        created_at: DateTime.to_unix(inserted_at),
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
 
-      json(conn, response)
+      json(conn, response_token(user, token, response_attrs))
     else
       _error ->
         put_status(conn, 400)
@@ -177,16 +189,7 @@ def token_exchange(
          true <- Enum.any?(scopes),
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes),
          {:ok, token} <- Token.exchange_token(app, auth) do
-      response = %{
+      json(conn, response_token(user, token))
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
-
-      json(conn, response)
     else
       {:auth_active, false} ->
         # Per https://github.com/tootsuite/mastodon/blob/
@@ -218,10 +221,12 @@ def token_exchange(
     token_exchange(conn, params)
   end
 
-  def token_revoke(conn, %{"token" => token} = params) do
+  # Bad request
+  def token_exchange(conn, params), do: bad_request(conn, params)
+
+  def token_revoke(conn, %{"token" => _token} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         %Token{} = token <- Repo.get_by(Token, token: token, app_id: app.id),
+         {:ok, _token} <- RevokeToken.revoke(app, params) do
-         {:ok, %Token{}} <- Repo.delete(token) do
       json(conn, %{})
     else
       _error ->
@@ -230,6 +235,15 @@ def token_revoke(conn, %{"token" => token} = params) do
     end
   end
 
+  def token_revoke(conn, params), do: bad_request(conn, params)
+
+  # Response for bad request
+  defp bad_request(conn, _) do
+    conn
+    |> put_status(500)
+    |> json(%{error: "Bad request"})
+  end
+
   @doc "Prepares OAuth request to provider for Ueberauth"
   def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do
     scope =
@@ -278,25 +292,22 @@ def callback(conn, params) do
     params = callback_params(params)
 
     with {:ok, registration} <- Authenticator.get_registration(conn) do
-      user = Repo.preload(registration, :user).user
       auth_attrs = Map.take(params, ~w(client_id redirect_uri scope scopes state))
 
-      if user do
+      case Repo.get_assoc(registration, :user) do
-        create_authorization(
+        {:ok, user} ->
-          conn,
+          create_authorization(conn, %{"authorization" => auth_attrs}, user: user)
-          %{"authorization" => auth_attrs},
-          user: user
-        )
-      else
-        registration_params =
-          Map.merge(auth_attrs, %{
-            "nickname" => Registration.nickname(registration),
-            "email" => Registration.email(registration)
-          })
 
-        conn
+        _ ->
-        |> put_session(:registration_id, registration.id)
+          registration_params =
-        |> registration_details(%{"authorization" => registration_params})
+            Map.merge(auth_attrs, %{
+              "nickname" => Registration.nickname(registration),
+              "email" => Registration.email(registration)
+            })
+
+          conn
+          |> put_session(:registration_id, registration.id)
+          |> registration_details(%{"authorization" => registration_params})
       end
     else
       _ ->
@@ -399,36 +410,30 @@ defp do_create_authorization(
     end
   end
 
-  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  defp get_app_from_request(conn, params) do
-  # decoding it.  Investigate sometime.
+    conn
-  defp fix_padding(token) do
+    |> fetch_client_credentials(params)
-    token
+    |> fetch_client
-    |> URI.decode()
-    |> Base.url_decode64!(padding: false)
-    |> Base.url_encode64(padding: false)
   end
 
-  defp get_app_from_request(conn, params) do
+  defp fetch_client({id, secret}) when is_binary(id) and is_binary(secret) do
-    # Per RFC 6749, HTTP Basic is preferred to body params
+    Repo.get_by(App, client_id: id, client_secret: secret)
-    {client_id, client_secret} =
+  end
-      with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-           {:ok, decoded} <- Base.decode64(encoded),
-           [id, secret] <-
-             String.split(decoded, ":")
-             |> Enum.map(fn s -> URI.decode_www_form(s) end) do
-        {id, secret}
-      else
-        _ -> {params["client_id"], params["client_secret"]}
-      end
 
-    if client_id && client_secret do
+  defp fetch_client({_id, _secret}), do: nil
-      Repo.get_by(
+
-        App,
+  defp fetch_client_credentials(conn, params) do
-        client_id: client_id,
+    # Per RFC 6749, HTTP Basic is preferred to body params
-        client_secret: client_secret
+    with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-      )
+         {:ok, decoded} <- Base.decode64(encoded),
+         [id, secret] <-
+           Enum.map(
+             String.split(decoded, ":"),
+             fn s -> URI.decode_www_form(s) end
+           ) do
+      {id, secret}
     else
-      nil
+      _ -> {params["client_id"], params["client_secret"]}
     end
   end
 
@@ -441,4 +446,16 @@ defp get_session_registration_id(conn), do: get_session(conn, :registration_id)
 
   defp put_session_registration_id(conn, registration_id),
     do: put_session(conn, :registration_id, registration_id)
+
+  defp response_token(%User{} = user, token, opts \\ %{}) do
+    %{
+      token_type: "Bearer",
+      access_token: token.token,
+      refresh_token: token.refresh_token,
+      expires_in: @expires_in,
+      scope: Enum.join(token.scopes, " "),
+      me: user.ap_id
+    }
+    |> Map.merge(opts)
+  end
 end

lib/pleroma/web/oauth/token.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
   import Ecto.Query
+  import Ecto.Changeset
 
   alias Pleroma.Repo
   alias Pleroma.User
@@ -13,6 +14,9 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+  @type t :: %__MODULE__{}
+
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
@@ -24,28 +28,67 @@ defmodule Pleroma.Web.OAuth.Token do
     timestamps()
   end
 
+  @doc "Gets token for app by access token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
+
+  @doc "Gets token for app by refresh token"
+  @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_refresh_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__,
+      where: t.app_id == ^app_id and t.refresh_token == ^token,
+      preload: [:user]
+    )
+    |> Repo.find_resource()
+  end
+
   def exchange_token(app, auth) do
     with {:ok, auth} <- Authorization.use_token(auth),
          true <- auth.app_id == app.id do
-      create_token(app, User.get_cached_by_id(auth.user_id), auth.scopes)
+      create_token(
+        app,
+        User.get_cached_by_id(auth.user_id),
+        %{scopes: auth.scopes}
+      )
     end
   end
 
-  def create_token(%App{} = app, %User{} = user, scopes \\ nil) do
+  defp put_token(changeset) do
-    scopes = scopes || app.scopes
+    changeset
-    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    |> change(%{token: Token.Utils.generate_token()})
-    refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
 
-    token = %Token{
+  defp put_refresh_token(changeset, attrs) do
-      token: token,
+    refresh_token = Map.get(attrs, :refresh_token, Token.Utils.generate_token())
-      refresh_token: refresh_token,
-      scopes: scopes,
-      user_id: user.id,
-      app_id: app.id,
-      valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10)
-    }
 
-    Repo.insert(token)
+    changeset
+    |> change(%{refresh_token: refresh_token})
+    |> validate_required([:refresh_token])
+    |> unique_constraint(:refresh_token)
+  end
+
+  defp put_valid_until(changeset, attrs) do
+    expires_in =
+      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in))
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
+    %__MODULE__{user_id: user.id, app_id: app.id}
+    |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes])
+    |> validate_required([:scopes, :user_id, :app_id])
+    |> put_valid_until(attrs)
+    |> put_token
+    |> put_refresh_token(attrs)
+    |> Repo.insert()
   end
 
   def delete_user_tokens(%User{id: user_id}) do
@@ -73,4 +116,10 @@ def get_user_tokens(%User{id: user_id}) do
     |> Repo.all()
     |> Repo.preload(:app)
   end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
 end

lib/pleroma/web/oauth/token/strategy/refresh_token.ex

@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do
+  @moduledoc """
+  Functions for dealing with refresh token strategy.
+  """
+
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke
+
+  @doc """
+  Will grant access token by refresh token.
+  """
+  @spec grant(Token.t()) :: {:ok, Token.t()} | {:error, any()}
+  def grant(token) do
+    access_token = Repo.preload(token, [:user, :app])
+
+    result =
+      Repo.transaction(fn ->
+        token_params = %{
+          app: access_token.app,
+          user: access_token.user,
+          scopes: access_token.scopes
+        }
+
+        access_token
+        |> revoke_access_token()
+        |> create_access_token(token_params)
+      end)
+
+    case result do
+      {:ok, {:error, reason}} -> {:error, reason}
+      {:ok, {:ok, token}} -> {:ok, token}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  defp revoke_access_token(token) do
+    Revoke.revoke(token)
+  end
+
+  defp create_access_token({:error, error}, _), do: {:error, error}
+
+  defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do
+    Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token))
+  end
+
+  defp add_refresh_token(params, token) do
+    case Config.get([:oauth2, :issue_new_refresh_token], false) do
+      true -> Map.put(params, :refresh_token, token)
+      false -> params
+    end
+  end
+end

lib/pleroma/web/oauth/token/strategy/revoke.ex

@@ -0,0 +1,22 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.Revoke do
+  @moduledoc """
+  Functions for dealing with revocation.
+  """
+
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Token
+
+  @doc "Finds and revokes access token for app and by token"
+  @spec revoke(App.t(), map()) :: {:ok, Token.t()} | {:error, :not_found | Ecto.Changeset.t()}
+  def revoke(%App{} = app, %{"token" => token} = _attrs) do
+    with {:ok, token} <- Token.get_by_token(app, token),
+         do: revoke(token)
+  end
+
+  @doc "Revokes access token"
+  @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
+  def revoke(%Token{} = token) do
+    Repo.delete(token)
+  end
+end

lib/pleroma/web/oauth/token/utils.ex

@@ -0,0 +1,30 @@
+defmodule Pleroma.Web.OAuth.Token.Utils do
+  @moduledoc """
+  Auxiliary functions for dealing with tokens.
+  """
+
+  @doc "convert token inserted_at to unix timestamp"
+  def format_created_at(%{inserted_at: inserted_at} = _token) do
+    inserted_at
+    |> DateTime.from_naive!("Etc/UTC")
+    |> DateTime.to_unix()
+  end
+
+  @doc false
+  @spec generate_token(keyword()) :: binary()
+  def generate_token(opts \\ []) do
+    opts
+    |> Keyword.get(:size, 32)
+    |> :crypto.strong_rand_bytes()
+    |> Base.url_encode64(padding: false)
+  end
+
+  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  # decoding it.  Investigate sometime.
+  def fix_padding(token) do
+    token
+    |> URI.decode()
+    |> Base.url_decode64!(padding: false)
+    |> Base.url_encode64(padding: false)
+  end
+end

lib/pleroma/web/router.ex

@@ -276,6 +276,9 @@ defmodule Pleroma.Web.Router do
 
       get("/suggestions", MastodonAPIController, :suggestions)
 
+      get("/conversations", MastodonAPIController, :conversations)
+      post("/conversations/:id/read", MastodonAPIController, :conversation_read)
+
       get("/endorsements", MastodonAPIController, :empty_array)
 
       get("/pleroma/flavour", MastodonAPIController, :get_flavour)

lib/pleroma/web/streamer.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Streamer do
   use GenServer
   require Logger
   alias Pleroma.Activity
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.User
@@ -71,6 +72,15 @@ def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do
     {:noreply, topics}
   end
 
+  def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do
+    user_topic = "direct:#{participation.user_id}"
+    Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n")
+
+    push_to_socket(topics, user_topic, participation)
+
+    {:noreply, topics}
+  end
+
   def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do
     # filter the recipient list if the activity is not public, see #270.
     recipient_lists =
@@ -192,6 +202,19 @@ defp represent_update(%Activity{} = activity) do
     |> Jason.encode!()
   end
 
+  def represent_conversation(%Participation{} = participation) do
+    %{
+      event: "conversation",
+      payload:
+        Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{
+          participation: participation,
+          user: participation.user
+        })
+        |> Jason.encode!()
+    }
+    |> Jason.encode!()
+  end
+
   def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do
     Enum.each(topics[topic] || [], fn socket ->
       # Get the current user so we have up-to-date blocks etc.
@@ -214,6 +237,12 @@ def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = ite
     end)
   end
 
+  def push_to_socket(topics, topic, %Participation{} = participation) do
+    Enum.each(topics[topic] || [], fn socket ->
+      send(socket.transport_pid, {:text, represent_conversation(participation)})
+    end)
+  end
+
   def push_to_socket(topics, topic, %Activity{
         data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id}
       }) do

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -352,7 +352,7 @@ def change_password(%{assigns: %{user: user}} = conn, params) do
   def delete_account(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->
-        Task.start(fn -> User.delete(user) end)
+        User.delete(user)
         json(conn, %{status: "success"})
 
       {:error, msg} ->

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -293,7 +293,7 @@ def search(_user, %{"q" => query} = params) do
   end
 
   def get_external_profile(for_user, uri) do
-    with %User{} = user <- User.get_or_fetch(uri) do
+    with {:ok, %User{} = user} <- User.get_or_fetch(uri) do
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
     else
       _e ->

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -181,6 +182,7 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
       |> Map.put("blocking_user", user)
       |> Map.put("user", user)
       |> Map.put(:visibility, "direct")
+      |> Map.put(:order, :desc)
 
     activities =
       ActivityPub.fetch_activities_query([user.ap_id], params)
@@ -653,7 +655,22 @@ defp build_info_cng(user, params) do
 
   defp parse_profile_bio(user, params) do
     if bio = params["description"] do
-      Map.put(params, "bio", User.parse_bio(bio, user))
+      emojis_text = (params["description"] || "") <> " " <> (params["name"] || "")
+
+      emojis =
+        ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+        |> Enum.dedup()
+
+      user_info =
+        user.info
+        |> Map.put(
+          "emoji",
+          emojis
+        )
+
+      params
+      |> Map.put("bio", User.parse_bio(bio, user))
+      |> Map.put("info", user_info)
     else
       params
     end

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -67,6 +67,13 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
         {String.trim(name, ":"), url}
       end)
 
+    emoji = Enum.dedup(emoji ++ user.info.emoji)
+
+    description_html =
+      (user.bio || "")
+      |> HTML.filter_tags(User.html_filter_policy(for_user))
+      |> Formatter.emojify(emoji)
+
     # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
     # For example: [{"name": "Pronoun", "value": "she/her"}, …]
     fields =
@@ -78,7 +85,7 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
       %{
         "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
         "description" => HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
-        "description_html" => HTML.filter_tags(user.bio, User.html_filter_policy(for_user)),
+        "description_html" => description_html,
         "favourites_count" => 0,
         "followers_count" => user_info[:follower_count],
         "following" => following,

mix.exs

@@ -16,11 +16,11 @@ def project do
 
       # Docs
       name: "Pleroma",
-      source_url: "https://git.pleroma.social/pleroma/pleroma",
-      source_url_pattern:
-        "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}",
       homepage_url: "https://pleroma.social/",
+      source_url: "https://git.pleroma.social/pleroma/pleroma",
       docs: [
+        source_url_pattern:
+          "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}",
         logo: "priv/static/static/logo.png",
         extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/**/*.md"),
         groups_for_extras: [
@@ -41,7 +41,7 @@ def project do
   def application do
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :quack],
+      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -87,7 +87,7 @@ defp deps do
       {:bbcode, "~> 0.1"},
       {:ex_machina, "~> 2.3", only: :test},
       {:credo, "~> 0.9.3", only: [:dev, :test]},
-      {:mock, "~> 0.3.1", only: :test},
+      {:mock, "~> 0.3.3", only: :test},
       {:crypt,
        git: "https://github.com/msantos/crypt", ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"},
       {:cors_plug, "~> 1.5"},
@@ -103,7 +103,7 @@ defp deps do
       {:ueberauth, "~> 0.4"},
       {:auto_linker,
        git: "https://git.pleroma.social/pleroma/auto_linker.git",
-       ref: "90613b4bae875a3610c275b7056b61ffdd53210d"},
+       ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},
@@ -114,7 +114,9 @@ defp deps do
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
       {:quantum, "~> 2.3"},
-      {:joken, "~> 2.0"}
+      {:joken, "~> 2.0"},
+      {:benchee, "~> 1.0"},
+      {:esshd, "~> 0.1.0"}
     ] ++ oauth_deps
   end
 

mix.lock

@@ -1,8 +1,9 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"},
-  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "90613b4bae875a3610c275b7056b61ffdd53210d", [ref: "90613b4bae875a3610c275b7056b61ffdd53210d"]},
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "cachex": {:hex, :cachex, "3.0.3", "4e2d3e05814a5738f5ff3903151d5c25636d72a3527251b753f501ad9c657967", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
   "calendar": {:hex, :calendar, "0.17.5", "0ff5b09a60b9677683aa2a6fee948558660501c74a289103ea099806bc41a352", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
@@ -18,10 +19,12 @@
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
   "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.0.8", "9eb6a1fcfc593e6619d45ef51afe607f1554c21ca188a1cd48eecc27223567f1", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"},
+  "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},
   "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},

priv/repo/migrations/20190408123347_create_conversations.exs

@@ -0,0 +1,26 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.CreateConversations do
+  use Ecto.Migration
+
+  def change do
+    create table(:conversations) do
+      add(:ap_id, :string, null: false)
+      timestamps()
+    end
+
+    create table(:conversation_participations) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:conversation_id, references(:conversations, on_delete: :delete_all))
+      add(:read, :boolean, default: false)
+
+      timestamps()
+    end
+
+    create index(:conversation_participations, [:conversation_id])
+    create unique_index(:conversation_participations, [:user_id, :conversation_id])
+    create unique_index(:conversations, [:ap_id])
+  end
+end

priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddParticipationUpdatedAtIndex do
+  use Ecto.Migration
+
+  def change do
+    create index(:conversation_participations, ["updated_at desc"])
+  end
+end

priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs

@@ -0,0 +1,8 @@
+defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do
+  use Ecto.Migration
+
+  def change do
+    drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts)
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do
+  use Ecto.Migration
+
+  def change do
+    create(unique_index(:oauth_tokens, [:refresh_token]))
+  end
+end

test/activity_test.exs

@@ -5,6 +5,7 @@
 defmodule Pleroma.ActivityTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   import Pleroma.Factory
 
   test "returns an activity by it's AP id" do
@@ -28,4 +29,48 @@ test "returns the activity that created an object" do
 
     assert activity == found_activity
   end
+
+  test "preloading a bookmark" do
+    user = insert(:user)
+    user2 = insert(:user)
+    user3 = insert(:user)
+    activity = insert(:note_activity)
+    {:ok, _bookmark} = Bookmark.create(user.id, activity.id)
+    {:ok, _bookmark2} = Bookmark.create(user2.id, activity.id)
+    {:ok, bookmark3} = Bookmark.create(user3.id, activity.id)
+
+    queried_activity =
+      Ecto.Query.from(Pleroma.Activity)
+      |> Activity.with_preloaded_bookmark(user3)
+      |> Repo.one()
+
+    assert queried_activity.bookmark == bookmark3
+  end
+
+  describe "getting a bookmark" do
+    test "when association is loaded" do
+      user = insert(:user)
+      activity = insert(:note_activity)
+      {:ok, bookmark} = Bookmark.create(user.id, activity.id)
+
+      queried_activity =
+        Ecto.Query.from(Pleroma.Activity)
+        |> Activity.with_preloaded_bookmark(user)
+        |> Repo.one()
+
+      assert Activity.get_bookmark(queried_activity, user) == bookmark
+    end
+
+    test "when association is not loaded" do
+      user = insert(:user)
+      activity = insert(:note_activity)
+      {:ok, bookmark} = Bookmark.create(user.id, activity.id)
+
+      queried_activity =
+        Ecto.Query.from(Pleroma.Activity)
+        |> Repo.one()
+
+      assert Activity.get_bookmark(queried_activity, user) == bookmark
+    end
+  end
 end

test/bbs/handler_test.exs

@@ -0,0 +1,83 @@
+defmodule Pleroma.BBS.HandlerTest do
+  use Pleroma.DataCase
+  alias Pleroma.Activity
+  alias Pleroma.BBS.Handler
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  import ExUnit.CaptureIO
+  import Pleroma.Factory
+  import Ecto.Query
+
+  test "getting the home timeline" do
+    user = insert(:user)
+    followed = insert(:user)
+
+    {:ok, user} = User.follow(user, followed)
+
+    {:ok, _first} = CommonAPI.post(user, %{"status" => "hey"})
+    {:ok, _second} = CommonAPI.post(followed, %{"status" => "hello"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "home")
+      end)
+
+    assert output =~ user.nickname
+    assert output =~ followed.nickname
+
+    assert output =~ "hey"
+    assert output =~ "hello"
+  end
+
+  test "posting" do
+    user = insert(:user)
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "p this is a test post")
+      end)
+
+    assert output =~ "Posted"
+
+    activity =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create")
+        )
+      )
+
+    assert activity.actor == user.ap_id
+    object = Object.normalize(activity)
+    assert object.data["content"] == "this is a test post"
+  end
+
+  test "replying" do
+    user = insert(:user)
+    another_user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post(another_user, %{"status" => "this is a test post"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "r #{activity.id} this is a reply")
+      end)
+
+    assert output =~ "Replied"
+
+    reply =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create"),
+          where: a.actor == ^user.ap_id
+        )
+      )
+
+    assert reply.actor == user.ap_id
+    object = Object.normalize(reply)
+    assert object.data["content"] == "this is a reply"
+    assert object.data["inReplyTo"] == activity.data["object"]
+  end
+end

test/conversation/participation_test.exs

@@ -0,0 +1,89 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Conversation.ParticipationTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Conversation.Participation
+  alias Pleroma.Web.CommonAPI
+
+  test "it creates a participation for a conversation and a user" do
+    user = insert(:user)
+    conversation = insert(:conversation)
+
+    {:ok, %Participation{} = participation} =
+      Participation.create_for_user_and_conversation(user, conversation)
+
+    assert participation.user_id == user.id
+    assert participation.conversation_id == conversation.id
+
+    :timer.sleep(1000)
+    # Creating again returns the same participation
+    {:ok, %Participation{} = participation_two} =
+      Participation.create_for_user_and_conversation(user, conversation)
+
+    assert participation.id == participation_two.id
+    refute participation.updated_at == participation_two.updated_at
+  end
+
+  test "recreating an existing participations sets it to unread" do
+    participation = insert(:participation, %{read: true})
+
+    {:ok, participation} =
+      Participation.create_for_user_and_conversation(
+        participation.user,
+        participation.conversation
+      )
+
+    refute participation.read
+  end
+
+  test "it marks a participation as read" do
+    participation = insert(:participation, %{read: false})
+    {:ok, participation} = Participation.mark_as_read(participation)
+
+    assert participation.read
+  end
+
+  test "it marks a participation as unread" do
+    participation = insert(:participation, %{read: true})
+    {:ok, participation} = Participation.mark_as_unread(participation)
+
+    refute participation.read
+  end
+
+  test "gets all the participations for a user, ordered by updated at descending" do
+    user = insert(:user)
+    {:ok, activity_one} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"})
+    :timer.sleep(1000)
+    {:ok, activity_two} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"})
+    :timer.sleep(1000)
+
+    {:ok, activity_three} =
+      CommonAPI.post(user, %{
+        "status" => "x",
+        "visibility" => "direct",
+        "in_reply_to_status_id" => activity_one.id
+      })
+
+    assert [participation_one, participation_two] = Participation.for_user(user)
+
+    object2 = Pleroma.Object.normalize(activity_two)
+    object3 = Pleroma.Object.normalize(activity_three)
+
+    assert participation_one.conversation.ap_id == object3.data["context"]
+    assert participation_two.conversation.ap_id == object2.data["context"]
+
+    # Pagination
+    assert [participation_one] = Participation.for_user(user, %{"limit" => 1})
+
+    assert participation_one.conversation.ap_id == object3.data["context"]
+
+    # With last_activity_id
+    assert [participation_one] =
+             Participation.for_user_with_last_activity_id(user, %{"limit" => 1})
+
+    assert participation_one.last_activity_id == activity_three.id
+  end
+end

test/conversation_test.exs

@@ -0,0 +1,137 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ConversationTest do
+  use Pleroma.DataCase
+  alias Pleroma.Conversation
+  alias Pleroma.Web.CommonAPI
+
+  import Pleroma.Factory
+
+  test "it creates a conversation for given ap_id" do
+    assert {:ok, %Conversation{} = conversation} =
+             Conversation.create_for_ap_id("https://some_ap_id")
+
+    # Inserting again returns the same
+    assert {:ok, conversation_two} = Conversation.create_for_ap_id("https://some_ap_id")
+    assert conversation_two.id == conversation.id
+  end
+
+  test "public posts don't create conversations" do
+    user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey"})
+
+    object = Pleroma.Object.normalize(activity)
+    context = object.data["context"]
+
+    conversation = Conversation.get_for_ap_id(context)
+
+    refute conversation
+  end
+
+  test "it creates or updates a conversation and participations for a given DM" do
+    har = insert(:user)
+    jafnhar = insert(:user, local: false)
+    tridi = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "direct"})
+
+    object = Pleroma.Object.normalize(activity)
+    context = object.data["context"]
+
+    conversation =
+      Conversation.get_for_ap_id(context)
+      |> Repo.preload(:participations)
+
+    assert conversation
+
+    assert Enum.find(conversation.participations, fn %{user_id: user_id} -> har.id == user_id end)
+
+    assert Enum.find(conversation.participations, fn %{user_id: user_id} ->
+             jafnhar.id == user_id
+           end)
+
+    {:ok, activity} =
+      CommonAPI.post(jafnhar, %{
+        "status" => "Hey @#{har.nickname}",
+        "visibility" => "direct",
+        "in_reply_to_status_id" => activity.id
+      })
+
+    object = Pleroma.Object.normalize(activity)
+    context = object.data["context"]
+
+    conversation_two =
+      Conversation.get_for_ap_id(context)
+      |> Repo.preload(:participations)
+
+    assert conversation_two.id == conversation.id
+
+    assert Enum.find(conversation_two.participations, fn %{user_id: user_id} ->
+             har.id == user_id
+           end)
+
+    assert Enum.find(conversation_two.participations, fn %{user_id: user_id} ->
+             jafnhar.id == user_id
+           end)
+
+    {:ok, activity} =
+      CommonAPI.post(tridi, %{
+        "status" => "Hey @#{har.nickname}",
+        "visibility" => "direct",
+        "in_reply_to_status_id" => activity.id
+      })
+
+    object = Pleroma.Object.normalize(activity)
+    context = object.data["context"]
+
+    conversation_three =
+      Conversation.get_for_ap_id(context)
+      |> Repo.preload([:participations, :users])
+
+    assert conversation_three.id == conversation.id
+
+    assert Enum.find(conversation_three.participations, fn %{user_id: user_id} ->
+             har.id == user_id
+           end)
+
+    assert Enum.find(conversation_three.participations, fn %{user_id: user_id} ->
+             jafnhar.id == user_id
+           end)
+
+    assert Enum.find(conversation_three.participations, fn %{user_id: user_id} ->
+             tridi.id == user_id
+           end)
+
+    assert Enum.find(conversation_three.users, fn %{id: user_id} ->
+             har.id == user_id
+           end)
+
+    assert Enum.find(conversation_three.users, fn %{id: user_id} ->
+             jafnhar.id == user_id
+           end)
+
+    assert Enum.find(conversation_three.users, fn %{id: user_id} ->
+             tridi.id == user_id
+           end)
+  end
+
+  test "create_or_bump_for returns the conversation with participations" do
+    har = insert(:user)
+    jafnhar = insert(:user, local: false)
+
+    {:ok, activity} =
+      CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "direct"})
+
+    {:ok, conversation} = Conversation.create_or_bump_for(activity)
+
+    assert length(conversation.participations) == 2
+
+    {:ok, activity} =
+      CommonAPI.post(har, %{"status" => "Hey @#{jafnhar.nickname}", "visibility" => "public"})
+
+    assert {:error, _} = Conversation.create_or_bump_for(activity)
+  end
+end

test/formatter_test.exs

@@ -147,7 +147,7 @@ test "gives a replacement for user links, using local nicknames in user links te
     end
 
     test "gives a replacement for user links when the user is using Osada" do
-      mike = User.get_or_fetch("mike@osada.macgirvin.com")
+      {:ok, mike} = User.get_or_fetch("mike@osada.macgirvin.com")
 
       text = "@mike@osada.macgirvin.com test"
 
@@ -248,7 +248,7 @@ test "it adds cool emoji" do
     text = "I love :firefox:"
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
+      "I love <img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
 
     assert Formatter.emojify(text) == expected_result
   end
@@ -263,7 +263,7 @@ test "it does not add XSS emoji" do
     }
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
+      "I love <img class=\"emoji\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
 
     assert Formatter.emojify(text, custom_emoji) == expected_result
   end

test/media_proxy_test.exs

@@ -7,15 +7,15 @@ defmodule Pleroma.MediaProxyTest do
   import Pleroma.Web.MediaProxy
   alias Pleroma.Web.MediaProxy.MediaProxyController
 
+  setup do
+    enabled = Pleroma.Config.get([:media_proxy, :enabled])
+    on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
+    :ok
+  end
+
   describe "when enabled" do
     setup do
-      enabled = Pleroma.Config.get([:media_proxy, :enabled])
+      Pleroma.Config.put([:media_proxy, :enabled], true)
-
-      unless enabled do
-        Pleroma.Config.put([:media_proxy, :enabled], true)
-        on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
-      end
-
       :ok
     end
 

test/plugs/oauth_plug_test.exs

@@ -38,6 +38,26 @@ test "with valid token(downcase), it assigns the user", %{conn: conn} = opts do
     assert conn.assigns[:user] == opts[:user]
   end
 
+  test "with valid token(downcase) in url parameters, it assings the user", opts do
+    conn =
+      :get
+      |> build_conn("/?access_token=#{opts[:token]}")
+      |> put_req_header("content-type", "application/json")
+      |> fetch_query_params()
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
+  test "with valid token(downcase) in body parameters, it assigns the user", opts do
+    conn =
+      :post
+      |> build_conn("/api/v1/statuses", access_token: opts[:token], status: "test")
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
   test "with invalid token, it not assigns the user", %{conn: conn} do
     conn =
       conn

test/repo_test.exs

@@ -0,0 +1,44 @@
+defmodule Pleroma.RepoTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  describe "find_resource/1" do
+    test "returns user" do
+      user = insert(:user)
+      query = from(t in Pleroma.User, where: t.id == ^user.id)
+      assert Repo.find_resource(query) == {:ok, user}
+    end
+
+    test "returns not_found" do
+      query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
+      assert Repo.find_resource(query) == {:error, :not_found}
+    end
+  end
+
+  describe "get_assoc/2" do
+    test "get assoc from preloaded data" do
+      user = %Pleroma.User{name: "Agent Smith"}
+      token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user}
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-one assoc from repo" do
+      user = insert(:user, name: "Jimi Hendrix")
+      token = refresh_record(insert(:oauth_token, user: user))
+
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-many assoc from repo" do
+      user = insert(:user)
+      notification = refresh_record(insert(:notification, user: user))
+
+      assert Repo.get_assoc(user, :notifications) == {:ok, [notification]}
+    end
+
+    test "return error if has not assoc " do
+      token = insert(:oauth_token, user: nil)
+      assert Repo.get_assoc(token, :user) == {:error, :not_found}
+    end
+  end
+end

test/support/factory.ex

@@ -5,6 +5,23 @@
 defmodule Pleroma.Factory do
   use ExMachina.Ecto, repo: Pleroma.Repo
 
+  def participation_factory do
+    conversation = insert(:conversation)
+    user = insert(:user)
+
+    %Pleroma.Conversation.Participation{
+      conversation: conversation,
+      user: user,
+      read: false
+    }
+  end
+
+  def conversation_factory do
+    %Pleroma.Conversation{
+      ap_id: sequence(:ap_id, &"https://some_conversation/#{&1}")
+    }
+  end
+
   def user_factory do
     user = %Pleroma.User{
       name: sequence(:name, &"Test テスト User #{&1}"),

test/user_test.exs

@@ -362,7 +362,7 @@ test "it creates confirmed user if :confirmed option is given" do
   describe "get_or_fetch/1" do
     test "gets an existing user by nickname" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch(user.nickname)
 
       assert user == fetched_user
     end
@@ -379,7 +379,7 @@ test "gets an existing user by ap_id" do
           info: %{}
         )
 
-      fetched_user = User.get_or_fetch(ap_id)
+      {:ok, fetched_user} = User.get_or_fetch(ap_id)
       freshed_user = refresh_record(user)
       assert freshed_user == fetched_user
     end
@@ -388,14 +388,14 @@ test "gets an existing user by ap_id" do
   describe "fetching a user from nickname or trying to build one" do
     test "gets an existing user" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch_by_nickname(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(user.nickname)
 
       assert user == fetched_user
     end
 
     test "gets an existing user, case insensitive" do
       user = insert(:user, nickname: "nick")
-      fetched_user = User.get_or_fetch_by_nickname("NICK")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("NICK")
 
       assert user == fetched_user
     end
@@ -403,7 +403,7 @@ test "gets an existing user, case insensitive" do
     test "gets an existing user by fully qualified nickname" do
       user = insert(:user)
 
-      fetched_user =
+      {:ok, fetched_user} =
         User.get_or_fetch_by_nickname(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
       assert user == fetched_user
@@ -413,24 +413,24 @@ test "gets an existing user by fully qualified nickname, case insensitive" do
       user = insert(:user, nickname: "nick")
       casing_altered_fqn = String.upcase(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
-      fetched_user = User.get_or_fetch_by_nickname(casing_altered_fqn)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(casing_altered_fqn)
 
       assert user == fetched_user
     end
 
     test "fetches an external user via ostatus if no user exists" do
-      fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
       assert fetched_user.nickname == "shp@social.heldscal.la"
     end
 
     test "returns nil if no user could be fetched" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
-      assert fetched_user == nil
+      assert fetched_user == "not found nonexistant@social.heldscal.la"
     end
 
     test "returns nil for nonexistant local user" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant")
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant")
-      assert fetched_user == nil
+      assert fetched_user == "not found nonexistant"
     end
 
     test "updates an existing user, if stale" do
@@ -448,7 +448,7 @@ test "updates an existing user, if stale" do
 
       assert orig_user.last_refreshed_at == a_week_ago
 
-      user = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
+      {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
       assert user.info.source_data["endpoints"]
 
       refute user.last_refreshed_at == orig_user.last_refreshed_at
@@ -829,10 +829,12 @@ test ".delete_user_activities deletes all create activities" do
     user = insert(:user)
 
     {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
-    {:ok, _} = User.delete_user_activities(user)
 
-    # TODO: Remove favorites, repeats, delete activities.
+    Ecto.Adapters.SQL.Sandbox.unboxed_run(Repo, fn ->
-    refute Activity.get_by_id(activity.id)
+      {:ok, _} = User.delete_user_activities(user)
+      # TODO: Remove favorites, repeats, delete activities.
+      refute Activity.get_by_id(activity.id)
+    end)
   end
 
   test ".delete deactivates a user, all follow relationships and all create activities" do
@@ -1107,7 +1109,7 @@ test "preserves hosts in user links text" do
       expected_text =
         "A.k.a. <span class='h-card'><a data-user='#{remote_user.id}' class='u-url mention' href='#{
           remote_user.ap_id
-        }'>" <> "@<span>nick@domain.com</span></a></span>"
+        }'>@<span>nick@domain.com</span></a></span>"
 
       assert expected_text == User.parse_bio(bio, user)
     end

test/web/activity_pub/activity_pub_test.exs

@@ -22,6 +22,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     :ok
   end
 
+  describe "streaming out participations" do
+    test "it streams them out" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"})
+
+      {:ok, conversation} = Pleroma.Conversation.create_or_bump_for(activity)
+
+      participations =
+        conversation.participations
+        |> Repo.preload(:user)
+
+      with_mock Pleroma.Web.Streamer,
+        stream: fn _, _ -> nil end do
+        ActivityPub.stream_out_participations(conversation.participations)
+
+        Enum.each(participations, fn participation ->
+          assert called(Pleroma.Web.Streamer.stream("participation", participation))
+        end)
+      end
+    end
+  end
+
   describe "fetching restricted by visibility" do
     test "it restricts by the appropriate visibility" do
       user = insert(:user)
@@ -130,9 +152,15 @@ test "drops activities beyond a certain limit" do
     end
 
     test "doesn't drop activities with content being null" do
+      user = insert(:user)
+
       data = %{
-        "ok" => true,
+        "actor" => user.ap_id,
+        "to" => [],
         "object" => %{
+          "actor" => user.ap_id,
+          "to" => [],
+          "type" => "Note",
           "content" => nil
         }
       }
@@ -148,8 +176,17 @@ test "returns the activity if one with the same id is already in" do
     end
 
     test "inserts a given map into the activity database, giving it an id if it has none." do
+      user = insert(:user)
+
       data = %{
-        "ok" => true
+        "actor" => user.ap_id,
+        "to" => [],
+        "object" => %{
+          "actor" => user.ap_id,
+          "to" => [],
+          "type" => "Note",
+          "content" => "hey"
+        }
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
@@ -159,9 +196,16 @@ test "inserts a given map into the activity database, giving it an id if it has
       given_id = "bla"
 
       data = %{
-        "ok" => true,
         "id" => given_id,
-        "context" => "blabla"
+        "actor" => user.ap_id,
+        "to" => [],
+        "context" => "blabla",
+        "object" => %{
+          "actor" => user.ap_id,
+          "to" => [],
+          "type" => "Note",
+          "content" => "hey"
+        }
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
@@ -172,26 +216,39 @@ test "inserts a given map into the activity database, giving it an id if it has
     end
 
     test "adds a context when none is there" do
+      user = insert(:user)
+
       data = %{
-        "id" => "some_id",
+        "actor" => user.ap_id,
+        "to" => [],
         "object" => %{
-          "id" => "object_id"
+          "actor" => user.ap_id,
+          "to" => [],
+          "type" => "Note",
+          "content" => "hey"
         }
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
+      object = Pleroma.Object.normalize(activity)
 
       assert is_binary(activity.data["context"])
-      assert is_binary(activity.data["object"]["context"])
+      assert is_binary(object.data["context"])
       assert activity.data["context_id"]
-      assert activity.data["object"]["context_id"]
+      assert object.data["context_id"]
     end
 
     test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do
+      user = insert(:user)
+
       data = %{
+        "actor" => user.ap_id,
+        "to" => [],
         "object" => %{
+          "actor" => user.ap_id,
+          "to" => [],
           "type" => "Note",
-          "ok" => true
+          "content" => "hey"
         }
       }
 

test/web/activity_pub/transmogrifier_test.exs

@@ -219,7 +219,7 @@ test "it rejects incoming follow requests from blocked users when deny_follow_bl
       Pleroma.Config.put([:user, :deny_follow_blocked], true)
 
       user = insert(:user)
-      target = User.get_or_fetch("http://mastodon.example.org/users/admin")
+      {:ok, target} = User.get_or_fetch("http://mastodon.example.org/users/admin")
 
       {:ok, user} = User.block(user, target)
 

test/web/auth/authenticator_test.exs

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Auth.AuthenticatorTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Web.Auth.Authenticator
+  import Pleroma.Factory
+
+  describe "fetch_user/1" do
+    test "returns user by name" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.nickname) == user
+    end
+
+    test "returns user by email" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.email) == user
+    end
+
+    test "returns nil" do
+      assert Authenticator.fetch_user("email") == nil
+    end
+  end
+
+  describe "fetch_credentials/1" do
+    test "returns name and password from authorization params" do
+      params = %{"authorization" => %{"name" => "test", "password" => "test-pass"}}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns name and password with grant_type 'password'" do
+      params = %{"grant_type" => "password", "username" => "test", "password" => "test-pass"}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns error" do
+      assert Authenticator.fetch_credentials(%{}) == {:error, :invalid_credentials}
+    end
+  end
+end

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -300,6 +300,65 @@ test "direct timeline", %{conn: conn} do
     assert status["url"] != direct.data["id"]
   end
 
+  test "Conversations", %{conn: conn} do
+    user_one = insert(:user)
+    user_two = insert(:user)
+
+    {:ok, user_two} = User.follow(user_two, user_one)
+
+    {:ok, direct} =
+      CommonAPI.post(user_one, %{
+        "status" => "Hi @#{user_two.nickname}!",
+        "visibility" => "direct"
+      })
+
+    {:ok, _follower_only} =
+      CommonAPI.post(user_one, %{
+        "status" => "Hi @#{user_two.nickname}!",
+        "visibility" => "private"
+      })
+
+    res_conn =
+      conn
+      |> assign(:user, user_one)
+      |> get("/api/v1/conversations")
+
+    assert response = json_response(res_conn, 200)
+
+    assert [
+             %{
+               "id" => res_id,
+               "accounts" => res_accounts,
+               "last_status" => res_last_status,
+               "unread" => unread
+             }
+           ] = response
+
+    assert length(res_accounts) == 2
+    assert is_binary(res_id)
+    assert unread == true
+    assert res_last_status["id"] == direct.id
+
+    # Apparently undocumented API endpoint
+    res_conn =
+      conn
+      |> assign(:user, user_one)
+      |> post("/api/v1/conversations/#{res_id}/read")
+
+    assert response = json_response(res_conn, 200)
+    assert length(response["accounts"]) == 2
+    assert response["last_status"]["id"] == direct.id
+    assert response["unread"] == false
+
+    # (vanilla) Mastodon frontend behaviour
+    res_conn =
+      conn
+      |> assign(:user, user_one)
+      |> get("/api/v1/statuses/#{res_last_status["id"]}/context")
+
+    assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200)
+  end
+
   test "doesn't include DMs from blocked users", %{conn: conn} do
     blocker = insert(:user)
     blocked = insert(:user)
@@ -2351,6 +2410,33 @@ test "requires 'write' permission", %{conn: conn} do
         end
       end
     end
+
+    test "updates profile emojos", %{conn: conn} do
+      user = insert(:user)
+
+      note = "*sips :blank:*"
+      name = "I am :firefox:"
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "note" => note,
+          "display_name" => name
+        })
+
+      assert json_response(conn, 200)
+
+      conn =
+        conn
+        |> get("/api/v1/accounts/#{user.id}")
+
+      assert user = json_response(conn, 200)
+
+      assert user["note"] == note
+      assert user["display_name"] == name
+      assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"]
+    end
   end
 
   test "get instance information", %{conn: conn} do

test/web/mastodon_api/status_view_test.exs

@@ -168,6 +168,8 @@ test "tells if the status is bookmarked" do
 
     {:ok, _bookmark} = Bookmark.create(user.id, activity.id)
 
+    activity = Activity.get_by_id_with_object(activity.id)
+
     status = StatusView.render("status.json", %{activity: activity, for: user})
 
     assert status.bookmarked == true

test/web/oauth/oauth_controller_test.exs

@@ -12,6 +12,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @oauth_config_path [:oauth2, :issue_new_refresh_token]
   @session_opts [
     store: :cookie,
     key: "_test",
@@ -714,4 +715,199 @@ test "rejects an invalid authorization code" do
       refute Map.has_key?(resp, "access_token")
     end
   end
+
+  describe "POST /oauth/token - refresh token" do
+    setup do
+      oauth_token_config = Pleroma.Config.get(@oauth_config_path)
+
+      on_exit(fn ->
+        Pleroma.Config.get(@oauth_config_path, oauth_token_config)
+      end)
+    end
+
+    test "issues a new access token with keep fresh token" do
+      Pleroma.Config.put(@oauth_config_path, true)
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      new_token = Repo.get_by(Token, token: response["access_token"])
+      assert new_token.refresh_token == token.refresh_token
+      assert new_token.scopes == auth.scopes
+      assert new_token.user_id == user.id
+      assert new_token.app_id == app.id
+    end
+
+    test "issues a new access token with new fresh token" do
+      Pleroma.Config.put(@oauth_config_path, false)
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      new_token = Repo.get_by(Token, token: response["access_token"])
+      refute new_token.refresh_token == token.refresh_token
+      assert new_token.scopes == auth.scopes
+      assert new_token.user_id == user.id
+      assert new_token.app_id == app.id
+    end
+
+    test "returns 400 if we try use access token" do
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(400)
+
+      assert %{"error" => "Invalid credentials"} == response
+    end
+
+    test "returns 400 if refresh_token invalid" do
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => "token.refresh_token",
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(400)
+
+      assert %{"error" => "Invalid credentials"} == response
+    end
+
+    test "issues a new token if token expired" do
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      change =
+        Ecto.Changeset.change(
+          token,
+          %{valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -86_400 * 30)}
+        )
+
+      {:ok, access_token} = Repo.update(change)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => access_token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      token = Repo.get_by(Token, token: response["access_token"])
+      assert token
+      assert token.scopes == auth.scopes
+      assert token.user_id == user.id
+      assert token.app_id == app.id
+    end
+  end
+
+  describe "POST /oauth/token - bad request" do
+    test "returns 500" do
+      response =
+        build_conn()
+        |> post("/oauth/token", %{})
+        |> json_response(500)
+
+      assert %{"error" => "Bad request"} == response
+    end
+  end
+
+  describe "POST /oauth/revoke - bad request" do
+    test "returns 500" do
+      response =
+        build_conn()
+        |> post("/oauth/revoke", %{})
+        |> json_response(500)
+
+      assert %{"error" => "Bad request"} == response
+    end
+  end
 end

test/web/twitter_api/twitter_api_controller_test.exs

@@ -1611,6 +1611,34 @@ test "it unlocks an account", %{conn: conn} do
 
       assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user})
     end
+
+    # Broken before the change to class="emoji" and non-<img/> in the DB
+    @tag :skip
+    test "it formats emojos", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> post("/api/account/update_profile.json", %{
+          "bio" => "I love our :moominmamma:​"
+        })
+
+      assert response = json_response(conn, 200)
+
+      assert %{
+               "description" => "I love our :moominmamma:",
+               "description_html" =>
+                 ~s{I love our <img class="emoji" alt="moominmamma" title="moominmamma" src="} <>
+                   _
+             } = response
+
+      conn =
+        conn
+        |> get("/api/users/show.json?user_id=#{user.nickname}")
+
+      assert response == json_response(conn, 200)
+    end
   end
 
   defp valid_user(_context) do

test/web/twitter_api/views/activity_view_test.exs

@@ -100,7 +100,7 @@ test "a create activity with a summary containing emoji" do
     expected = ":firefox: meow"
 
     expected_html =
-      "<img height=\"32px\" width=\"32px\" alt=\"firefox\" title=\"firefox\" src=\"http://localhost:4001/emoji/Firefox.gif\" /> meow"
+      "<img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"http://localhost:4001/emoji/Firefox.gif\" /> meow"
 
     assert result["summary"] == expected
     assert result["summary_html"] == expected_html
@@ -371,4 +371,14 @@ test "a peertube video" do
     assert length(result["attachments"]) == 1
     assert result["summary"] == "Friday Night"
   end
+
+  test "special characters are not escaped in text field for status created" do
+    text = "<3 is on the way"
+
+    {:ok, activity} = CommonAPI.post(insert(:user), %{"status" => text})
+
+    result = ActivityView.render("activity.json", activity: activity)
+
+    assert result["text"] == text
+  end
 end

test/web/twitter_api/views/user_view_test.exs

@@ -32,7 +32,7 @@ test "A user with an avatar object", %{user: user} do
 
   test "A user with emoji in username" do
     expected =
-      "<img height=\"32px\" width=\"32px\" alt=\"karjalanpiirakka\" title=\"karjalanpiirakka\" src=\"/file.png\" /> man"
+      "<img class=\"emoji\" alt=\"karjalanpiirakka\" title=\"karjalanpiirakka\" src=\"/file.png\" /> man"
 
     user =
       insert(:user, %{