From a2e03d4f3cfc7ccae98ae19db253aec8f1d3e9d0 Mon Sep 17 00:00:00 2001
From: Zachary Dunn <zdunn76@gmail.com>
Date: Mon, 18 Mar 2019 13:56:59 +0000
Subject: [PATCH 01/43] Initial attempt at updating return type

---
 lib/pleroma/user.ex                           | 33 ++++++++++++-------
 lib/pleroma/web/activity_pub/relay.ex         |  4 +--
 .../web/activity_pub/transmogrifier.ex        | 24 +++++++-------
 .../mastodon_api/mastodon_api_controller.ex   |  2 +-
 lib/pleroma/web/twitter_api/twitter_api.ex    |  2 +-
 5 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 1ce9882f6..6e8103c1c 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -411,7 +411,7 @@ defmodule Pleroma.User do
     Enum.map(
       followed_identifiers,
       fn followed_identifier ->
-        with %User{} = followed <- get_or_fetch(followed_identifier),
+        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
              {:ok, follower} <- maybe_direct_follow(follower, followed),
              {:ok, _} <- ActivityPub.follow(follower, followed) do
           followed
@@ -492,7 +492,14 @@ defmodule Pleroma.User do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
+    Cachex.fetch!(:user_cache, key, fn ->
+      user_result = get_or_fetch_by_nickname(nickname)
+
+      case user_result do
+        {:ok, user} -> {:commit, user}
+        {:error, error} -> {:ignore, error}
+      end
+    end)
   end
 
   def get_cached_by_nickname_or_id(nickname_or_id) do
@@ -529,7 +536,7 @@ defmodule Pleroma.User do
 
   def get_or_fetch_by_nickname(nickname) do
     with %User{} = user <- get_by_nickname(nickname) do
-      user
+      {:ok, user}
     else
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
@@ -538,9 +545,9 @@ defmodule Pleroma.User do
             {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
           end
 
-          user
+          {:ok, user}
         else
-          _e -> nil
+          _e -> {:error, "Error"}
         end
     end
   end
@@ -939,7 +946,7 @@ defmodule Pleroma.User do
     Enum.map(
       blocked_identifiers,
       fn blocked_identifier ->
-        with %User{} = blocked <- get_or_fetch(blocked_identifier),
+        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
              {:ok, blocker} <- block(blocker, blocked),
              {:ok, _} <- ActivityPub.block(blocker, blocked) do
           blocked
@@ -1157,17 +1164,19 @@ defmodule Pleroma.User do
     user = get_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
-      user
+      {:ok, user}
     else
       user = fetch_by_ap_id(ap_id)
 
-      if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
-        with %User{} = user do
+      with %User{} = user do
+        if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end
-      end
 
-      user
+        {:ok, user}
+      else
+        _ -> {:error, "Could not fetch by AP id"}
+      end
     end
   end
 
@@ -1209,7 +1218,7 @@ defmodule Pleroma.User do
   end
 
   def get_public_key_for_ap_id(ap_id) do
-    with %User{} = user <- get_or_fetch_by_ap_id(ap_id),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key_from_info(user.info) do
       {:ok, public_key}
     else
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index 01fef71b9..2bab9b03c 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def follow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.follow(local_user, target_user) do
       Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}")
       {:ok, activity}
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def unfollow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
       {:ok, activity}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 1247e4b61..a90b934e5 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -373,7 +373,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       |> fix_addressing
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
       params = %{
@@ -402,7 +402,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
       if not User.locked?(followed) do
         ActivityPub.accept(%{
@@ -425,7 +425,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -451,7 +451,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -475,7 +475,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -488,7 +488,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
@@ -543,7 +543,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     object_id = Utils.get_ap_id(object_id)
 
     with actor <- get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          :ok <- contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
@@ -562,7 +562,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = data
       ) do
     with actor <- get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
@@ -580,7 +580,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = _data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.unfollow(follower, followed, id, false) do
       User.unfollow(follower, followed)
       {:ok, activity}
@@ -599,7 +599,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked),
-         %User{} = blocker <- User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do
       User.unblock(blocker, blocked)
       {:ok, activity}
@@ -613,7 +613,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
       User.unfollow(blocker, blocked)
       User.block(blocker, blocked)
@@ -632,7 +632,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = data
       ) do
     with actor <- get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index e578f707e..cbf2a7ce1 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1450,7 +1450,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
               x,
               "id",
               case User.get_or_fetch(x["acct"]) do
-                %{id: id} -> id
+                {:ok, %User{} = %{id: id}} -> id
                 _ -> 0
               end
             )
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d57100491..feadf8670 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -312,7 +312,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   end
 
   def get_external_profile(for_user, uri) do
-    with %User{} = user <- User.get_or_fetch(uri) do
+    with {:ok, %User{} = user} <- User.get_or_fetch(uri) do
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
     else
       _e ->

From 5ba14c664b46faf633692cafa4290c0e6521d03c Mon Sep 17 00:00:00 2001
From: Zachary Dunn <zdunn76@gmail.com>
Date: Mon, 18 Mar 2019 14:05:10 +0000
Subject: [PATCH 02/43] Fix missing end brace

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index a90b934e5..546134f46 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -613,7 +613,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         {:ok, %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
       User.unfollow(blocker, blocked)
       User.block(blocker, blocked)

From af4338da0c26d992a5189fe940aa08ba69222e3b Mon Sep 17 00:00:00 2001
From: Zachary Dunn <zdunn76@gmail.com>
Date: Mon, 18 Mar 2019 14:52:24 +0000
Subject: [PATCH 03/43] Use better error message

---
 lib/pleroma/user.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 6e8103c1c..dcc03ae95 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -547,7 +547,7 @@ defmodule Pleroma.User do
 
           {:ok, user}
         else
-          _e -> {:error, "Error"}
+          e -> {:error, e}
         end
     end
   end

From e572786dad231a954c87ce54bb63e9be7fb396e9 Mon Sep 17 00:00:00 2001
From: Zachary Dunn <zdunn76@gmail.com>
Date: Mon, 18 Mar 2019 14:53:30 +0000
Subject: [PATCH 04/43] Run

---
 lib/pleroma/user.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index dcc03ae95..a29f836be 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -492,6 +492,7 @@ defmodule Pleroma.User do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
+
     Cachex.fetch!(:user_cache, key, fn ->
       user_result = get_or_fetch_by_nickname(nickname)
 

From 97b35e00b049c8f908484163b5ffdbcb55db7867 Mon Sep 17 00:00:00 2001
From: Zachary Dunn <zdunn76@gmail.com>
Date: Mon, 18 Mar 2019 14:59:52 +0000
Subject: [PATCH 05/43] Fix with expression always matching

---
 lib/pleroma/user.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index a29f836be..5e8dfc669 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1167,9 +1167,7 @@ defmodule Pleroma.User do
     if !is_nil(user) and !User.needs_update?(user) do
       {:ok, user}
     else
-      user = fetch_by_ap_id(ap_id)
-
-      with %User{} = user do
+      with %User{} = user <- fetch_by_ap_id(ap_id) do
         if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end

From c810fb81a489dc3faeb1f21092c89a3131188ac1 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 14:55:09 +0200
Subject: [PATCH 06/43] Basic SSH daemon.

---
 lib/pleroma/bbs/bbs.ex | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 lib/pleroma/bbs/bbs.ex

diff --git a/lib/pleroma/bbs/bbs.ex b/lib/pleroma/bbs/bbs.ex
new file mode 100644
index 000000000..486ab9183
--- /dev/null
+++ b/lib/pleroma/bbs/bbs.ex
@@ -0,0 +1,34 @@
+defmodule Pleroma.BBS do
+  def start_daemon do
+    :ok = :ssh.start()
+
+    options = [
+      system_dir: 'ssh_keys',
+      auth_method_kb_interactive_data: fn (_, user, _) -> {
+        'Welcome to Pleroma BBS',
+        'Hello #{user}',
+        'Password: ',
+        false }
+      end,
+      auth_methods: 'keyboard-interactive,password',
+      pwdfun: fn(user, password) -> true end,
+      shell: &start_prompt/1
+    ]
+    :ssh.daemon(13121, options)
+  end
+
+  def start_prompt(user) do
+    spawn(__MODULE__, :prompt, [user])
+  end
+
+  def prompt(user) do
+    IO.puts("Hey #{user}.\n")
+    IO.puts("Here's your timeline:\n")
+
+    user = Pleroma.User.get_cached_by_nickname(to_string(user))
+    Pleroma.Web.TwitterAPI.TwitterAPI.fetch_friend_statuses(user)
+    |> Enum.each(fn (status) ->
+      IO.puts("#{status["user"]["name"]} (#{status["user"]["screen_name"]}): #{status["text"]}")
+    end)
+  end
+end

From 17ab9fa45bf8a359746d30f4a91cb57a9c94f206 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 18:01:16 +0200
Subject: [PATCH 07/43] BBS: Some more functionality.

---
 lib/pleroma/bbs/authenticator.ex |  16 ++++
 lib/pleroma/bbs/bbs.ex           |  34 --------
 lib/pleroma/bbs/handler.ex       | 130 +++++++++++++++++++++++++++++++
 mix.exs                          |   5 +-
 mix.lock                         |   1 +
 5 files changed, 150 insertions(+), 36 deletions(-)
 create mode 100644 lib/pleroma/bbs/authenticator.ex
 delete mode 100644 lib/pleroma/bbs/bbs.ex
 create mode 100644 lib/pleroma/bbs/handler.ex

diff --git a/lib/pleroma/bbs/authenticator.ex b/lib/pleroma/bbs/authenticator.ex
new file mode 100644
index 000000000..a2c153720
--- /dev/null
+++ b/lib/pleroma/bbs/authenticator.ex
@@ -0,0 +1,16 @@
+defmodule Pleroma.BBS.Authenticator do
+  use Sshd.PasswordAuthenticator
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+
+  def authenticate(username, password) do
+    username = to_string(username)
+    password = to_string(password)
+
+    with %User{} = user <- User.get_by_nickname(username) do
+      Pbkdf2.checkpw(password, user.password_hash)
+    else
+      _e -> false
+    end
+  end
+end
diff --git a/lib/pleroma/bbs/bbs.ex b/lib/pleroma/bbs/bbs.ex
deleted file mode 100644
index 486ab9183..000000000
--- a/lib/pleroma/bbs/bbs.ex
+++ /dev/null
@@ -1,34 +0,0 @@
-defmodule Pleroma.BBS do
-  def start_daemon do
-    :ok = :ssh.start()
-
-    options = [
-      system_dir: 'ssh_keys',
-      auth_method_kb_interactive_data: fn (_, user, _) -> {
-        'Welcome to Pleroma BBS',
-        'Hello #{user}',
-        'Password: ',
-        false }
-      end,
-      auth_methods: 'keyboard-interactive,password',
-      pwdfun: fn(user, password) -> true end,
-      shell: &start_prompt/1
-    ]
-    :ssh.daemon(13121, options)
-  end
-
-  def start_prompt(user) do
-    spawn(__MODULE__, :prompt, [user])
-  end
-
-  def prompt(user) do
-    IO.puts("Hey #{user}.\n")
-    IO.puts("Here's your timeline:\n")
-
-    user = Pleroma.User.get_cached_by_nickname(to_string(user))
-    Pleroma.Web.TwitterAPI.TwitterAPI.fetch_friend_statuses(user)
-    |> Enum.each(fn (status) ->
-      IO.puts("#{status["user"]["name"]} (#{status["user"]["screen_name"]}): #{status["text"]}")
-    end)
-  end
-end
diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
new file mode 100644
index 000000000..010929ed2
--- /dev/null
+++ b/lib/pleroma/bbs/handler.ex
@@ -0,0 +1,130 @@
+defmodule Pleroma.BBS.Handler do
+  @moduledoc """
+  An example implementation of `Sshd.ShellHandler`, implementing a very simple
+  Read-Eval-Loop, that does nothing.
+  """
+  use Sshd.ShellHandler
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.ActivityPub.ActivityPub
+
+  def on_shell(username, _pubkey, _ip, _port) do
+    :ok = IO.puts "Welcome to #{Pleroma.Config.get([:instance, :name])}!"
+    user = Pleroma.User.get_by_nickname(to_string(username))
+    Logger.debug("#{inspect user}")
+    loop(run_state([user: user]))
+  end
+
+  def on_connect(username, ip, port, method) do
+    Logger.debug fn ->
+      """
+      Incoming SSH shell #{inspect self()} requested for #{username} from #{inspect ip}:#{inspect port} using #{inspect method}
+      """
+    end
+  end
+
+  def on_disconnect(username, ip, port) do
+    Logger.debug fn ->
+      "Disconnecting SSH shell for #{username} from #{inspect ip}:#{inspect port}"
+    end
+  end
+
+  defp loop(state) do
+    self_pid = self()
+    counter  = state.counter
+    prefix   = state.prefix
+    user     = state.user
+
+    input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end)
+    wait_input state, input
+  end
+
+  def puts_activity(activity) do
+    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    IO.puts("#{status.id} by #{status.account.display_name} (#{status.account.acct}):")
+    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts("")
+  end
+
+  def handle_command(state, "help") do
+    IO.puts("Available commands:")
+    IO.puts("help - This help")
+    IO.puts("home - Show the home timeline")
+    IO.puts("p <text> - Post the given text")
+    IO.puts("quit - Quit")
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "p " <> text) do
+    text = String.trim(text)
+
+    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+      IO.puts("Posted!")
+    else
+      _e -> IO.puts("Could not post...")
+    end
+    state
+  end
+
+  def handle_command(state, "home") do
+    user = state.user
+    params =
+      %{}
+      |> Map.put("type", ["Create", "Announce"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("muting_user", user)
+      |> Map.put("user", user)
+
+    activities =
+      [user.ap_id | user.following]
+      |> ActivityPub.fetch_activities(params)
+      |> ActivityPub.contain_timeline(user)
+      |> Enum.reverse()
+
+    Enum.each(activities, fn (activity) ->
+      puts_activity(activity)
+    end)
+
+    state
+  end
+
+  def handle_command(_state, command) do
+    IO.puts("Unknown command '#{command}'")
+  end
+
+  defp wait_input(state, input) do
+    receive do
+      {:input, ^input, "quit\n"} ->
+        IO.puts "Exiting..."
+
+      {:input, ^input, code} when is_binary(code) ->
+        code = String.trim(code)
+
+        handle_command(state, code)
+
+        loop(%{state | counter: state.counter + 1})
+
+      {:error, :interrupted} ->
+        IO.puts "Caught Ctrl+C..."
+        loop(%{state | counter: state.counter + 1})
+
+      {:input, ^input, msg} ->
+        :ok = Logger.warn "received unknown message: #{inspect msg}"
+        loop(%{state | counter: state.counter + 1})
+    end
+  end
+
+  defp run_state(opts) do
+    %{prefix: "pleroma", counter: 1, user: opts[:user]}
+  end
+
+  defp io_get(pid, prefix, counter, username) do
+    prompt = prompt(prefix, counter, username)
+    send pid, {:input, self(), IO.gets(:stdio, prompt)}
+  end
+
+  defp prompt(prefix, counter, username) do
+    prompt = "#{username}@#{prefix}:#{counter}>"
+    prompt <> " "
+  end
+end
diff --git a/mix.exs b/mix.exs
index 333f21a91..58b0db9a1 100644
--- a/mix.exs
+++ b/mix.exs
@@ -41,7 +41,7 @@ defmodule Pleroma.Mixfile do
   def application do
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin],
+      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -94,7 +94,8 @@ defmodule Pleroma.Mixfile do
       {:auto_linker,
        git: "https://git.pleroma.social/pleroma/auto_linker.git",
        ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"},
-      {:pleroma_job_queue, "~> 0.2.0"}
+      {:pleroma_job_queue, "~> 0.2.0"},
+      {:esshd, "~> 0.1.0"}
     ]
   end
 
diff --git a/mix.lock b/mix.lock
index f401258e9..a8b06b433 100644
--- a/mix.lock
+++ b/mix.lock
@@ -18,6 +18,7 @@
   "earmark": {:hex, :earmark, "1.3.0", "17f0c38eaafb4800f746b457313af4b2442a8c2405b49c645768680f900be603", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},

From 3fc4ea45df240c84135727a239d6bf20fd016a3f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 18:14:51 +0200
Subject: [PATCH 08/43] BBS: Add documentation.

---
 .gitignore     |  1 +
 docs/config.md | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/.gitignore b/.gitignore
index 04c61ede7..e8052d817 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@
 /test/tmp/
 /doc
 /instance
+/priv/ssh_keys
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*
diff --git a/docs/config.md b/docs/config.md
index 3624e295b..4fb4f530b 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -356,3 +356,21 @@ Pleroma account will be created with the same name as the LDAP user name.
 
 * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator
 * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication
+
+## BBS / SSH access
+
+To enable simple command line interface accessible over ssh, add a setting like this to your configuration file:
+
+```exs
+app_dir = File.cwd!
+priv_dir = Path.join([app_dir, "priv/ssh_keys"])
+
+config :esshd,
+  enabled: true,
+  priv_dir: priv_dir,
+  handler: "Pleroma.BBS.Handler",
+  port: 10_022,
+  password_authenticator: "Pleroma.BBS.Authenticator"
+```
+
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`

From 10fdc080a0048a4776abb4bd1b5aa22d8c65e2da Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 20:35:10 +0200
Subject: [PATCH 09/43] BBS: Tests and formatting.

---
 config/config.exs          |  3 +++
 lib/pleroma/bbs/handler.ex | 40 +++++++++++++++++++++-----------------
 test/bbs/handler_test.exs  | 30 ++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 18 deletions(-)
 create mode 100644 test/bbs/handler_test.exs

diff --git a/config/config.exs b/config/config.exs
index 0df38d75a..33724346e 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -378,6 +378,9 @@ config :pleroma, :ldap,
   base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
   uid: System.get_env("LDAP_UID") || "cn"
 
+config :esshd,
+  enabled: false
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index 010929ed2..d999dcf76 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -8,34 +8,36 @@ defmodule Pleroma.BBS.Handler do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   def on_shell(username, _pubkey, _ip, _port) do
-    :ok = IO.puts "Welcome to #{Pleroma.Config.get([:instance, :name])}!"
+    :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
     user = Pleroma.User.get_by_nickname(to_string(username))
-    Logger.debug("#{inspect user}")
-    loop(run_state([user: user]))
+    Logger.debug("#{inspect(user)}")
+    loop(run_state(user: user))
   end
 
   def on_connect(username, ip, port, method) do
-    Logger.debug fn ->
+    Logger.debug(fn ->
       """
-      Incoming SSH shell #{inspect self()} requested for #{username} from #{inspect ip}:#{inspect port} using #{inspect method}
+      Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{
+        inspect(port)
+      } using #{inspect(method)}
       """
-    end
+    end)
   end
 
   def on_disconnect(username, ip, port) do
-    Logger.debug fn ->
-      "Disconnecting SSH shell for #{username} from #{inspect ip}:#{inspect port}"
-    end
+    Logger.debug(fn ->
+      "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}"
+    end)
   end
 
   defp loop(state) do
     self_pid = self()
-    counter  = state.counter
-    prefix   = state.prefix
-    user     = state.user
+    counter = state.counter
+    prefix = state.prefix
+    user = state.user
 
     input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end)
-    wait_input state, input
+    wait_input(state, input)
   end
 
   def puts_activity(activity) do
@@ -63,11 +65,13 @@ defmodule Pleroma.BBS.Handler do
     else
       _e -> IO.puts("Could not post...")
     end
+
     state
   end
 
   def handle_command(state, "home") do
     user = state.user
+
     params =
       %{}
       |> Map.put("type", ["Create", "Announce"])
@@ -81,7 +85,7 @@ defmodule Pleroma.BBS.Handler do
       |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
-    Enum.each(activities, fn (activity) ->
+    Enum.each(activities, fn activity ->
       puts_activity(activity)
     end)
 
@@ -95,7 +99,7 @@ defmodule Pleroma.BBS.Handler do
   defp wait_input(state, input) do
     receive do
       {:input, ^input, "quit\n"} ->
-        IO.puts "Exiting..."
+        IO.puts("Exiting...")
 
       {:input, ^input, code} when is_binary(code) ->
         code = String.trim(code)
@@ -105,11 +109,11 @@ defmodule Pleroma.BBS.Handler do
         loop(%{state | counter: state.counter + 1})
 
       {:error, :interrupted} ->
-        IO.puts "Caught Ctrl+C..."
+        IO.puts("Caught Ctrl+C...")
         loop(%{state | counter: state.counter + 1})
 
       {:input, ^input, msg} ->
-        :ok = Logger.warn "received unknown message: #{inspect msg}"
+        :ok = Logger.warn("received unknown message: #{inspect(msg)}")
         loop(%{state | counter: state.counter + 1})
     end
   end
@@ -120,7 +124,7 @@ defmodule Pleroma.BBS.Handler do
 
   defp io_get(pid, prefix, counter, username) do
     prompt = prompt(prefix, counter, username)
-    send pid, {:input, self(), IO.gets(:stdio, prompt)}
+    send(pid, {:input, self(), IO.gets(:stdio, prompt)})
   end
 
   defp prompt(prefix, counter, username) do
diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs
new file mode 100644
index 000000000..ee5f194bb
--- /dev/null
+++ b/test/bbs/handler_test.exs
@@ -0,0 +1,30 @@
+defmodule Pleroma.BBS.HandlerTest do
+  use Pleroma.DataCase
+  alias Pleroma.BBS.Handler
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.User
+
+  import ExUnit.CaptureIO
+  import Pleroma.Factory
+
+  test "getting the home timeline" do
+    user = insert(:user)
+    followed = insert(:user)
+
+    {:ok, user} = User.follow(user, followed)
+
+    {:ok, _first} = CommonAPI.post(user, %{"status" => "hey"})
+    {:ok, _second} = CommonAPI.post(followed, %{"status" => "hello"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "home")
+      end)
+
+    assert output =~ user.nickname
+    assert output =~ followed.nickname
+
+    assert output =~ "hey"
+    assert output =~ "hello"
+  end
+end

From e3bf6655ba412268e4f5fee645609c9738e453ef Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 21:14:21 +0200
Subject: [PATCH 10/43] Add replying.

---
 lib/pleroma/bbs/handler.ex | 21 ++++++++++++++--
 test/bbs/handler_test.exs  | 50 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index d999dcf76..7749eb3af 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.BBS.Handler do
   use Sshd.ShellHandler
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Activity
 
   def on_shell(username, _pubkey, _ip, _port) do
     :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
@@ -42,7 +43,7 @@ defmodule Pleroma.BBS.Handler do
 
   def puts_activity(activity) do
     status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
-    IO.puts("#{status.id} by #{status.account.display_name} (#{status.account.acct}):")
+    IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
     IO.puts(HtmlSanitizeEx.strip_tags(status.content))
     IO.puts("")
   end
@@ -52,11 +53,27 @@ defmodule Pleroma.BBS.Handler do
     IO.puts("help - This help")
     IO.puts("home - Show the home timeline")
     IO.puts("p <text> - Post the given text")
+    IO.puts("r <id> <text> - Reply to the post with the given id")
     IO.puts("quit - Quit")
 
     state
   end
 
+  def handle_command(%{user: user} = state, "r " <> text) do
+    text = String.trim(text)
+    [activity_id, rest] = String.split(text, " ", parts: 2)
+
+    with %Activity{} <- Activity.get_by_id(activity_id),
+         {:ok, _activity} <-
+           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+      IO.puts("Replied!")
+    else
+      _e -> IO.puts("Could not reply...")
+    end
+
+    state
+  end
+
   def handle_command(%{user: user} = state, "p " <> text) do
     text = String.trim(text)
 
@@ -104,7 +121,7 @@ defmodule Pleroma.BBS.Handler do
       {:input, ^input, code} when is_binary(code) ->
         code = String.trim(code)
 
-        handle_command(state, code)
+        state = handle_command(state, code)
 
         loop(%{state | counter: state.counter + 1})
 
diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs
index ee5f194bb..a22c6d64d 100644
--- a/test/bbs/handler_test.exs
+++ b/test/bbs/handler_test.exs
@@ -3,9 +3,12 @@ defmodule Pleroma.BBS.HandlerTest do
   alias Pleroma.BBS.Handler
   alias Pleroma.Web.CommonAPI
   alias Pleroma.User
+  alias Pleroma.Repo
+  alias Pleroma.Activity
 
   import ExUnit.CaptureIO
   import Pleroma.Factory
+  import Ecto.Query
 
   test "getting the home timeline" do
     user = insert(:user)
@@ -27,4 +30,51 @@ defmodule Pleroma.BBS.HandlerTest do
     assert output =~ "hey"
     assert output =~ "hello"
   end
+
+  test "posting" do
+    user = insert(:user)
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "p this is a test post")
+      end)
+
+    assert output =~ "Posted"
+
+    activity =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create")
+        )
+      )
+
+    assert activity.actor == user.ap_id
+    assert activity.data["object"]["content"] == "this is a test post"
+  end
+
+  test "replying" do
+    user = insert(:user)
+    another_user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post(another_user, %{"status" => "this is a test post"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "r #{activity.id} this is a reply")
+      end)
+
+    assert output =~ "Replied"
+
+    reply =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create"),
+          where: a.actor == ^user.ap_id
+        )
+      )
+
+    assert reply.actor == user.ap_id
+    assert reply.data["object"]["content"] == "this is a reply"
+    assert reply.data["object"]["inReplyTo"] == activity.data["object"]["id"]
+  end
 end

From 629ad1766ce5da434bf095f6baa81a460334e1b2 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 31 Mar 2019 21:53:17 +0200
Subject: [PATCH 11/43] BBS: Some fixes.

---
 lib/pleroma/bbs/handler.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index 7749eb3af..1ebba77d2 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -91,7 +91,7 @@ defmodule Pleroma.BBS.Handler do
 
     params =
       %{}
-      |> Map.put("type", ["Create", "Announce"])
+      |> Map.put("type", ["Create"])
       |> Map.put("blocking_user", user)
       |> Map.put("muting_user", user)
       |> Map.put("user", user)
@@ -100,7 +100,6 @@ defmodule Pleroma.BBS.Handler do
       [user.ap_id | user.following]
       |> ActivityPub.fetch_activities(params)
       |> ActivityPub.contain_timeline(user)
-      |> Enum.reverse()
 
     Enum.each(activities, fn activity ->
       puts_activity(activity)
@@ -109,8 +108,9 @@ defmodule Pleroma.BBS.Handler do
     state
   end
 
-  def handle_command(_state, command) do
+  def handle_command(state, command) do
     IO.puts("Unknown command '#{command}'")
+    state
   end
 
   defp wait_input(state, input) do

From b421dd3dd23fd2df6086bcad1d46a4262f259459 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 12 Apr 2019 07:32:46 +0200
Subject: [PATCH 12/43] installation/download-mastofe-build.sh: Add mastofe
 CI-artifacts download

[ci skip]
---
 installation/download-mastofe-build.sh | 42 ++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100755 installation/download-mastofe-build.sh

diff --git a/installation/download-mastofe-build.sh b/installation/download-mastofe-build.sh
new file mode 100755
index 000000000..ab31a7ee7
--- /dev/null
+++ b/installation/download-mastofe-build.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+project_id="74"
+project_branch="rebase/glitch-soc"
+static_dir="instance/static"
+# For bundling:
+# project_branch="pleroma"
+# static_dir="priv/static"
+
+if [[ ! -d "${static_dir}" ]]
+then
+	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
+	exit 1
+fi
+
+last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+
+echo "branch:${project_branch}"
+echo "Last-Modified:${last_modified}"
+
+artifact="mastofe.zip"
+
+if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+then
+	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	then
+		echo "MastoFE is up-to-date, exiting…"
+		exit 0
+	fi
+fi
+
+curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+
+# TODO: Update the emoji as well
+rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit
+unzip -q "${artifact}" || exit
+
+cp public/assets/sw.js "${static_dir}/sw.js" || exit
+cp -r public/packs "${static_dir}/packs" || exit
+
+echo "${last_modified}" > mastofe.timestamp
+rm -fr public
+rm -i "${artifact}"

From 378b964d8eba2f216fa6f670c2de42ecee0c6277 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 12 Apr 2019 07:39:49 +0200
Subject: [PATCH 13/43] installation/download-mastofe-build.sh: Add copyright
 header

[ci skip]
---
 installation/download-mastofe-build.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/installation/download-mastofe-build.sh b/installation/download-mastofe-build.sh
index ab31a7ee7..7e293867d 100755
--- a/installation/download-mastofe-build.sh
+++ b/installation/download-mastofe-build.sh
@@ -1,4 +1,7 @@
 #!/bin/sh
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
 project_id="74"
 project_branch="rebase/glitch-soc"
 static_dir="instance/static"

From 963d5774af7efb57fa306b3ac164049f8958a72c Mon Sep 17 00:00:00 2001
From: Sachin Joshi <satchin.joshi@gmail.com>
Date: Wed, 24 Apr 2019 07:06:17 +0545
Subject: [PATCH 14/43] fix the status notification with special char

---
 lib/pleroma/web/twitter_api/views/activity_view.ex |  2 +-
 test/web/twitter_api/views/activity_view_test.exs  | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index c64152da8..1007a2a48 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -289,7 +289,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
       "statusnet_html" => html,
-      "text" => text,
+      "text" => HtmlEntities.decode(text),
       "is_local" => activity.local,
       "is_post_verb" => true,
       "created_at" => created_at,
diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs
index d84ab7420..85815ba7e 100644
--- a/test/web/twitter_api/views/activity_view_test.exs
+++ b/test/web/twitter_api/views/activity_view_test.exs
@@ -371,4 +371,14 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
     assert length(result["attachments"]) == 1
     assert result["summary"] == "Friday Night"
   end
+
+  test "special characters are not escaped in text field for status created" do
+    text = "<3 is on the way"
+
+    {:ok, activity} = CommonAPI.post(insert(:user), %{"status" => text})
+
+    result = ActivityView.render("activity.json", activity: activity)
+
+    assert result["text"] == text
+  end
 end

From b0951a884914a06d283b55ed65c6322e6e4d27ae Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 29 Apr 2019 18:15:30 +0200
Subject: [PATCH 15/43] WebPush: Use Object.normalize, rewrite tests so they
 test reality.

---
 lib/pleroma/web/push/impl.ex | 25 +++++++------
 test/web/push/impl_test.exs  | 71 ++++++++++++++++++++++--------------
 2 files changed, 57 insertions(+), 39 deletions(-)

diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index 2233480c5..35d3ff07c 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -21,8 +21,10 @@ defmodule Pleroma.Web.Push.Impl do
   @doc "Performs sending notifications for user subscriptions"
   @spec perform(Notification.t()) :: list(any) | :error
   def perform(
-        %{activity: %{data: %{"type" => activity_type}, id: activity_id}, user_id: user_id} =
-          notif
+        %{
+          activity: %{data: %{"type" => activity_type}, id: activity_id} = activity,
+          user_id: user_id
+        } = notif
       )
       when activity_type in @types do
     actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
@@ -30,13 +32,14 @@ defmodule Pleroma.Web.Push.Impl do
     type = Activity.mastodon_notification_type(notif.activity)
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
+    object = Object.normalize(activity)
 
     for subscription <- fetch_subsriptions(user_id),
         get_in(subscription.data, ["alerts", type]) do
       %{
         title: format_title(notif),
         access_token: subscription.token.token,
-        body: format_body(notif, actor),
+        body: format_body(notif, actor, object),
         notification_id: notif.id,
         notification_type: type,
         icon: avatar_url,
@@ -95,25 +98,25 @@ defmodule Pleroma.Web.Push.Impl do
   end
 
   def format_body(
-        %{activity: %{data: %{"type" => "Create", "object" => %{"content" => content}}}},
-        actor
+        %{activity: %{data: %{"type" => "Create"}}},
+        actor,
+        %{data: %{"content" => content}}
       ) do
     "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
-        %{activity: %{data: %{"type" => "Announce", "object" => activity_id}}},
-        actor
+        %{activity: %{data: %{"type" => "Announce"}}},
+        actor,
+        %{data: %{"content" => content}}
       ) do
-    %Activity{data: %{"object" => %{"id" => object_id}}} = Activity.get_by_ap_id(activity_id)
-    %Object{data: %{"content" => content}} = Object.get_by_ap_id(object_id)
-
     "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
         %{activity: %{data: %{"type" => type}}},
-        actor
+        actor,
+        _object
       )
       when type in ["Follow", "Like"] do
     case type do
diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs
index 49b2a9203..1e948086a 100644
--- a/test/web/push/impl_test.exs
+++ b/test/web/push/impl_test.exs
@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.Push.ImplTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Push.Impl
   alias Pleroma.Web.Push.Subscription
 
@@ -52,16 +54,12 @@ defmodule Pleroma.Web.Push.ImplTest do
       data: %{alerts: %{"follow" => true, "mention" => false}}
     )
 
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "<Lorem ipsum dolor sit amet."})
+
     notif =
       insert(:notification,
         user: user,
-        activity: %Pleroma.Activity{
-          data: %{
-            "type" => "Create",
-            "actor" => user.ap_id,
-            "object" => %{"content" => "<Lorem ipsum dolor sit amet."}
-          }
-        }
+        activity: activity
       )
 
     assert Impl.perform(notif) == [:ok, :ok]
@@ -100,48 +98,65 @@ defmodule Pleroma.Web.Push.ImplTest do
   end
 
   test "renders body for create activity" do
+    user = insert(:user, nickname: "Bob")
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" =>
+          "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
+      })
+
+    object = Object.normalize(activity)
+
     assert Impl.format_body(
              %{
-               activity: %{
-                 data: %{
-                   "type" => "Create",
-                   "object" => %{
-                     "content" =>
-                       "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
-                   }
-                 }
-               }
+               activity: activity
              },
-             %{nickname: "Bob"}
+             user,
+             object
            ) ==
              "@Bob: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
   end
 
   test "renders body for follow activity" do
-    assert Impl.format_body(%{activity: %{data: %{"type" => "Follow"}}}, %{nickname: "Bob"}) ==
+    user = insert(:user, nickname: "Bob")
+    other_user = insert(:user)
+    {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
+    object = Object.normalize(activity)
+
+    assert Impl.format_body(%{activity: activity}, user, object) ==
              "@Bob has followed you"
   end
 
   test "renders body for announce activity" do
     user = insert(:user)
 
-    note =
-      insert(:note, %{
-        data: %{
-          "content" =>
-            "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
-        }
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" =>
+          "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
       })
 
-    note_activity = insert(:note_activity, %{note: note})
-    announce_activity = insert(:announce_activity, %{user: user, note_activity: note_activity})
+    {:ok, announce_activity, _} = CommonAPI.repeat(activity.id, user)
+    object = Object.normalize(activity)
 
-    assert Impl.format_body(%{activity: announce_activity}, user) ==
+    assert Impl.format_body(%{activity: announce_activity}, user, object) ==
              "@#{user.nickname} repeated: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
   end
 
   test "renders body for like activity" do
-    assert Impl.format_body(%{activity: %{data: %{"type" => "Like"}}}, %{nickname: "Bob"}) ==
+    user = insert(:user, nickname: "Bob")
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" =>
+          "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
+      })
+
+    {:ok, activity, _} = CommonAPI.favorite(activity.id, user)
+    object = Object.normalize(activity)
+
+    assert Impl.format_body(%{activity: activity}, user, object) ==
              "@Bob has favorited your post"
   end
 end

From f8f6af4cc1e9389023c04dba0984f523704f11b1 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 29 Apr 2019 21:25:30 +0300
Subject: [PATCH 16/43] Fix MigrateOldBookmarks migration crashing on deleted
 activities

---
 priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
index ebe69696e..134b7c6f7 100644
--- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
+++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs
@@ -18,7 +18,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do
     |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} ->
       Enum.each(bookmarks, fn ap_id ->
         activity = Activity.get_create_by_object_ap_id(ap_id)
-        {:ok, _} = Bookmark.create(user_id, activity.id)
+	unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id)
       end)
     end)
 

From ce4825c1dc8a2b7ec2712170f45cde0ae14b46cf Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 30 Apr 2019 20:21:28 +0300
Subject: [PATCH 17/43] Do not normalize objects in stream_out unless the
 activity type is Create

Saves quite a bit of time with delete activities because they would
always query the db
---
 lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 604ffae7b..483a2153f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -168,7 +168,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce", "Delete"] do
-      object = Object.normalize(activity)
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -180,6 +179,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         end
 
         if activity.data["type"] in ["Create"] do
+          object = Object.normalize(activity)
+
           object.data
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)

From 85fa2fbce4ee315a15b517fae4bc9b5474d1db5a Mon Sep 17 00:00:00 2001
From: Sachin Joshi <satchin.joshi@gmail.com>
Date: Wed, 1 May 2019 01:37:17 +0545
Subject: [PATCH 18/43] add scrubber for html special char

---
 lib/pleroma/html.ex                           | 27 +++++++++++++++----
 .../web/twitter_api/views/activity_view.ex    |  2 +-
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index cf6c0ee0a..eb33d12d9 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -28,12 +28,18 @@ defmodule Pleroma.HTML do
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+  def get_cached_scrubbed_html_for_activity(
+        content,
+        scrubbers,
+        activity,
+        key \\ "",
+        callback \\ fn x -> x end
+      ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
-      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
+      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end
 
@@ -42,16 +48,27 @@ defmodule Pleroma.HTML do
       content,
       HtmlSanitizeEx.Scrubber.StripTags,
       activity,
-      key
+      key,
+      &HtmlEntities.decode/1
     )
   end
 
   def ensure_scrubbed_html(
         content,
         scrubbers,
-        false = _fake
+        fake,
+        callback
       ) do
-    {:commit, filter_tags(content, scrubbers)}
+    content =
+      content
+      |> filter_tags(scrubbers)
+      |> callback.()
+
+    if fake do
+      {:ignore, content}
+    else
+      {:commit, content}
+    end
   end
 
   def ensure_scrubbed_html(
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 1007a2a48..c64152da8 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -289,7 +289,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
       "statusnet_html" => html,
-      "text" => HtmlEntities.decode(text),
+      "text" => text,
       "is_local" => activity.local,
       "is_post_verb" => true,
       "created_at" => created_at,

From 2982061dfa7533893eb4a3cca2977e19bd962953 Mon Sep 17 00:00:00 2001
From: xse <xse@riseup.net>
Date: Wed, 1 May 2019 01:26:14 +0200
Subject: [PATCH 19/43] Fix syntax highlighting + clarify
 :frontend_configuration

---
 docs/config.md | 40 ++++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/docs/config.md b/docs/config.md
index 7e31e6fb7..ad55d44a7 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -37,7 +37,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 An example for Sendgrid adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,
   api_key: "YOUR_API_KEY"
@@ -45,7 +45,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 
 An example for SMTP adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.SMTP,
   relay: "smtp.gmail.com",
@@ -109,7 +109,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
 
 An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
-```
+```elixir
 config :logger,
   backends: [{ExSyslogger, :ex_syslogger}]
 
@@ -118,7 +118,7 @@ config :logger, :ex_syslogger,
 ```
 
 Another example, keeping console output and adding the pid to syslog output:
-```
+```elixir
 config :logger,
   backends: [:console, {ExSyslogger, :ex_syslogger}]
 
@@ -130,7 +130,7 @@ config :logger, :ex_syslogger,
 See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 An example of logging info to local syslog, but warn to a Slack channel:
-```
+```elixir
 config :logger,
   backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ],
   level: :info
@@ -156,14 +156,30 @@ Frontends can access these settings at `/api/pleroma/frontend_configurations`
 
 To add your own configuration for PleromaFE, use it like this:
 
-`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}`
+```elixir
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "pleroma-dark",
+    # ... see /priv/static/static/config.json for the available keys.
+},
+  masto_fe: %{
+    showInstanceSpecificPanel: true
+  }
+```
 
-These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys.
+These settings **need to be complete**, they will override the defaults.
+
+NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. 
 
 ## :fe
 __THIS IS DEPRECATED__
 
-If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`.
+If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
+Please **set this option to false** in your config like this: 
+
+```elixir
+config :pleroma, :fe, false
+```
 
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
@@ -274,7 +290,7 @@ their ActivityPub ID.
 
 An example:
 
-```exs
+```elixir
 config :pleroma, :mrf_user_allowlist,
   "example.org": ["https://example.org/users/admin"]
 ```
@@ -303,7 +319,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end
 
 Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example:
 
-```exs
+```elixir
 config :pleroma, :admin_token, "somerandomtoken"
 ```
 
@@ -387,7 +403,7 @@ Configuration for the `auto_linker` library:
 
 Example:
 
-```exs
+```elixir
 config :auto_linker,
   opts: [
     scheme: true,
@@ -460,7 +476,7 @@ Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you ha
 Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`,
 per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables:
 
-```
+```elixir
 # Twitter
 config :ueberauth, Ueberauth.Strategy.Twitter.OAuth,
   consumer_key: System.get_env("TWITTER_CONSUMER_KEY"),

From 04fd7cf81729c5ccec0b1f857acb96fc562fbdb0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 1 May 2019 15:18:12 +0700
Subject: [PATCH 20/43] Update `auto_linker`

---
 mix.exs  | 2 +-
 mix.lock | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index efaa06a1c..9ded9931c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -102,7 +102,7 @@ defmodule Pleroma.Mixfile do
       {:ueberauth, "~> 0.4"},
       {:auto_linker,
        git: "https://git.pleroma.social/pleroma/auto_linker.git",
-       ref: "90613b4bae875a3610c275b7056b61ffdd53210d"},
+       ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},
diff --git a/mix.lock b/mix.lock
index 979d599b4..08221eadc 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,6 +1,6 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"},
-  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "90613b4bae875a3610c275b7056b61ffdd53210d", [ref: "90613b4bae875a3610c275b7056b61ffdd53210d"]},
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},

From f11e7037c21d611cddd7f2eab64ebfc39630a078 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Wed, 1 May 2019 16:09:53 +0700
Subject: [PATCH 21/43] test fixes

---
 lib/mix/tasks/pleroma/user.ex                 |  2 +-
 lib/pleroma/user.ex                           | 18 +++++++-------
 .../activity_pub/activity_pub_controller.ex   |  2 +-
 .../web/activity_pub/transmogrifier.ex        |  2 +-
 .../mastodon_api/mastodon_api_controller.ex   |  2 +-
 test/formatter_test.exs                       |  2 +-
 test/user_test.exs                            | 24 +++++++++----------
 test/web/activity_pub/transmogrifier_test.exs |  2 +-
 8 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index b396ff0de..9e2523b18 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -126,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
-    unless not proceed? do
+    if proceed? do
       Common.start_pleroma()
 
       params = %{
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 755d8f773..1c62f238e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -513,7 +513,7 @@ defmodule Pleroma.User do
 
       case user_result do
         {:ok, user} -> {:commit, user}
-        {:error, error} -> {:ignore, error}
+        {:error, _error} -> {:ignore, nil}
       end
     end)
   end
@@ -563,7 +563,7 @@ defmodule Pleroma.User do
 
           {:ok, user}
         else
-          e -> {:error, e}
+          _e -> {:error, "not found " <> nickname}
         end
     end
   end
@@ -1210,11 +1210,11 @@ defmodule Pleroma.User do
 
     case ap_try do
       {:ok, user} ->
-        user
+        {:ok, user}
 
       _ ->
         case OStatus.make_user(ap_id) do
-          {:ok, user} -> user
+          {:ok, user} -> {:ok, user}
           _ -> {:error, "Could not fetch by AP id"}
         end
     end
@@ -1229,15 +1229,15 @@ defmodule Pleroma.User do
       # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
       should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
 
+      resp = fetch_by_ap_id(ap_id)
+
       if should_fetch_initial do
-        with {:ok, %User{} = user} = fetch_by_ap_id(ap_id) do
+        with {:ok, %User{} = user} = resp do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end
-
-        {:ok, user}
-      else
-        _ -> {:error, "Could not fetch by AP id"}
       end
+
+      resp
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 0b80566bf..c967ab7a9 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -155,7 +155,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = recipient <- User.get_cached_by_nickname(nickname),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(params["actor"]),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]),
          true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 80317171d..b774c2afa 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -126,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_implicit_addressing(object, _), do: object
 
   def fix_addressing(object) do
-    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
     followers_collection = User.ap_followers(user)
 
     object
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 727e1c310..ed585098a 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1653,7 +1653,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
               x,
               "id",
               case User.get_or_fetch(x["acct"]) do
-                {:ok, %User{} = %{id: id}} -> id
+                {:ok, %User{id: id}} -> id
                 _ -> 0
               end
             )
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index 97eb2f583..fdaf29742 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -147,7 +147,7 @@ defmodule Pleroma.FormatterTest do
     end
 
     test "gives a replacement for user links when the user is using Osada" do
-      mike = User.get_or_fetch("mike@osada.macgirvin.com")
+      {:ok, mike} = User.get_or_fetch("mike@osada.macgirvin.com")
 
       text = "@mike@osada.macgirvin.com test"
 
diff --git a/test/user_test.exs b/test/user_test.exs
index 7be47e5fb..67266cb7a 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -362,7 +362,7 @@ defmodule Pleroma.UserTest do
   describe "get_or_fetch/1" do
     test "gets an existing user by nickname" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch(user.nickname)
 
       assert user == fetched_user
     end
@@ -379,7 +379,7 @@ defmodule Pleroma.UserTest do
           info: %{}
         )
 
-      fetched_user = User.get_or_fetch(ap_id)
+      {:ok, fetched_user} = User.get_or_fetch(ap_id)
       freshed_user = refresh_record(user)
       assert freshed_user == fetched_user
     end
@@ -388,14 +388,14 @@ defmodule Pleroma.UserTest do
   describe "fetching a user from nickname or trying to build one" do
     test "gets an existing user" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch_by_nickname(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(user.nickname)
 
       assert user == fetched_user
     end
 
     test "gets an existing user, case insensitive" do
       user = insert(:user, nickname: "nick")
-      fetched_user = User.get_or_fetch_by_nickname("NICK")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("NICK")
 
       assert user == fetched_user
     end
@@ -403,7 +403,7 @@ defmodule Pleroma.UserTest do
     test "gets an existing user by fully qualified nickname" do
       user = insert(:user)
 
-      fetched_user =
+      {:ok, fetched_user} =
         User.get_or_fetch_by_nickname(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
       assert user == fetched_user
@@ -413,24 +413,24 @@ defmodule Pleroma.UserTest do
       user = insert(:user, nickname: "nick")
       casing_altered_fqn = String.upcase(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
-      fetched_user = User.get_or_fetch_by_nickname(casing_altered_fqn)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(casing_altered_fqn)
 
       assert user == fetched_user
     end
 
     test "fetches an external user via ostatus if no user exists" do
-      fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
       assert fetched_user.nickname == "shp@social.heldscal.la"
     end
 
     test "returns nil if no user could be fetched" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
-      assert fetched_user == nil
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
+      assert fetched_user == "not found nonexistant@social.heldscal.la"
     end
 
     test "returns nil for nonexistant local user" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant")
-      assert fetched_user == nil
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant")
+      assert fetched_user == "not found nonexistant"
     end
 
     test "updates an existing user, if stale" do
@@ -448,7 +448,7 @@ defmodule Pleroma.UserTest do
 
       assert orig_user.last_refreshed_at == a_week_ago
 
-      user = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
+      {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
       assert user.info.source_data["endpoints"]
 
       refute user.last_refreshed_at == orig_user.last_refreshed_at
diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs
index 78429c7c6..c24b50f8c 100644
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@ -219,7 +219,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       Pleroma.Config.put([:user, :deny_follow_blocked], true)
 
       user = insert(:user)
-      target = User.get_or_fetch("http://mastodon.example.org/users/admin")
+      {:ok, target} = User.get_or_fetch("http://mastodon.example.org/users/admin")
 
       {:ok, user} = User.block(user, target)
 

From 51e26f14f7fc342c23fe1fe643a1cf444ef9392b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 1 May 2019 13:52:44 +0300
Subject: [PATCH 22/43] Remove redundant ensure_scrubbed_html

It is never used as handling for fake and non-fake activities was merged
into one function above it
---
 lib/pleroma/html.ex | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index eb33d12d9..726c370ad 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -71,14 +71,6 @@ defmodule Pleroma.HTML do
     end
   end
 
-  def ensure_scrubbed_html(
-        content,
-        scrubbers,
-        true = _fake
-      ) do
-    {:ignore, filter_tags(content, scrubbers)}
-  end
-
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
     generate_scrubber_signature([scrubber])
   end

From c854bff8f528b4f1560707ac3aa74f4a37044f52 Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Wed, 1 May 2019 13:28:04 +0000
Subject: [PATCH 23/43] Refactored Pleroma.Web.Auth.Authenticator

---
 lib/pleroma/web/auth/authenticator.ex         | 26 ++++++++++++
 lib/pleroma/web/auth/ldap_authenticator.ex    | 41 ++++++++----------
 lib/pleroma/web/auth/pleroma_authenticator.ex | 15 +++----
 test/media_proxy_test.exs                     | 14 +++----
 test/web/auth/authenticator_test.exs          | 42 +++++++++++++++++++
 5 files changed, 97 insertions(+), 41 deletions(-)
 create mode 100644 test/web/auth/authenticator_test.exs

diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex
index b02f595dc..d4e0ffa80 100644
--- a/lib/pleroma/web/auth/authenticator.ex
+++ b/lib/pleroma/web/auth/authenticator.ex
@@ -42,4 +42,30 @@ defmodule Pleroma.Web.Auth.Authenticator do
     implementation().oauth_consumer_template() ||
       Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html")
   end
+
+  @doc "Gets user by nickname or email for auth."
+  @spec fetch_user(String.t()) :: User.t() | nil
+  def fetch_user(name) do
+    User.get_by_nickname_or_email(name)
+  end
+
+  # Gets name and password from conn
+  #
+  @spec fetch_credentials(Plug.Conn.t() | map()) ::
+          {:ok, {name :: any, password :: any}} | {:error, :invalid_credentials}
+  def fetch_credentials(%Plug.Conn{params: params} = _),
+    do: fetch_credentials(params)
+
+  def fetch_credentials(params) do
+    case params do
+      %{"authorization" => %{"name" => name, "password" => password}} ->
+        {:ok, {name, password}}
+
+      %{"grant_type" => "password", "username" => name, "password" => password} ->
+        {:ok, {name, password}}
+
+      _ ->
+        {:error, :invalid_credentials}
+    end
+  end
 end
diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex
index 363c99597..177c05636 100644
--- a/lib/pleroma/web/auth/ldap_authenticator.ex
+++ b/lib/pleroma/web/auth/ldap_authenticator.ex
@@ -7,6 +7,9 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
   require Logger
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
   @base Pleroma.Web.Auth.PleromaAuthenticator
 
@@ -20,30 +23,20 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
   defdelegate oauth_consumer_template, to: @base
 
   def get_user(%Plug.Conn{} = conn) do
-    if Pleroma.Config.get([:ldap, :enabled]) do
-      {name, password} =
-        case conn.params do
-          %{"authorization" => %{"name" => name, "password" => password}} ->
-            {name, password}
-
-          %{"grant_type" => "password", "username" => name, "password" => password} ->
-            {name, password}
-        end
-
-      case ldap_user(name, password) do
-        %User{} = user ->
-          {:ok, user}
-
-        {:error, {:ldap_connection_error, _}} ->
-          # When LDAP is unavailable, try default authenticator
-          @base.get_user(conn)
-
-        error ->
-          error
-      end
+    with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])},
+         {:ok, {name, password}} <- fetch_credentials(conn),
+         %User{} = user <- ldap_user(name, password) do
+      {:ok, user}
     else
-      # Fall back to default authenticator
-      @base.get_user(conn)
+      {:error, {:ldap_connection_error, _}} ->
+        # When LDAP is unavailable, try default authenticator
+        @base.get_user(conn)
+
+      {:ldap, _} ->
+        @base.get_user(conn)
+
+      error ->
+        error
     end
   end
 
@@ -94,7 +87,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
     case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do
       :ok ->
-        case User.get_by_nickname_or_email(name) do
+        case fetch_user(name) do
           %User{} = user ->
             user
 
diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex
index d647f1e05..dd79cdcf7 100644
--- a/lib/pleroma/web/auth/pleroma_authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_authenticator.ex
@@ -8,19 +8,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
   alias Pleroma.Repo
   alias Pleroma.User
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
-    {name, password} =
-      case conn.params do
-        %{"authorization" => %{"name" => name, "password" => password}} ->
-          {name, password}
-
-        %{"grant_type" => "password", "username" => name, "password" => password} ->
-          {name, password}
-      end
-
-    with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
+    with {:ok, {name, password}} <- fetch_credentials(conn),
+         {_, %User{} = user} <- {:user, fetch_user(name)},
          {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do
       {:ok, user}
     else
diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs
index a4331478e..0a02039a6 100644
--- a/test/media_proxy_test.exs
+++ b/test/media_proxy_test.exs
@@ -7,15 +7,15 @@ defmodule Pleroma.MediaProxyTest do
   import Pleroma.Web.MediaProxy
   alias Pleroma.Web.MediaProxy.MediaProxyController
 
+  setup do
+    enabled = Pleroma.Config.get([:media_proxy, :enabled])
+    on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
+    :ok
+  end
+
   describe "when enabled" do
     setup do
-      enabled = Pleroma.Config.get([:media_proxy, :enabled])
-
-      unless enabled do
-        Pleroma.Config.put([:media_proxy, :enabled], true)
-        on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
-      end
-
+      Pleroma.Config.put([:media_proxy, :enabled], true)
       :ok
     end
 
diff --git a/test/web/auth/authenticator_test.exs b/test/web/auth/authenticator_test.exs
new file mode 100644
index 000000000..fea5c8209
--- /dev/null
+++ b/test/web/auth/authenticator_test.exs
@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Auth.AuthenticatorTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Web.Auth.Authenticator
+  import Pleroma.Factory
+
+  describe "fetch_user/1" do
+    test "returns user by name" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.nickname) == user
+    end
+
+    test "returns user by email" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.email) == user
+    end
+
+    test "returns nil" do
+      assert Authenticator.fetch_user("email") == nil
+    end
+  end
+
+  describe "fetch_credentials/1" do
+    test "returns name and password from authorization params" do
+      params = %{"authorization" => %{"name" => "test", "password" => "test-pass"}}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns name and password with grant_type 'password'" do
+      params = %{"grant_type" => "password", "username" => "test", "password" => "test-pass"}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns error" do
+      assert Authenticator.fetch_credentials(%{}) == {:error, :invalid_credentials}
+    end
+  end
+end

From 533d8cd5816343ccfb6e26495124416e9808554c Mon Sep 17 00:00:00 2001
From: AkiraFukushima <h3.poteto@gmail.com>
Date: Thu, 2 May 2019 21:04:00 +0900
Subject: [PATCH 24/43] Parse access_token from body parameters and URL
 parameters

---
 lib/pleroma/plugs/oauth_plug.ex | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex
index 5888d596a..9d43732eb 100644
--- a/lib/pleroma/plugs/oauth_plug.ex
+++ b/lib/pleroma/plugs/oauth_plug.ex
@@ -16,6 +16,16 @@ defmodule Pleroma.Plugs.OAuthPlug do
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
+  def call(%{params: %{"access_token" => access_token}} = conn, _) do
+    with {:ok, user, token_record} <- fetch_user_and_token(access_token) do
+      conn
+      |> assign(:token, token_record)
+      |> assign(:user, user)
+    else
+      _ -> conn
+    end
+  end
+
   def call(conn, _) do
     with {:ok, token_str} <- fetch_token_str(conn),
          {:ok, user, token_record} <- fetch_user_and_token(token_str) do

From dff6afc7c88f20cb719a4189d463605589869e8e Mon Sep 17 00:00:00 2001
From: AkiraFukushima <h3.poteto@gmail.com>
Date: Mon, 29 Apr 2019 23:53:48 +0900
Subject: [PATCH 25/43] fix: Add mix deps.get before unit-testing

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c07f1a5d3..dc99b81ee 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -48,6 +48,7 @@ unit-testing:
   - name: postgres:9.6.2
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
+    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix test --trace --preload-modules
@@ -77,4 +78,4 @@ docs-deploy:
     - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}" 
+    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}"

From a53a6c9d64f2c32ca3b53a4317980b3e7c0b37a5 Mon Sep 17 00:00:00 2001
From: AkiraFukushima <h3.poteto@gmail.com>
Date: Thu, 2 May 2019 22:25:21 +0900
Subject: [PATCH 26/43] Add oauth plug tests for url and body parameters

---
 test/plugs/oauth_plug_test.exs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/plugs/oauth_plug_test.exs b/test/plugs/oauth_plug_test.exs
index 17fdba916..5a2ed11cc 100644
--- a/test/plugs/oauth_plug_test.exs
+++ b/test/plugs/oauth_plug_test.exs
@@ -38,6 +38,26 @@ defmodule Pleroma.Plugs.OAuthPlugTest do
     assert conn.assigns[:user] == opts[:user]
   end
 
+  test "with valid token(downcase) in url parameters, it assings the user", opts do
+    conn =
+      :get
+      |> build_conn("/?access_token=#{opts[:token]}")
+      |> put_req_header("content-type", "application/json")
+      |> fetch_query_params()
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
+  test "with valid token(downcase) in body parameters, it assigns the user", opts do
+    conn =
+      :post
+      |> build_conn("/api/v1/statuses", access_token: opts[:token], status: "test")
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
   test "with invalid token, it not assigns the user", %{conn: conn} do
     conn =
       conn

From 38b79461dfe6d14eb95799013b2fdd502e73245c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 2 May 2019 22:19:14 +0300
Subject: [PATCH 27/43] Fix embeded relationships in Mastodon API

Currently some endpoints render accounts without for user resulting in
embedded relationship being empty. It causes bugs in followers/following
tab in pleroma-fe but I fixed it for other endpoints as well just in
case
---
 .../web/mastodon_api/mastodon_api_controller.ex  | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index ed585098a..78dae1c64 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -704,7 +704,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  def favourited_by(conn, %{"id" => id}) do
+  def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
@@ -712,13 +712,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{users: users, as: :user})
+      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
   end
 
-  def reblogged_by(conn, %{"id" => id}) do
+  def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
@@ -726,7 +726,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
@@ -783,7 +783,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       conn
       |> add_link_headers(:followers, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -800,7 +800,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       conn
       |> add_link_headers(:following, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -808,7 +808,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     with {:ok, follow_requests} <- User.get_follow_requests(followed) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: follow_requests, as: :user})
+      |> render("accounts.json", %{for: followed, users: follow_requests, as: :user})
     end
   end
 
@@ -1235,7 +1235,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
          {:ok, users} = Pleroma.List.get_following(list) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     end
   end
 

From acb04306b6954aac8d66a74438d0e213d93a9046 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Fri, 3 May 2019 11:45:04 +0000
Subject: [PATCH 28/43] Standardize construction of websocket URL

This follows up on the change made in d747bd98
---
 lib/pleroma/plugs/http_security_plug.ex                 | 2 +-
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index f701aaaa5..a476f1d49 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -35,7 +35,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
-    websocket_url = String.replace(static_url, "http", "ws")
+    websocket_url = Pleroma.Web.Endpoint.websocket_url()
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index ed585098a..201a21f50 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1295,8 +1295,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       initial_state =
         %{
           meta: %{
-            streaming_api_base_url:
-              String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
+            streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(),
             access_token: token,
             locale: "en",
             domain: Pleroma.Web.Endpoint.host(),

From 85b5c60694e07d3bfb1f885d5fda14be6b7bade9 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 7 Feb 2019 16:41:20 +0100
Subject: [PATCH 29/43] Pleroma.Formatter: width/height to class=emoji

---
 lib/pleroma/formatter.ex                          | 4 +---
 lib/pleroma/html.ex                               | 2 ++
 test/formatter_test.exs                           | 4 ++--
 test/web/twitter_api/views/activity_view_test.exs | 2 +-
 test/web/twitter_api/views/user_view_test.exs     | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index dab8910c1..0ec6bcee0 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -113,9 +113,7 @@ defmodule Pleroma.Formatter do
 
       html =
         if not strip do
-          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
-            MediaProxy.url(file)
-          }' />"
+          "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
         else
           ""
         end
diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 726c370ad..d1da746de 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -151,6 +151,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])
@@ -221,6 +222,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index fdaf29742..06f4f6e50 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -248,7 +248,7 @@ defmodule Pleroma.FormatterTest do
     text = "I love :firefox:"
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
+      "I love <img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
 
     assert Formatter.emojify(text) == expected_result
   end
@@ -263,7 +263,7 @@ defmodule Pleroma.FormatterTest do
     }
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
+      "I love <img class=\"emoji\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
 
     assert Formatter.emojify(text, custom_emoji) == expected_result
   end
diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs
index 85815ba7e..1aa533b48 100644
--- a/test/web/twitter_api/views/activity_view_test.exs
+++ b/test/web/twitter_api/views/activity_view_test.exs
@@ -100,7 +100,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
     expected = ":firefox: meow"
 
     expected_html =
-      "<img height=\"32px\" width=\"32px\" alt=\"firefox\" title=\"firefox\" src=\"http://localhost:4001/emoji/Firefox.gif\" /> meow"
+      "<img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"http://localhost:4001/emoji/Firefox.gif\" /> meow"
 
     assert result["summary"] == expected
     assert result["summary_html"] == expected_html
diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs
index c99dbddeb..74526673c 100644
--- a/test/web/twitter_api/views/user_view_test.exs
+++ b/test/web/twitter_api/views/user_view_test.exs
@@ -32,7 +32,7 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
 
   test "A user with emoji in username" do
     expected =
-      "<img height=\"32px\" width=\"32px\" alt=\"karjalanpiirakka\" title=\"karjalanpiirakka\" src=\"/file.png\" /> man"
+      "<img class=\"emoji\" alt=\"karjalanpiirakka\" title=\"karjalanpiirakka\" src=\"/file.png\" /> man"
 
     user =
       insert(:user, %{

From d70af32127bda1431bacad58e7c6516a2e652fcd Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 11 Feb 2019 23:47:32 +0100
Subject: [PATCH 30/43] Pleroma.User: remove emojify on parse_bio

---
 lib/pleroma/user.ex                           | 21 ++++++++-----------
 .../mastodon_api/mastodon_api_controller.ex   |  2 +-
 test/user_test.exs                            |  2 +-
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 1c62f238e..1741ce684 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.User do
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
   alias Pleroma.Bookmark
-  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -1331,18 +1330,15 @@ defmodule Pleroma.User do
     end
   end
 
-  def parse_bio(bio, user \\ %User{info: %{source_data: %{}}})
-  def parse_bio(nil, _user), do: ""
-  def parse_bio(bio, _user) when bio == "", do: bio
+  def parse_bio(bio) when is_binary(bio) and bio != "" do
+    bio
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
+  end
 
-  def parse_bio(bio, user) do
-    emoji =
-      (user.info.source_data["tag"] || [])
-      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
-        {String.trim(name, ":"), url}
-      end)
+  def parse_bio(_), do: ""
 
+  def parse_bio(bio, user) when is_binary(bio) and bio != "" do
     # TODO: get profile URLs other than user.ap_id
     profile_urls = [user.ap_id]
 
@@ -1352,9 +1348,10 @@ defmodule Pleroma.User do
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
     )
     |> elem(0)
-    |> Formatter.emojify(emoji)
   end
 
+  def parse_bio(_, _), do: ""
+
   def tag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: tag(user_identifier, tags)
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 201a21f50..564dbb829 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -86,7 +86,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     user_params =
       %{}
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end)
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
       |> add_if_present(params, "avatar", :avatar, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
diff --git a/test/user_test.exs b/test/user_test.exs
index 67266cb7a..6d21b56f7 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1103,7 +1103,7 @@ defmodule Pleroma.UserTest do
       expected_text =
         "A.k.a. <span class='h-card'><a data-user='#{remote_user.id}' class='u-url mention' href='#{
           remote_user.ap_id
-        }'>" <> "@<span>nick@domain.com</span></a></span>"
+        }'>@<span>nick@domain.com</span></a></span>"
 
       assert expected_text == User.parse_bio(bio, user)
     end

From 2f76a40d028c45e99425b061298a1b05e4b59923 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 12 Feb 2019 14:59:34 +0100
Subject: [PATCH 31/43] formatter.ex: Add get_emoji_map/1

---
 lib/pleroma/formatter.ex                      | 11 ++++++++
 lib/pleroma/user/info.ex                      |  1 +
 .../web/activity_pub/transmogrifier.ex        | 14 ++++++++--
 .../web/activity_pub/views/user_view.ex       |  7 ++++-
 lib/pleroma/web/common_api/common_api.ex      |  9 ++----
 .../web/twitter_api/twitter_api_controller.ex | 12 +++++++-
 .../twitter_api_controller_test.exs           | 28 +++++++++++++++++++
 7 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 0ec6bcee0..3d7c36d21 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -128,12 +128,23 @@ defmodule Pleroma.Formatter do
 
   def demojify(text, nil), do: text
 
+  @doc "Outputs a list of the emoji-shortcodes in a text"
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []
 
+  @doc "Outputs a list of the emoji-Maps in a text"
+  def get_emoji_map(text) when is_binary(text) do
+    get_emoji(text)
+    |> Enum.reduce(%{}, fn {name, file, _group}, acc ->
+      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+    end)
+  end
+
+  def get_emoji_map(_), do: []
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index a3658d57f..1b81619ce 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -41,6 +41,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+    field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
       default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index b774c2afa..508f3532f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -856,10 +856,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("tag", tags ++ mentions)
   end
 
+  def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do
+    user_info = add_emoji_tags(user_info)
+
+    object
+    |> Map.put(:info, user_info)
+  end
+
   # TODO: we should probably send mtime instead of unix epoch time for updated
-  def add_emoji_tags(object) do
+  def add_emoji_tags(%{"emoji" => emoji} = object) do
     tags = object["tag"] || []
-    emoji = object["emoji"] || []
 
     out =
       emoji
@@ -877,6 +883,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("tag", tags ++ out)
   end
 
+  def add_emoji_tags(object) do
+    object
+  end
+
   def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index 5926a3294..1254fdf6c 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -69,6 +69,11 @@ defmodule Pleroma.Web.ActivityPub.UserView do
 
     endpoints = render("endpoints.json", %{user: user})
 
+    user_tags =
+      user
+      |> Transmogrifier.add_emoji_tags()
+      |> Map.get("tag", [])
+
     %{
       "id" => user.ap_id,
       "type" => "Person",
@@ -87,7 +92,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "tag" => user.info.source_data["tag"] || []
+      "tag" => (user.info.source_data["tag"] || []) ++ user_tags
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index ecd183110..b53869c75 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -151,8 +151,8 @@ defmodule Pleroma.Web.CommonAPI do
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
-         cw <- data["spoiler_text"],
-         full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
+         cw <- data["spoiler_text"] || "",
+         full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
            make_note_data(
@@ -170,10 +170,7 @@ defmodule Pleroma.Web.CommonAPI do
            Map.put(
              object,
              "emoji",
-             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
-             |> Enum.reduce(%{}, fn {name, file, _}, acc ->
-               Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
-             end)
+             Formatter.get_emoji_map(full_payload)
            ) do
       res =
         ActivityPub.create(
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 79ed9dad2..261cc4462 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -653,7 +654,16 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   defp parse_profile_bio(user, params) do
     if bio = params["description"] do
-      Map.put(params, "bio", User.parse_bio(bio, user))
+      user_info =
+        user.info
+        |> Map.put(
+          "emojis",
+          Formatter.get_emoji_map(params["description"])
+        )
+
+      params
+      |> Map.put("bio", User.parse_bio(bio, user))
+      |> Map.put("info", user_info)
     else
       params
     end
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index 43ad71a16..90718cfb4 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -1611,6 +1611,34 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
       assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user})
     end
+
+    # Broken before the change to class="emoji" and non-<img/> in the DB
+    @tag :skip
+    test "it formats emojos", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> post("/api/account/update_profile.json", %{
+          "bio" => "I love our :moominmamma:​"
+        })
+
+      assert response = json_response(conn, 200)
+
+      assert %{
+               "description" => "I love our :moominmamma:",
+               "description_html" =>
+                 ~s{I love our <img class="emoji" alt="moominmamma" title="moominmamma" src="} <>
+                   _
+             } = response
+
+      conn =
+        conn
+        |> get("/api/users/show.json?user_id=#{user.nickname}")
+
+      assert response == json_response(conn, 200)
+    end
   end
 
   defp valid_user(_context) do

From 85434669bb0d363528b738a6d32dd9ce18401065 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 22 Feb 2019 22:35:47 +0100
Subject: [PATCH 32/43] Web.TwitterAPI.UserView: Also view local user emojis

---
 lib/pleroma/web/twitter_api/views/user_view.ex | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index ea015b8f0..6857055c9 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -67,6 +67,8 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
         {String.trim(name, ":"), url}
       end)
 
+    emoji = Enum.dedup(emoji ++ user.info.emoji)
+
     # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
     # For example: [{"name": "Pronoun", "value": "she/her"}, …]
     fields =

From 46bbf9e1cff4e00f2fbd95dcc21c038d7d686b86 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 23 Feb 2019 00:09:11 +0100
Subject: [PATCH 33/43] TwitterAPI: profile update with emoji_map

---
 lib/pleroma/web/twitter_api/twitter_api_controller.ex | 10 ++++++++--
 lib/pleroma/web/twitter_api/views/user_view.ex        |  7 ++++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 261cc4462..ef7b6fe65 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -654,11 +654,17 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   defp parse_profile_bio(user, params) do
     if bio = params["description"] do
+      emojis_text = (params["description"] || "") <> " " <> (params["name"] || "")
+
+      emojis =
+        ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+        |> Enum.dedup()
+
       user_info =
         user.info
         |> Map.put(
-          "emojis",
-          Formatter.get_emoji_map(params["description"])
+          "emoji",
+          emojis
         )
 
       params
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index 6857055c9..f0a4ddbd3 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -69,6 +69,11 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
 
     emoji = Enum.dedup(emoji ++ user.info.emoji)
 
+    description_html =
+      (user.bio || "")
+      |> HTML.filter_tags(User.html_filter_policy(for_user))
+      |> Formatter.emojify(emoji)
+
     # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
     # For example: [{"name": "Pronoun", "value": "she/her"}, …]
     fields =
@@ -80,7 +85,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       %{
         "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
         "description" => HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
-        "description_html" => HTML.filter_tags(user.bio, User.html_filter_policy(for_user)),
+        "description_html" => description_html,
         "favourites_count" => 0,
         "followers_count" => user_info[:follower_count],
         "following" => following,

From b5ad1715b2d4a2a5bdaefa2d56bde71120d23acb Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 23 Feb 2019 00:57:42 +0100
Subject: [PATCH 34/43] MastoAPI: profile update with emoji_map

---
 .../mastodon_api/mastodon_api_controller.ex   |  8 ++++++
 .../mastodon_api_controller_test.exs          | 27 +++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 564dbb829..0840c2c5a 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Bookmark
   alias Pleroma.Config
   alias Pleroma.Filter
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -96,6 +97,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         end
       end)
 
+    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
+
+    user_info_emojis =
+      ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+      |> Enum.dedup()
+
     info_params =
       [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role]
       |> Enum.reduce(%{}, fn key, acc ->
@@ -112,6 +119,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           _ -> :error
         end
       end)
+      |> Map.put(:emoji, user_info_emojis)
 
     info_cng = User.Info.profile_update(user.info, info_params)
 
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index c2a12d3c7..610aa486e 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -2351,6 +2351,33 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
         end
       end
     end
+
+    test "updates profile emojos", %{conn: conn} do
+      user = insert(:user)
+
+      note = "*sips :blank:*"
+      name = "I am :firefox:"
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "note" => note,
+          "display_name" => name
+        })
+
+      assert json_response(conn, 200)
+
+      conn =
+        conn
+        |> get("/api/v1/accounts/#{user.id}")
+
+      assert user = json_response(conn, 200)
+
+      assert user["note"] == note
+      assert user["display_name"] == name
+      assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"]
+    end
   end
 
   test "get instance information", %{conn: conn} do

From 0e37fddd5abad5db0378c8186513fbba0fbd6586 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 3 May 2019 19:15:55 +0200
Subject: [PATCH 35/43] Search: Add fts index on objects table.

---
 CHANGELOG.md                                  |  1 +
 lib/mix/tasks/benchmark.ex                    | 25 +++++++++++++++++++
 mix.exs                                       |  3 ++-
 mix.lock                                      |  2 ++
 ...0190501125843_add_fts_index_to_objects.exs |  8 ++++++
 5 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 lib/mix/tasks/benchmark.ex
 create mode 100644 priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 67b9649e1..038a001de 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -61,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Added support max_id & since_id for bookmark timeline endpoints.
 
 ### Fixed
+- Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
 - Followers counter not being updated when a follower is blocked
 - Deactivated users being able to request an access token
 - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
diff --git a/lib/mix/tasks/benchmark.ex b/lib/mix/tasks/benchmark.ex
new file mode 100644
index 000000000..0fbb4dbb1
--- /dev/null
+++ b/lib/mix/tasks/benchmark.ex
@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.Benchmark do
+  use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
+
+  def run(["search"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "search" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+      end
+    })
+  end
+
+  def run(["tag"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "tag" => fn ->
+        %{"type" => "Create", "tag" => "cofe"}
+        |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
+      end
+    })
+  end
+end
diff --git a/mix.exs b/mix.exs
index 9ded9931c..3b368e57e 100644
--- a/mix.exs
+++ b/mix.exs
@@ -111,7 +111,8 @@ defmodule Pleroma.Mixfile do
       {:prometheus_ecto, "~> 1.4"},
       {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
-      {:quack, "~> 0.1.1"}
+      {:quack, "~> 0.1.1"},
+      {:benchee, "~> 1.0", only: :dev}
     ] ++ oauth_deps
   end
 
diff --git a/mix.lock b/mix.lock
index 08221eadc..e97f4ec38 100644
--- a/mix.lock
+++ b/mix.lock
@@ -3,6 +3,7 @@
   "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
   "calendar": {:hex, :calendar, "0.17.4", "22c5e8d98a4db9494396e5727108dffb820ee0d18fed4b0aa8ab76e4f5bc32f1", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
@@ -17,6 +18,7 @@
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
   "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
diff --git a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs
new file mode 100644
index 000000000..9b274695e
--- /dev/null
+++ b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs
@@ -0,0 +1,8 @@
+defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do
+  use Ecto.Migration
+
+  def change do
+    drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts)
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

From a7709cc26726350a0c90e0e4117cab4a8a07bb35 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 3 May 2019 19:42:49 +0200
Subject: [PATCH 36/43] Mix: Also have benchee in test.

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index 3b368e57e..ef338c7c0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -112,7 +112,7 @@ defmodule Pleroma.Mixfile do
       {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
-      {:benchee, "~> 1.0", only: :dev}
+      {:benchee, "~> 1.0", only: [:dev, :test]}
     ] ++ oauth_deps
   end
 

From a23206a4ae0abe02036c40f9142ee1e6a46f5fae Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 3 May 2019 20:28:06 +0200
Subject: [PATCH 37/43] Just have Benchee in all environments.

---
 mix.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.exs b/mix.exs
index ef338c7c0..c553b835b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -112,7 +112,7 @@ defmodule Pleroma.Mixfile do
       {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
-      {:benchee, "~> 1.0", only: [:dev, :test]}
+      {:benchee, "~> 1.0"}
     ] ++ oauth_deps
   end
 

From 4c76f49e60287e6618f87a2bae3e0d4e8c444895 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 4 May 2019 15:06:18 +0200
Subject: [PATCH 38/43] BBS: small fixes.

---
 lib/pleroma/bbs/handler.ex | 6 +-----
 mix.exs                    | 2 +-
 test/bbs/handler_test.exs  | 4 ++--
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index 1ebba77d2..75ba35dc2 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -1,12 +1,8 @@
 defmodule Pleroma.BBS.Handler do
-  @moduledoc """
-  An example implementation of `Sshd.ShellHandler`, implementing a very simple
-  Read-Eval-Loop, that does nothing.
-  """
   use Sshd.ShellHandler
+  alias Pleroma.Activity
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Activity
 
   def on_shell(username, _pubkey, _ip, _port) do
     :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
diff --git a/mix.exs b/mix.exs
index 38e83e679..47f7c2903 100644
--- a/mix.exs
+++ b/mix.exs
@@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
       {:benchee, "~> 1.0"},
-                                                       {:esshd, "~> 0.1.0"}
+      {:esshd, "~> 0.1.0"}
     ] ++ oauth_deps
   end
 
diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs
index a22c6d64d..1386a2d16 100644
--- a/test/bbs/handler_test.exs
+++ b/test/bbs/handler_test.exs
@@ -1,10 +1,10 @@
 defmodule Pleroma.BBS.HandlerTest do
   use Pleroma.DataCase
+  alias Pleroma.Activity
   alias Pleroma.BBS.Handler
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.User
   alias Pleroma.Repo
-  alias Pleroma.Activity
+  alias Pleroma.User
 
   import ExUnit.CaptureIO
   import Pleroma.Factory

From c9d1cb2dcefb850357b953a7a2d308de39b35e07 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 4 May 2019 15:08:07 +0200
Subject: [PATCH 39/43] BBS: Use cached user fetcher.

---
 lib/pleroma/bbs/handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index 75ba35dc2..14e6a6807 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.BBS.Handler do
 
   def on_shell(username, _pubkey, _ip, _port) do
     :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
-    user = Pleroma.User.get_by_nickname(to_string(username))
+    user = Pleroma.User.get_cached_by_nickname(to_string(username))
     Logger.debug("#{inspect(user)}")
     loop(run_state(user: user))
   end

From c58fd4c038da8305d8840c38f525ceb9f13a644d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 4 May 2019 15:36:48 +0200
Subject: [PATCH 40/43] BBS: Fix tests.

---
 test/bbs/handler_test.exs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs
index 1386a2d16..148df6ddd 100644
--- a/test/bbs/handler_test.exs
+++ b/test/bbs/handler_test.exs
@@ -3,6 +3,7 @@ defmodule Pleroma.BBS.HandlerTest do
   alias Pleroma.Activity
   alias Pleroma.BBS.Handler
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -49,7 +50,8 @@ defmodule Pleroma.BBS.HandlerTest do
       )
 
     assert activity.actor == user.ap_id
-    assert activity.data["object"]["content"] == "this is a test post"
+    object = Object.normalize(activity)
+    assert object.data["content"] == "this is a test post"
   end
 
   test "replying" do
@@ -74,7 +76,8 @@ defmodule Pleroma.BBS.HandlerTest do
       )
 
     assert reply.actor == user.ap_id
-    assert reply.data["object"]["content"] == "this is a reply"
-    assert reply.data["object"]["inReplyTo"] == activity.data["object"]["id"]
+    object = Object.normalize(reply)
+    assert object.data["content"] == "this is a reply"
+    assert object.data["inReplyTo"] == activity.data["object"]
   end
 end

From eb0fb73ddbed109ca4dcd758b60a25ff0dafc883 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sat, 4 May 2019 15:47:50 +0200
Subject: [PATCH 41/43] BBS: Credo fixes.

---
 lib/pleroma/bbs/handler.ex | 2 +-
 test/bbs/handler_test.exs  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex
index 14e6a6807..106fe5d18 100644
--- a/lib/pleroma/bbs/handler.ex
+++ b/lib/pleroma/bbs/handler.ex
@@ -1,8 +1,8 @@
 defmodule Pleroma.BBS.Handler do
   use Sshd.ShellHandler
   alias Pleroma.Activity
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
 
   def on_shell(username, _pubkey, _ip, _port) do
     :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
diff --git a/test/bbs/handler_test.exs b/test/bbs/handler_test.exs
index 148df6ddd..7d5d68d11 100644
--- a/test/bbs/handler_test.exs
+++ b/test/bbs/handler_test.exs
@@ -2,10 +2,10 @@ defmodule Pleroma.BBS.HandlerTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
   alias Pleroma.BBS.Handler
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
 
   import ExUnit.CaptureIO
   import Pleroma.Factory

From ce6ca0fefe7feb1c31447287aec117d40233652a Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 6 May 2019 16:45:22 +0000
Subject: [PATCH 42/43] Merge branch 'develop' of
 https://git.pleroma.social/pleroma/pleroma into
 feature/845-improve-status-deletion

---
 CHANGELOG.md                                  |  1 +
 config/config.exs                             |  6 ++--
 docs/config.md                                |  1 +
 lib/mix/tasks/pleroma/user.ex                 |  4 +--
 lib/pleroma/activity.ex                       |  7 +++++
 lib/pleroma/user.ex                           | 30 +++++++++++--------
 lib/pleroma/user_invite_token.ex              |  2 +-
 .../controllers/util_controller.ex            |  2 +-
 test/user_test.exs                            |  8 +++--
 9 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 038a001de..0d44f6786 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deactivated users being able to request an access token
 - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
 - proper Twitter Card generation instead of a dummy
+- Deletions failing for users with a large number of posts
 - NodeInfo: Include admins in `staffAccounts`
 - ActivityPub: Crashing when requesting empty local user's outbox
 - Federation: Handling of objects without `summary` property
diff --git a/config/config.exs b/config/config.exs
index 1a9738cff..12dbe9941 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -232,7 +232,8 @@ config :pleroma, :instance,
   welcome_message: nil,
   max_report_comment_size: 1000,
   safe_dm_mentions: false,
-  healthcheck: false
+  healthcheck: false,
+  repo_batch_size: 500
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -416,7 +417,8 @@ config :pleroma_job_queue, :queues,
   web_push: 50,
   mailer: 10,
   transmogrifier: 20,
-  scheduled_activities: 10
+  scheduled_activities: 10,
+  background: 5
 
 config :pleroma, :fetch_initial_posts,
   enabled: false,
diff --git a/docs/config.md b/docs/config.md
index ad55d44a7..731eff99a 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `repo_batch_size`: Repo batch size. The number of loaded rows from the database to the memory for processing chunks. E.g. deleting user statuses.
 
 ## :logger
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 9e2523b18..6a83a8c0d 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -163,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete(user)
+      User.perform(:delete, user)
       Mix.shell().info("User #{nickname} deleted.")
     else
       _ ->
@@ -380,7 +380,7 @@ defmodule Mix.Tasks.Pleroma.User do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete_user_activities(user)
+      {:ok, _} = User.delete_user_activities(user)
       Mix.shell().info("User #{nickname} statuses deleted.")
     else
       _ ->
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 4a2ded518..73e63bb14 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @type actor :: String.t()
+
   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@@ -260,4 +262,9 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  @spec query_by_actor(actor()) :: Ecto.Query.t()
+  def query_by_actor(actor) do
+    from(a in Activity, where: a.actor == ^actor)
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 1741ce684..fd2ce81ad 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1164,7 +1164,12 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def delete(%User{} = user) do
+  @spec delete(User.t()) :: :ok
+  def delete(%User{} = user),
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user])
+
+  @spec perform(atom(), User.t()) :: {:ok, User.t()}
+  def perform(:delete, %User{} = user) do
     {:ok, user} = User.deactivate(user)
 
     # Remove all relationships
@@ -1180,22 +1185,23 @@ defmodule Pleroma.User do
   end
 
   def delete_user_activities(%User{ap_id: ap_id} = user) do
-    Activity
-    |> where(actor: ^ap_id)
-    |> Activity.with_preloaded_object()
-    |> Repo.all()
-    |> Enum.each(fn
-      %{data: %{"type" => "Create"}} = activity ->
-        activity |> Object.normalize() |> ActivityPub.delete()
+    stream =
+      ap_id
+      |> Activity.query_by_actor()
+      |> Activity.with_preloaded_object()
+      |> Repo.stream()
 
-      # TODO: Do something with likes, follows, repeats.
-      _ ->
-        "Doing nothing"
-    end)
+    Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
 
     {:ok, user}
   end
 
+  defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
+    Object.normalize(activity) |> ActivityPub.delete()
+  end
+
+  defp delete_activity(_activity), do: "Doing nothing"
+
   def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex
index 86f0a5486..fadc89891 100644
--- a/lib/pleroma/user_invite_token.ex
+++ b/lib/pleroma/user_invite_token.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do
     timestamps()
   end
 
-  @spec create_invite(map()) :: UserInviteToken.t()
+  @spec create_invite(map()) :: {:ok, UserInviteToken.t()}
   def create_invite(params \\ %{}) do
     %UserInviteToken{}
     |> cast(params, [:max_use, :expires_at])
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index 1122e6c5d..c03f8ab3a 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -352,7 +352,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   def delete_account(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->
-        Task.start(fn -> User.delete(user) end)
+        User.delete(user)
         json(conn, %{status: "success"})
 
       {:error, msg} ->
diff --git a/test/user_test.exs b/test/user_test.exs
index 6d21b56f7..adc77a264 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -829,10 +829,12 @@ defmodule Pleroma.UserTest do
     user = insert(:user)
 
     {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
-    {:ok, _} = User.delete_user_activities(user)
 
-    # TODO: Remove favorites, repeats, delete activities.
-    refute Activity.get_by_id(activity.id)
+    Ecto.Adapters.SQL.Sandbox.unboxed_run(Repo, fn ->
+      {:ok, _} = User.delete_user_activities(user)
+      # TODO: Remove favorites, repeats, delete activities.
+      refute Activity.get_by_id(activity.id)
+    end)
   end
 
   test ".delete deactivates a user, all follow relationships and all create activities" do

From 1040caf096347b638b9fda5b23fcccde87b32ede Mon Sep 17 00:00:00 2001
From: Maksim <parallel588@gmail.com>
Date: Mon, 6 May 2019 17:51:03 +0000
Subject: [PATCH 43/43] fix format

Modified-by: Maksim Pechnikov <parallel588@gmail.com>
---
 CHANGELOG.md                                  |   1 +
 config/config.exs                             |   4 +
 docs/api/differences_in_mastoapi_responses.md |   9 +-
 docs/config.md                                |   9 +-
 lib/pleroma/repo.ex                           |  28 +++
 lib/pleroma/web/oauth/app.ex                  |   1 +
 lib/pleroma/web/oauth/authorization.ex        |   8 +
 lib/pleroma/web/oauth/oauth_controller.ex     | 157 +++++++-------
 lib/pleroma/web/oauth/token.ex                |  77 +++++--
 .../web/oauth/token/strategy/refresh_token.ex |  54 +++++
 .../web/oauth/token/strategy/revoke.ex        |  22 ++
 lib/pleroma/web/oauth/token/utils.ex          |  30 +++
 ...33552_add_refresh_token_index_to_token.exs |   7 +
 test/repo_test.exs                            |  44 ++++
 test/web/oauth/oauth_controller_test.exs      | 196 ++++++++++++++++++
 15 files changed, 561 insertions(+), 86 deletions(-)
 create mode 100644 lib/pleroma/web/oauth/token/strategy/refresh_token.ex
 create mode 100644 lib/pleroma/web/oauth/token/strategy/revoke.ex
 create mode 100644 lib/pleroma/web/oauth/token/utils.ex
 create mode 100644 priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs
 create mode 100644 test/repo_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d44f6786..210aae2e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/)
 - ActivityPub C2S: OAuth endpoints
 - Metadata RelMe provider
+- OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 
 ### Changed
diff --git a/config/config.exs b/config/config.exs
index 7792e9a87..946ba9adf 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -473,6 +473,10 @@ config :pleroma, Pleroma.ScheduledActivity,
   total_user_limit: 300,
   enabled: true
 
+config :pleroma, :oauth2,
+  token_expires_in: 600,
+  issue_new_refresh_token: true
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md
index 1350ace43..d3ba41b6a 100644
--- a/docs/api/differences_in_mastoapi_responses.md
+++ b/docs/api/differences_in_mastoapi_responses.md
@@ -1,6 +1,6 @@
 # Differences in Mastodon API responses from vanilla Mastodon
 
-A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance` 
+A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance`
 
 ## Flake IDs
 
@@ -80,3 +80,10 @@ Additional parameters can be added to the JSON body/Form data:
 - `hide_favorites` - if true, user's favorites timeline will be hidden
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
 - `default_scope` - the scope returned under `privacy` key in Source subentity
+
+## Authentication
+
+*Pleroma supports refreshing tokens.
+
+`POST /oauth/token`
+Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.
diff --git a/docs/config.md b/docs/config.md
index bbdc22688..d999952e1 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -474,7 +474,7 @@ Authentication / authorization settings.
 * `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`.
 * `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable.
 
-# OAuth consumer mode
+## OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
 Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies).
@@ -527,6 +527,13 @@ config :ueberauth, Ueberauth,
   ]
 ```
 
+## OAuth 2.0 provider - :oauth2
+
+Configure OAuth 2 provider capabilities:
+
+* `token_expires_in` - The lifetime in seconds of the access token.
+* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index aa5d427ae..f57e088bc 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -19,4 +19,32 @@ defmodule Pleroma.Repo do
   def init(_, opts) do
     {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
   end
+
+  @doc "find resource based on prepared query"
+  @spec find_resource(Ecto.Query.t()) :: {:ok, struct()} | {:error, :not_found}
+  def find_resource(%Ecto.Query{} = query) do
+    case __MODULE__.one(query) do
+      nil -> {:error, :not_found}
+      resource -> {:ok, resource}
+    end
+  end
+
+  def find_resource(_query), do: {:error, :not_found}
+
+  @doc """
+  Gets association from cache or loads if need
+
+  ## Examples
+
+    iex> Repo.get_assoc(token, :user)
+    %User{}
+
+  """
+  @spec get_assoc(struct(), atom()) :: {:ok, struct()} | {:error, :not_found}
+  def get_assoc(resource, association) do
+    case __MODULE__.preload(resource, association) do
+      %{^association => assoc} when not is_nil(assoc) -> {:ok, assoc}
+      _ -> {:error, :not_found}
+    end
+  end
 end
diff --git a/lib/pleroma/web/oauth/app.ex b/lib/pleroma/web/oauth/app.ex
index 3476da484..bccc2ac96 100644
--- a/lib/pleroma/web/oauth/app.ex
+++ b/lib/pleroma/web/oauth/app.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.App do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @type t :: %__MODULE__{}
   schema "apps" do
     field(:client_name, :string)
     field(:redirect_uris, :string)
diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex
index 3461f9983..ca3901cc4 100644
--- a/lib/pleroma/web/oauth/authorization.ex
+++ b/lib/pleroma/web/oauth/authorization.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
   import Ecto.Changeset
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
@@ -63,4 +64,11 @@ defmodule Pleroma.Web.OAuth.Authorization do
     )
     |> Repo.delete_all()
   end
+
+  @doc "gets auth for app by token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 688eaca11..e3c01217d 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -13,11 +13,15 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken
 
   import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
 
   if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth)
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+
   plug(:fetch_session)
   plug(:fetch_flash)
 
@@ -138,25 +142,33 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     Authenticator.handle_error(conn, error)
   end
 
+  @doc "Renew access_token with refresh_token"
+  def token_exchange(
+        conn,
+        %{"grant_type" => "refresh_token", "refresh_token" => token} = params
+      ) do
+    with %App{} = app <- get_app_from_request(conn, params),
+         {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token),
+         {:ok, token} <- RefreshToken.grant(token) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
+
+      json(conn, response_token(user, token, response_attrs))
+    else
+      _error ->
+        put_status(conn, 400)
+        |> json(%{error: "Invalid credentials"})
+    end
+  end
+
   def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         fixed_token = fix_padding(params["code"]),
-         %Authorization{} = auth <-
-           Repo.get_by(Authorization, token: fixed_token, app_id: app.id),
+         fixed_token = Token.Utils.fix_padding(params["code"]),
+         {:ok, auth} <- Authorization.get_by_token(app, fixed_token),
          %User{} = user <- User.get_cached_by_id(auth.user_id),
-         {:ok, token} <- Token.exchange_token(app, auth),
-         {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        created_at: DateTime.to_unix(inserted_at),
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
+         {:ok, token} <- Token.exchange_token(app, auth) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
 
-      json(conn, response)
+      json(conn, response_token(user, token, response_attrs))
     else
       _error ->
         put_status(conn, 400)
@@ -177,16 +189,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
          true <- Enum.any?(scopes),
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes),
          {:ok, token} <- Token.exchange_token(app, auth) do
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
-
-      json(conn, response)
+      json(conn, response_token(user, token))
     else
       {:auth_active, false} ->
         # Per https://github.com/tootsuite/mastodon/blob/
@@ -218,10 +221,12 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     token_exchange(conn, params)
   end
 
-  def token_revoke(conn, %{"token" => token} = params) do
+  # Bad request
+  def token_exchange(conn, params), do: bad_request(conn, params)
+
+  def token_revoke(conn, %{"token" => _token} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         %Token{} = token <- Repo.get_by(Token, token: token, app_id: app.id),
-         {:ok, %Token{}} <- Repo.delete(token) do
+         {:ok, _token} <- RevokeToken.revoke(app, params) do
       json(conn, %{})
     else
       _error ->
@@ -230,6 +235,15 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     end
   end
 
+  def token_revoke(conn, params), do: bad_request(conn, params)
+
+  # Response for bad request
+  defp bad_request(conn, _) do
+    conn
+    |> put_status(500)
+    |> json(%{error: "Bad request"})
+  end
+
   @doc "Prepares OAuth request to provider for Ueberauth"
   def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do
     scope =
@@ -278,25 +292,22 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     params = callback_params(params)
 
     with {:ok, registration} <- Authenticator.get_registration(conn) do
-      user = Repo.preload(registration, :user).user
       auth_attrs = Map.take(params, ~w(client_id redirect_uri scope scopes state))
 
-      if user do
-        create_authorization(
-          conn,
-          %{"authorization" => auth_attrs},
-          user: user
-        )
-      else
-        registration_params =
-          Map.merge(auth_attrs, %{
-            "nickname" => Registration.nickname(registration),
-            "email" => Registration.email(registration)
-          })
+      case Repo.get_assoc(registration, :user) do
+        {:ok, user} ->
+          create_authorization(conn, %{"authorization" => auth_attrs}, user: user)
 
-        conn
-        |> put_session(:registration_id, registration.id)
-        |> registration_details(%{"authorization" => registration_params})
+        _ ->
+          registration_params =
+            Map.merge(auth_attrs, %{
+              "nickname" => Registration.nickname(registration),
+              "email" => Registration.email(registration)
+            })
+
+          conn
+          |> put_session(:registration_id, registration.id)
+          |> registration_details(%{"authorization" => registration_params})
       end
     else
       _ ->
@@ -399,36 +410,30 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     end
   end
 
-  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
-  # decoding it.  Investigate sometime.
-  defp fix_padding(token) do
-    token
-    |> URI.decode()
-    |> Base.url_decode64!(padding: false)
-    |> Base.url_encode64(padding: false)
+  defp get_app_from_request(conn, params) do
+    conn
+    |> fetch_client_credentials(params)
+    |> fetch_client
   end
 
-  defp get_app_from_request(conn, params) do
-    # Per RFC 6749, HTTP Basic is preferred to body params
-    {client_id, client_secret} =
-      with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-           {:ok, decoded} <- Base.decode64(encoded),
-           [id, secret] <-
-             String.split(decoded, ":")
-             |> Enum.map(fn s -> URI.decode_www_form(s) end) do
-        {id, secret}
-      else
-        _ -> {params["client_id"], params["client_secret"]}
-      end
+  defp fetch_client({id, secret}) when is_binary(id) and is_binary(secret) do
+    Repo.get_by(App, client_id: id, client_secret: secret)
+  end
 
-    if client_id && client_secret do
-      Repo.get_by(
-        App,
-        client_id: client_id,
-        client_secret: client_secret
-      )
+  defp fetch_client({_id, _secret}), do: nil
+
+  defp fetch_client_credentials(conn, params) do
+    # Per RFC 6749, HTTP Basic is preferred to body params
+    with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
+         {:ok, decoded} <- Base.decode64(encoded),
+         [id, secret] <-
+           Enum.map(
+             String.split(decoded, ":"),
+             fn s -> URI.decode_www_form(s) end
+           ) do
+      {id, secret}
     else
-      nil
+      _ -> {params["client_id"], params["client_secret"]}
     end
   end
 
@@ -441,4 +446,16 @@ defmodule Pleroma.Web.OAuth.OAuthController do
 
   defp put_session_registration_id(conn, registration_id),
     do: put_session(conn, :registration_id, registration_id)
+
+  defp response_token(%User{} = user, token, opts \\ %{}) do
+    %{
+      token_type: "Bearer",
+      access_token: token.token,
+      refresh_token: token.refresh_token,
+      expires_in: @expires_in,
+      scope: Enum.join(token.scopes, " "),
+      me: user.ap_id
+    }
+    |> Map.merge(opts)
+  end
 end
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 399140003..4e5d1d118 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
   import Ecto.Query
+  import Ecto.Changeset
 
   alias Pleroma.Repo
   alias Pleroma.User
@@ -13,6 +14,9 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+  @type t :: %__MODULE__{}
+
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
@@ -24,28 +28,67 @@ defmodule Pleroma.Web.OAuth.Token do
     timestamps()
   end
 
+  @doc "Gets token for app by access token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
+
+  @doc "Gets token for app by refresh token"
+  @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_refresh_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__,
+      where: t.app_id == ^app_id and t.refresh_token == ^token,
+      preload: [:user]
+    )
+    |> Repo.find_resource()
+  end
+
   def exchange_token(app, auth) do
     with {:ok, auth} <- Authorization.use_token(auth),
          true <- auth.app_id == app.id do
-      create_token(app, User.get_cached_by_id(auth.user_id), auth.scopes)
+      create_token(
+        app,
+        User.get_cached_by_id(auth.user_id),
+        %{scopes: auth.scopes}
+      )
     end
   end
 
-  def create_token(%App{} = app, %User{} = user, scopes \\ nil) do
-    scopes = scopes || app.scopes
-    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
-    refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+  defp put_token(changeset) do
+    changeset
+    |> change(%{token: Token.Utils.generate_token()})
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
 
-    token = %Token{
-      token: token,
-      refresh_token: refresh_token,
-      scopes: scopes,
-      user_id: user.id,
-      app_id: app.id,
-      valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10)
-    }
+  defp put_refresh_token(changeset, attrs) do
+    refresh_token = Map.get(attrs, :refresh_token, Token.Utils.generate_token())
 
-    Repo.insert(token)
+    changeset
+    |> change(%{refresh_token: refresh_token})
+    |> validate_required([:refresh_token])
+    |> unique_constraint(:refresh_token)
+  end
+
+  defp put_valid_until(changeset, attrs) do
+    expires_in =
+      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in))
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
+    %__MODULE__{user_id: user.id, app_id: app.id}
+    |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes])
+    |> validate_required([:scopes, :user_id, :app_id])
+    |> put_valid_until(attrs)
+    |> put_token
+    |> put_refresh_token(attrs)
+    |> Repo.insert()
   end
 
   def delete_user_tokens(%User{id: user_id}) do
@@ -73,4 +116,10 @@ defmodule Pleroma.Web.OAuth.Token do
     |> Repo.all()
     |> Repo.preload(:app)
   end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
 end
diff --git a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex
new file mode 100644
index 000000000..7df0be14e
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex
@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do
+  @moduledoc """
+  Functions for dealing with refresh token strategy.
+  """
+
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke
+
+  @doc """
+  Will grant access token by refresh token.
+  """
+  @spec grant(Token.t()) :: {:ok, Token.t()} | {:error, any()}
+  def grant(token) do
+    access_token = Repo.preload(token, [:user, :app])
+
+    result =
+      Repo.transaction(fn ->
+        token_params = %{
+          app: access_token.app,
+          user: access_token.user,
+          scopes: access_token.scopes
+        }
+
+        access_token
+        |> revoke_access_token()
+        |> create_access_token(token_params)
+      end)
+
+    case result do
+      {:ok, {:error, reason}} -> {:error, reason}
+      {:ok, {:ok, token}} -> {:ok, token}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  defp revoke_access_token(token) do
+    Revoke.revoke(token)
+  end
+
+  defp create_access_token({:error, error}, _), do: {:error, error}
+
+  defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do
+    Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token))
+  end
+
+  defp add_refresh_token(params, token) do
+    case Config.get([:oauth2, :issue_new_refresh_token], false) do
+      true -> Map.put(params, :refresh_token, token)
+      false -> params
+    end
+  end
+end
diff --git a/lib/pleroma/web/oauth/token/strategy/revoke.ex b/lib/pleroma/web/oauth/token/strategy/revoke.ex
new file mode 100644
index 000000000..dea63ca54
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/strategy/revoke.ex
@@ -0,0 +1,22 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.Revoke do
+  @moduledoc """
+  Functions for dealing with revocation.
+  """
+
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Token
+
+  @doc "Finds and revokes access token for app and by token"
+  @spec revoke(App.t(), map()) :: {:ok, Token.t()} | {:error, :not_found | Ecto.Changeset.t()}
+  def revoke(%App{} = app, %{"token" => token} = _attrs) do
+    with {:ok, token} <- Token.get_by_token(app, token),
+         do: revoke(token)
+  end
+
+  @doc "Revokes access token"
+  @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
+  def revoke(%Token{} = token) do
+    Repo.delete(token)
+  end
+end
diff --git a/lib/pleroma/web/oauth/token/utils.ex b/lib/pleroma/web/oauth/token/utils.ex
new file mode 100644
index 000000000..a81560a1c
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/utils.ex
@@ -0,0 +1,30 @@
+defmodule Pleroma.Web.OAuth.Token.Utils do
+  @moduledoc """
+  Auxiliary functions for dealing with tokens.
+  """
+
+  @doc "convert token inserted_at to unix timestamp"
+  def format_created_at(%{inserted_at: inserted_at} = _token) do
+    inserted_at
+    |> DateTime.from_naive!("Etc/UTC")
+    |> DateTime.to_unix()
+  end
+
+  @doc false
+  @spec generate_token(keyword()) :: binary()
+  def generate_token(opts \\ []) do
+    opts
+    |> Keyword.get(:size, 32)
+    |> :crypto.strong_rand_bytes()
+    |> Base.url_encode64(padding: false)
+  end
+
+  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  # decoding it.  Investigate sometime.
+  def fix_padding(token) do
+    token
+    |> URI.decode()
+    |> Base.url_decode64!(padding: false)
+    |> Base.url_encode64(padding: false)
+  end
+end
diff --git a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs
new file mode 100644
index 000000000..449f2a3d4
--- /dev/null
+++ b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs
@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do
+  use Ecto.Migration
+
+  def change do
+    create(unique_index(:oauth_tokens, [:refresh_token]))
+  end
+end
diff --git a/test/repo_test.exs b/test/repo_test.exs
new file mode 100644
index 000000000..5382289c7
--- /dev/null
+++ b/test/repo_test.exs
@@ -0,0 +1,44 @@
+defmodule Pleroma.RepoTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  describe "find_resource/1" do
+    test "returns user" do
+      user = insert(:user)
+      query = from(t in Pleroma.User, where: t.id == ^user.id)
+      assert Repo.find_resource(query) == {:ok, user}
+    end
+
+    test "returns not_found" do
+      query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
+      assert Repo.find_resource(query) == {:error, :not_found}
+    end
+  end
+
+  describe "get_assoc/2" do
+    test "get assoc from preloaded data" do
+      user = %Pleroma.User{name: "Agent Smith"}
+      token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user}
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-one assoc from repo" do
+      user = insert(:user, name: "Jimi Hendrix")
+      token = refresh_record(insert(:oauth_token, user: user))
+
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-many assoc from repo" do
+      user = insert(:user)
+      notification = refresh_record(insert(:notification, user: user))
+
+      assert Repo.get_assoc(user, :notifications) == {:ok, [notification]}
+    end
+
+    test "return error if has not assoc " do
+      token = insert(:oauth_token, user: nil)
+      assert Repo.get_assoc(token, :user) == {:error, :not_found}
+    end
+  end
+end
diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs
index 6e96537ec..cb6836983 100644
--- a/test/web/oauth/oauth_controller_test.exs
+++ b/test/web/oauth/oauth_controller_test.exs
@@ -12,6 +12,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @oauth_config_path [:oauth2, :issue_new_refresh_token]
   @session_opts [
     store: :cookie,
     key: "_test",
@@ -714,4 +715,199 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
       refute Map.has_key?(resp, "access_token")
     end
   end
+
+  describe "POST /oauth/token - refresh token" do
+    setup do
+      oauth_token_config = Pleroma.Config.get(@oauth_config_path)
+
+      on_exit(fn ->
+        Pleroma.Config.get(@oauth_config_path, oauth_token_config)
+      end)
+    end
+
+    test "issues a new access token with keep fresh token" do
+      Pleroma.Config.put(@oauth_config_path, true)
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      new_token = Repo.get_by(Token, token: response["access_token"])
+      assert new_token.refresh_token == token.refresh_token
+      assert new_token.scopes == auth.scopes
+      assert new_token.user_id == user.id
+      assert new_token.app_id == app.id
+    end
+
+    test "issues a new access token with new fresh token" do
+      Pleroma.Config.put(@oauth_config_path, false)
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      new_token = Repo.get_by(Token, token: response["access_token"])
+      refute new_token.refresh_token == token.refresh_token
+      assert new_token.scopes == auth.scopes
+      assert new_token.user_id == user.id
+      assert new_token.app_id == app.id
+    end
+
+    test "returns 400 if we try use access token" do
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => token.token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(400)
+
+      assert %{"error" => "Invalid credentials"} == response
+    end
+
+    test "returns 400 if refresh_token invalid" do
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => "token.refresh_token",
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(400)
+
+      assert %{"error" => "Invalid credentials"} == response
+    end
+
+    test "issues a new token if token expired" do
+      user = insert(:user)
+      app = insert(:oauth_app, scopes: ["read", "write"])
+
+      {:ok, auth} = Authorization.create_authorization(app, user, ["write"])
+      {:ok, token} = Token.exchange_token(app, auth)
+
+      change =
+        Ecto.Changeset.change(
+          token,
+          %{valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -86_400 * 30)}
+        )
+
+      {:ok, access_token} = Repo.update(change)
+
+      response =
+        build_conn()
+        |> post("/oauth/token", %{
+          "grant_type" => "refresh_token",
+          "refresh_token" => access_token.refresh_token,
+          "client_id" => app.client_id,
+          "client_secret" => app.client_secret
+        })
+        |> json_response(200)
+
+      ap_id = user.ap_id
+
+      assert match?(
+               %{
+                 "scope" => "write",
+                 "token_type" => "Bearer",
+                 "expires_in" => 600,
+                 "access_token" => _,
+                 "refresh_token" => _,
+                 "me" => ^ap_id
+               },
+               response
+             )
+
+      refute Repo.get_by(Token, token: token.token)
+      token = Repo.get_by(Token, token: response["access_token"])
+      assert token
+      assert token.scopes == auth.scopes
+      assert token.user_id == user.id
+      assert token.app_id == app.id
+    end
+  end
+
+  describe "POST /oauth/token - bad request" do
+    test "returns 500" do
+      response =
+        build_conn()
+        |> post("/oauth/token", %{})
+        |> json_response(500)
+
+      assert %{"error" => "Bad request"} == response
+    end
+  end
+
+  describe "POST /oauth/revoke - bad request" do
+    test "returns 500" do
+      response =
+        build_conn()
+        |> post("/oauth/revoke", %{})
+        |> json_response(500)
+
+      assert %{"error" => "Bad request"} == response
+    end
+  end
 end