Merge branch 'develop' into 'oembed_provider'

# Conflicts:
#   lib/pleroma/activity.ex

lib/mix/tasks/pleroma/instance.ex

@@ -105,6 +105,7 @@ def run(["gen" | rest]) do
         )
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
+      signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
       {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
 
       result_config =
@@ -120,6 +121,7 @@ def run(["gen" | rest]) do
           dbpass: dbpass,
           version: Pleroma.Mixfile.project() |> Keyword.get(:version),
           secret: secret,
+          signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
           web_push_private_key: Base.url_encode64(web_push_private_key, padding: false)
         )

lib/mix/tasks/pleroma/sample_config.eex

@@ -7,7 +7,8 @@ use Mix.Config
 
 config :pleroma, Pleroma.Web.Endpoint,
    url: [host: "<%= domain %>", scheme: "https", port: <%= port %>],
-   secret_key_base: "<%= secret %>"
+   secret_key_base: "<%= secret %>",
+   signing_salt: "<%= signing_salt %>"
 
 config :pleroma, :instance,
   name: "<%= name %>",

lib/pleroma/PasswordResetToken.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do
   alias Pleroma.{User, PasswordResetToken, Repo}
 
   schema "password_reset_tokens" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:token, :string)
     field(:used, :boolean, default: false)
 

lib/pleroma/activity.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
@@ -40,25 +41,7 @@ def get_by_id(id) do
     Repo.get(Activity, id)
   end
 
-  # TODO:
+  def by_object_ap_id(ap_id) do
-  # Go through these and fix them everywhere.
-  # Wrong name, only returns create activities
-  def all_by_object_ap_id_q(ap_id) do
-    from(
-      activity in Activity,
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data)
-    )
-  end
-
-  # Wrong name, returns all.
-  def all_non_create_by_object_ap_id_q(ap_id) do
     from(
       activity in Activity,
       where:
@@ -71,12 +54,7 @@ def all_non_create_by_object_ap_id_q(ap_id) do
     )
   end
 
-  # Wrong name plz fix thx
+  def create_by_object_ap_id(ap_ids) when is_list(ap_ids) do
-  def all_by_object_ap_id(ap_id) do
-    Repo.all(all_by_object_ap_id_q(ap_id))
-  end
-
-  def create_activity_by_object_id_query(ap_ids) do
     from(
       activity in Activity,
       where:
@@ -90,19 +68,37 @@ def create_activity_by_object_id_query(ap_ids) do
     )
   end
 
-  def get_create_activity_by_object_ap_id(ap_id) when is_binary(ap_id) do
+  def create_by_object_ap_id(ap_id) do
-    create_activity_by_object_id_query([ap_id])
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data)
+    )
+  end
+
+  def get_all_create_by_object_ap_id(ap_id) do
+    Repo.all(create_by_object_ap_id(ap_id))
+  end
+
+  def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
+    create_by_object_ap_id(ap_id)
     |> Repo.one()
   end
 
-  def get_create_activity_by_object_ap_id(_), do: nil
+  def get_create_by_object_ap_id(_), do: nil
 
   def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
   def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
-    get_create_activity_by_object_ap_id(ap_id)
+    get_create_by_object_ap_id(ap_id)
   end
 
   def get_in_reply_to_activity(_), do: nil

lib/pleroma/application.ex

@@ -99,6 +99,7 @@ def start(_type, _args) do
           ],
           id: :cachex_idem
         ),
+        worker(Pleroma.FlakeId, []),
         worker(Pleroma.Web.Federator.RetryQueue, []),
         worker(Pleroma.Web.Federator, []),
         worker(Pleroma.Stats, []),

lib/pleroma/clippy.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Clippy do
+  @moduledoc false
+  # No software is complete until they have a Clippy implementation.
+  # A ballmer peak _may_ be required to change this module.
+
+  def tip() do
+    tips()
+    |> Enum.random()
+    |> puts()
+  end
+
+  def tips() do
+    host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+    [
+      "“πλήρωμα” is “pleroma” in greek",
+      "For an extended Pleroma Clippy Experience, use the “Redmond” themes in Pleroma FE settings",
+      "Staff accounts and MRF policies of Pleroma instances are disclosed on the NodeInfo endpoints for easy transparency!\n
+- https://catgirl.science/misc/nodeinfo.lua?#{host}
+- https://fediverse.network/#{host}/federation",
+      "Pleroma can federate to the Dark Web!\n
+- Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor)
+- i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation",
+      "Lists of Pleroma instances:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma",
+      "Pleroma uses the LitePub protocol - https://litepub.social",
+      "To receive more federated posts, subscribe to relays!\n
+- How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment
+- Relays: https://fediverse.network/activityrelay"
+    ]
+  end
+
+  @spec puts(String.t() | [[IO.ANSI.ansicode() | String.t(), ...], ...]) :: nil
+  def puts(text_or_lines) do
+    import IO.ANSI
+
+    lines =
+      if is_binary(text_or_lines) do
+        String.split(text_or_lines, ~r/\n/)
+      else
+        text_or_lines
+      end
+
+    longest_line_size =
+      lines
+      |> Enum.map(&charlist_count_text/1)
+      |> Enum.sort(&>=/2)
+      |> List.first()
+
+    pad_text = longest_line_size
+
+    pad =
+      for(_ <- 1..pad_text, do: "_")
+      |> Enum.join("")
+
+    pad_spaces =
+      for(_ <- 1..pad_text, do: " ")
+      |> Enum.join("")
+
+    spaces = "      "
+
+    pre_lines = [
+      "  /  \\#{spaces}  _#{pad}___",
+      "  |  |#{spaces} / #{pad_spaces}   \\"
+    ]
+
+    for l <- pre_lines do
+      IO.puts(l)
+    end
+
+    clippy_lines = [
+      "  #{bright()}@  @#{reset()}#{spaces} ",
+      "  || ||#{spaces}",
+      "  || ||   <--",
+      "  |\\_/|      ",
+      "  \\___/      "
+    ]
+
+    noclippy_line = "             "
+
+    env = %{
+      max_size: pad_text,
+      pad: pad,
+      pad_spaces: pad_spaces,
+      spaces: spaces,
+      pre_lines: pre_lines,
+      noclippy_line: noclippy_line
+    }
+
+    # surrond one/five line clippy with blank lines around to not fuck up the layout
+    #
+    # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched
+    # features anyway?
+    lines =
+      if length(lines) == 1 or length(lines) == 5 do
+        [""] ++ lines ++ [""]
+      else
+        lines
+      end
+
+    clippy_line(lines, clippy_lines, env)
+  rescue
+    e ->
+      IO.puts("(Clippy crashed, sorry: #{inspect(e)})")
+      IO.puts(text_or_lines)
+  end
+
+  defp clippy_line([line | lines], [prefix | clippy_lines], env) do
+    IO.puts([prefix <> "| ", rpad_line(line, env.max_size)])
+    clippy_line(lines, clippy_lines, env)
+  end
+
+  # more text lines but clippy's complete
+  defp clippy_line([line | lines], [], env) do
+    IO.puts([env.noclippy_line, "| ", rpad_line(line, env.max_size)])
+
+    if lines == [] do
+      IO.puts(env.noclippy_line <> "\\_#{env.pad}___/")
+    end
+
+    clippy_line(lines, [], env)
+  end
+
+  # no more text lines but clippy's not complete
+  defp clippy_line([], [clippy | clippy_lines], env) do
+    if env.pad do
+      IO.puts(clippy <> "\\_#{env.pad}___/")
+      clippy_line([], clippy_lines, %{env | pad: nil})
+    else
+      IO.puts(clippy)
+      clippy_line([], clippy_lines, env)
+    end
+  end
+
+  defp clippy_line(_, _, _) do
+  end
+
+  defp rpad_line(line, max) do
+    pad = max - (charlist_count_text(line) - 2)
+    pads = Enum.join(for(_ <- 1..pad, do: " "))
+    [IO.ANSI.format(line), pads <> " |"]
+  end
+
+  defp charlist_count_text(line) do
+    if is_list(line) do
+      text = Enum.join(Enum.filter(line, &is_binary/1))
+      String.length(text)
+    else
+      String.length(line)
+    end
+  end
+end

lib/pleroma/filter.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Filter do
   alias Pleroma.{User, Repo}
 
   schema "filters" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -0,0 +1,183 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeId do
+  @moduledoc """
+  Flake is a decentralized, k-ordered id generation service.
+
+  Adapted from:
+
+  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
+  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
+  """
+
+  @type t :: binary
+
+  @behaviour Ecto.Type
+  use GenServer
+  require Logger
+  alias __MODULE__
+  import Kernel, except: [to_string: 1]
+
+  defstruct node: nil, time: 0, sq: 0
+
+  @doc "Converts a binary Flake to a String"
+  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
+    Kernel.to_string(id)
+  end
+
+  def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do
+    encode_base62(flake)
+  end
+
+  def to_string(s), do: s
+
+  for i <- [-1, 0] do
+    def from_string(unquote(i)), do: <<0::integer-size(128)>>
+    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
+  end
+
+  def from_string(flake = <<_::integer-size(128)>>), do: flake
+
+  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
+    case Integer.parse(string) do
+      {id, _} -> <<0::integer-size(64), id::integer-size(64)>>
+      _ -> nil
+    end
+  end
+
+  def from_string(string) do
+    string |> decode_base62 |> from_integer
+  end
+
+  def to_integer(<<integer::integer-size(128)>>), do: integer
+
+  def from_integer(integer) do
+    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
+      <<integer::integer-size(128)>>
+  end
+
+  @doc "Generates a Flake"
+  @spec get :: binary
+  def get, do: to_string(:gen_server.call(:flake, :get))
+
+  # -- Ecto.Type API
+  @impl Ecto.Type
+  def type, do: :uuid
+
+  @impl Ecto.Type
+  def cast(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def load(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def dump(value) do
+    {:ok, FlakeId.from_string(value)}
+  end
+
+  def autogenerate(), do: get()
+
+  # -- GenServer API
+  def start_link do
+    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
+  end
+
+  @impl GenServer
+  def init([]) do
+    {:ok, %FlakeId{node: mac(), time: time()}}
+  end
+
+  @impl GenServer
+  def handle_call(:get, _from, state) do
+    {flake, new_state} = get(time(), state)
+    {:reply, flake, new_state}
+  end
+
+  # Matches when the calling time is the same as the state time. Incr. sq
+  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
+    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Matches when the times are different, reset sq
+  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
+    new_state = %FlakeId{time: newtime, node: node, sq: 0}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Error when clock is running backwards
+  defp get(newtime, %FlakeId{time: time}) when newtime < time do
+    {:error, :clock_running_backwards}
+  end
+
+  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
+    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
+  end
+
+  defp nthchar_base62(n) when n <= 9, do: ?0 + n
+  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
+  defp nthchar_base62(n), do: ?a + n - 36
+
+  defp encode_base62(<<integer::integer-size(128)>>) do
+    integer
+    |> encode_base62([])
+    |> List.to_string()
+  end
+
+  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
+  defp encode_base62(int, []) when int == 0, do: '0'
+  defp encode_base62(int, acc) when int == 0, do: acc
+
+  defp encode_base62(int, acc) do
+    r = rem(int, 62)
+    id = div(int, 62)
+    acc = [nthchar_base62(r) | acc]
+    encode_base62(id, acc)
+  end
+
+  defp decode_base62(s) do
+    decode_base62(String.to_charlist(s), 0)
+  end
+
+  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
+    do: decode_base62(cs, 62 * acc + (c - ?0))
+
+  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
+    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
+
+  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
+    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
+
+  defp decode_base62([], acc), do: acc
+
+  defp time do
+    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
+    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
+  end
+
+  def mac do
+    {:ok, addresses} = :inet.getifaddrs()
+
+    macids =
+      Enum.reduce(addresses, [], fn {_iface, attrs}, acc ->
+        case attrs[:hwaddr] do
+          [0, 0, 0 | _] -> acc
+          mac when is_list(mac) -> [mac_to_worker_id(mac) | acc]
+          _ -> acc
+        end
+      end)
+
+    List.first(macids)
+  end
+
+  def mac_to_worker_id(mac) do
+    <<worker::integer-size(48)>> = :binary.list_to_bin(mac)
+    worker
+  end
+end

lib/pleroma/formatter.ex

@@ -130,7 +130,7 @@ def add_links({subs, text}) do
   end
 
   @doc "Adds the links to mentioned users"
-  def add_user_links({subs, text}, mentions) do
+  def add_user_links({subs, text}, mentions, options \\ []) do
     mentions =
       mentions
       |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
@@ -152,12 +152,16 @@ def add_user_links({subs, text}, mentions) do
               ap_id
             end
 
-          short_match = String.split(match, "@") |> tl() |> hd()
+          nickname =
+            if options[:format] == :full do
+              User.full_nickname(match)
+            else
+              User.local_nickname(match)
+            end
 
           {uuid,
-           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>@<span>#{
+           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>" <>
-             short_match
+             "@<span>#{nickname}</span></a></span>"}
-           }</span></a></span>"}
         end)
 
     {subs, uuid_text}

lib/pleroma/list.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.List do
   alias Pleroma.{User, Repo, Activity}
 
   schema "lists" do
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
 

lib/pleroma/notification.ex

@@ -9,8 +9,8 @@ defmodule Pleroma.Notification do
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
-    belongs_to(:activity, Pleroma.Activity)
+    belongs_to(:activity, Activity, type: Pleroma.FlakeId)
 
     timestamps()
   end
@@ -96,7 +96,7 @@ def dismiss(%{id: user_id} = _user, id) do
     end
   end
 
-  def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
       when type in ["Create", "Like", "Announce", "Follow"] do
     users = get_notified_from_activity(activity)
 

lib/pleroma/object.ex

@@ -85,7 +85,7 @@ def swap_object_with_tombstone(object) do
 
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
-         Repo.delete_all(Activity.all_non_create_by_object_ap_id_q(id)),
+         Repo.delete_all(Activity.by_object_ap_id(id)),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
       {:ok, object}
     end

lib/pleroma/reverse_proxy.ex

@@ -275,11 +275,24 @@ defp build_resp_headers(headers, opts) do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
+    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
-    if has_cache? do
+    cond do
+      has_cache? && has_cache_control? ->
         headers
-    else
+
-      List.keystore(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
+      has_cache? ->
+        # There's caching header present but no cache-control -- we need to explicitely override it to public
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+
+      true ->
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
     end
   end
 

lib/pleroma/uploaders/s3.ex

@@ -9,12 +9,20 @@ defmodule Pleroma.Uploaders.S3 do
   # The file name is re-encoded with S3's constraints here to comply with previous links with less strict filenames
   def get_file(file) do
     config = Pleroma.Config.get([__MODULE__])
+    bucket = Keyword.fetch!(config, :bucket)
+
+    bucket_with_namespace =
+      if namespace = Keyword.get(config, :bucket_namespace) do
+        namespace <> ":" <> bucket
+      else
+        bucket
+      end
 
     {:ok,
      {:url,
       Path.join([
         Keyword.fetch!(config, :public_endpoint),
-        Keyword.fetch!(config, :bucket),
+        bucket_with_namespace,
         strict_encode(URI.decode(file))
       ])}}
   end

lib/pleroma/uploaders/uploader.ex

@@ -27,18 +27,47 @@ defmodule Pleroma.Uploaders.Uploader do
     This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL.
   * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
   * `{:error, String.t}` error information if the file failed to be saved to the backend.
+  * `:wait_callback` will wait for an http post request at `/api/pleroma/upload_callback/:upload_path` and call the uploader's `http_callback/3` method.
 
 
   """
+  @type file_spec :: {:file | :url, String.t()}
   @callback put_file(Pleroma.Upload.t()) ::
-              :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()}
+              :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
+
+  @callback http_callback(Plug.Conn.t(), Map.t()) ::
+              {:ok, Plug.Conn.t()}
+              | {:ok, Plug.Conn.t(), file_spec()}
+              | {:error, Plug.Conn.t(), String.t()}
+  @optional_callbacks http_callback: 2
+
+  @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
 
-  @spec put_file(module(), Pleroma.Upload.t()) ::
-          {:ok, {:file | :url, String.t()}} | {:error, String.t()}
   def put_file(uploader, upload) do
     case uploader.put_file(upload) do
       :ok -> {:ok, {:file, upload.path}}
-      other -> other
+      :wait_callback -> handle_callback(uploader, upload)
+      {:ok, _} = ok -> ok
+      {:error, _} = error -> error
+    end
+  end
+
+  defp handle_callback(uploader, upload) do
+    :global.register_name({__MODULE__, upload.path}, self())
+
+    receive do
+      {__MODULE__, pid, conn, params} ->
+        case uploader.http_callback(conn, params) do
+          {:ok, conn, ok} ->
+            send(pid, {__MODULE__, conn})
+            {:ok, ok}
+
+          {:error, conn, error} ->
+            send(pid, {__MODULE__, conn})
+            {:error, error}
+        end
+    after
+      30_000 -> {:error, "Uploader callback timeout"}
     end
   end
 end

lib/pleroma/user.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.User do
 
   @type t :: %__MODULE__{}
 
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/
@@ -35,7 +37,7 @@ defmodule Pleroma.User do
     field(:avatar, :map)
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
-    field(:search_distance, :float, virtual: true)
+    field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
@@ -473,8 +475,7 @@ def get_cached_by_nickname_or_id(nickname_or_id) do
   def get_by_nickname(nickname) do
     Repo.get_by(User, nickname: nickname) ||
       if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
-        [local_nickname, _] = String.split(nickname, "@")
+        Repo.get_by(User, nickname: local_nickname(nickname))
-        Repo.get_by(User, nickname: local_nickname)
       end
   end
 
@@ -537,6 +538,12 @@ def get_followers(user, page \\ nil) do
     {:ok, Repo.all(q)}
   end
 
+  def get_followers_ids(user, page \\ nil) do
+    q = get_followers_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
   def get_friends_query(%User{id: id, following: following}, nil) do
     from(
       u in User,
@@ -561,6 +568,12 @@ def get_friends(user, page \\ nil) do
     {:ok, Repo.all(q)}
   end
 
+  def get_friends_ids(user, page \\ nil) do
+    q = get_friends_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
   def get_follow_requests_query(%User{} = user) do
     from(
       a in Activity,
@@ -692,37 +705,120 @@ def get_recipients_from_activity(%Activity{recipients: to}) do
     Repo.all(query)
   end
 
-  def search(query, resolve \\ false) do
+  def search(query, resolve \\ false, for_user \\ nil) do
-    # strip the beginning @ off if there is a query
+    # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
-    if resolve do
+    if resolve, do: User.get_or_fetch_by_nickname(query)
-      User.get_or_fetch_by_nickname(query)
+
+    fts_results = do_search(fts_search_subquery(query), for_user)
+
+    {:ok, trigram_results} =
+      Repo.transaction(fn ->
+        Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
+        do_search(trigram_search_subquery(query), for_user)
+      end)
+
+    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
   end
 
-    inner =
+  defp do_search(subquery, for_user, options \\ []) do
+    q =
+      from(
+        s in subquery(subquery),
+        order_by: [desc: s.search_rank],
+        limit: ^(options[:limit] || 20)
+      )
+
+    results =
+      q
+      |> Repo.all()
+      |> Enum.filter(&(&1.search_rank > 0))
+
+    boost_search_results(results, for_user)
+  end
+
+  defp fts_search_subquery(query) do
+    processed_query =
+      query
+      |> String.replace(~r/\W+/, " ")
+      |> String.trim()
+      |> String.split()
+      |> Enum.map(&(&1 <> ":*"))
+      |> Enum.join(" | ")
+
     from(
       u in User,
       select_merge: %{
-          search_distance:
+        search_rank:
           fragment(
-              "? <-> (? || coalesce(?, ''))",
+            """
+            ts_rank_cd(
+              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
+              to_tsquery('simple', ?),
+              32
+            )
+            """,
+            u.nickname,
+            u.name,
+            ^processed_query
+          )
+      },
+      where:
+        fragment(
+          """
+            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
+          """,
+          u.nickname,
+          u.name,
+          ^processed_query
+        )
+    )
+  end
+
+  defp trigram_search_subquery(query) do
+    from(
+      u in User,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
             ^query,
             u.nickname,
             u.name
           )
       },
-        where: not is_nil(u.nickname)
+      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^query)
     )
+  end
 
-    q =
+  defp boost_search_results(results, nil), do: results
-      from(
+
-        s in subquery(inner),
+  defp boost_search_results(results, for_user) do
-        order_by: s.search_distance,
+    friends_ids = get_friends_ids(for_user)
-        limit: 20
+    followers_ids = get_followers_ids(for_user)
+
+    Enum.map(
+      results,
+      fn u ->
+        search_rank_coef =
+          cond do
+            u.id in friends_ids ->
+              1.2
+
+            u.id in followers_ids ->
+              1.1
+
+            true ->
+              1
+          end
+
+        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
+      end
     )
-
+    |> Enum.sort_by(&(-&1.search_rank))
-    Repo.all(q)
   end
 
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
@@ -833,7 +929,7 @@ def local_user_query do
   def active_local_user_query do
     from(
       u in local_user_query(),
-      where: fragment("?->'deactivated' @> 'false'", u.info)
+      where: fragment("not (?->'deactivated' @> 'true')", u.info)
     )
   end
 
@@ -1023,7 +1119,7 @@ def parse_bio(bio, user) do
       end)
 
     bio
-    |> CommonUtils.format_input(mentions, tags, "text/plain")
+    |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full])
     |> Formatter.emojify(emoji)
   end
 
@@ -1074,6 +1170,16 @@ defp local_nickname_regex() do
     end
   end
 
+  def local_nickname(nickname_or_mention) do
+    nickname_or_mention
+    |> full_nickname()
+    |> String.split("@")
+    |> hd()
+  end
+
+  def full_nickname(nickname_or_mention),
+    do: String.trim_leading(nickname_or_mention, "@")
+
   def error_user(ap_id) do
     %User{
       name: ap_id,

lib/pleroma/user/info.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.User.Info do
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
     field(:hide_network, :boolean, default: false)
-    field(:pinned_activities, {:array, :integer}, default: [])
+    field(:pinned_activities, {:array, :string}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -92,7 +92,7 @@ def insert(map, local \\ true) when is_map(map) do
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
-    if activity.data["type"] in ["Create", "Announce"] do
+    if activity.data["type"] in ["Create", "Announce", "Delete"] do
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -103,6 +103,7 @@ def stream_out(activity) do
           Pleroma.Web.Streamer.stream("public:local", activity)
         end
 
+        if activity.data["type"] in ["Create"] do
           activity.data["object"]
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)
@@ -115,6 +116,7 @@ def stream_out(activity) do
               Pleroma.Web.Streamer.stream("public:local:media", activity)
             end
           end
+        end
       else
         if !Enum.member?(activity.data["cc"] || [], public) &&
              !Enum.member?(
@@ -138,8 +140,9 @@ def create(%{to: to, actor: actor, context: context, object: object} = params) d
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
-         :ok <- maybe_federate(activity),
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.increase_note_count(actor) do
+         {:ok, _actor} <- User.increase_note_count(actor),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -224,10 +227,11 @@ def announce(
         %User{ap_id: _} = user,
         %Object{data: %{"id" => _}} = object,
         activity_id \\ nil,
-        local \\ true
+        local \\ true,
+        public \\ true
       ) do
     with true <- is_public?(object),
-         announce_data <- make_announce_data(user, object, activity_id),
+         announce_data <- make_announce_data(user, object, activity_id, public),
          {:ok, activity} <- insert(announce_data, local),
          {:ok, object} <- add_announce_to_object(activity, object),
          :ok <- maybe_federate(activity) do
@@ -285,8 +289,9 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
 
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.decrease_note_count(user) do
+         {:ok, _actor} <- User.decrease_note_count(user),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -405,6 +410,8 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
     |> Enum.reverse()
   end
 
+  defp restrict_since(query, %{"since_id" => ""}), do: query
+
   defp restrict_since(query, %{"since_id" => since_id}) do
     from(activity in query, where: activity.id > ^since_id)
   end
@@ -460,6 +467,8 @@ defp restrict_local(query, %{"local_only" => true}) do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_max(query, %{"max_id" => ""}), do: query
+
   defp restrict_max(query, %{"max_id" => max_id}) do
     from(activity in query, where: activity.id < ^max_id)
   end
@@ -796,13 +805,24 @@ def fetch_and_contain_remote_object_from_id(id) do
     end
   end
 
-  def is_public?(%Object{data: %{"type" => "Tombstone"}}) do
+  def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
-    false
+  def is_public?(%Object{data: data}), do: is_public?(data)
+  def is_public?(%Activity{data: data}), do: is_public?(data)
+  def is_public?(%{"directMessage" => true}), do: false
+
+  def is_public?(data) do
+    "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
   end
 
-  def is_public?(activity) do
+  def is_private?(activity) do
-    "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
+    !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers"))
-                                                         (activity.data["cc"] || []))
+  end
+
+  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
+  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
+
+  def is_direct?(activity) do
+    !is_public?(activity) && !is_private?(activity)
   end
 
   def visible_for_user?(activity, nil) do

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
+  alias Pleroma.User
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  # XXX: this should become User.normalize_by_ap_id() or similar, really.
+  defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
+  defp normalize_by_ap_id(uri) when is_binary(uri), do: User.get_cached_by_ap_id(uri)
+  defp normalize_by_ap_id(_), do: nil
+
+  defp score_nickname("followbot@" <> _), do: 1.0
+  defp score_nickname("federationbot@" <> _), do: 1.0
+  defp score_nickname("federation_bot@" <> _), do: 1.0
+  defp score_nickname(_), do: 0.0
+
+  defp score_displayname("federation bot"), do: 1.0
+  defp score_displayname("federationbot"), do: 1.0
+  defp score_displayname("fedibot"), do: 1.0
+  defp score_displayname(_), do: 0.0
+
+  defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+    nick_score =
+      nickname
+      |> String.downcase()
+      |> score_nickname()
+
+    name_score =
+      displayname
+      |> String.downcase()
+      |> score_displayname()
+
+    nick_score + name_score
+  end
+
+  defp determine_if_followbot(_), do: 0.0
+
+  @impl true
+  def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
+    %User{} = actor = normalize_by_ap_id(actor_id)
+
+    score = determine_if_followbot(actor)
+
+    # TODO: scan biography data for keywords and score it somehow.
+    if score < 0.8 do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/relay.ex

@@ -40,7 +40,7 @@ def unfollow(target_instance) do
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
          %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
-      ActivityPub.announce(user, object)
+      ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -93,12 +93,47 @@ def fix_addressing_list(map, field) do
     end
   end
 
-  def fix_addressing(map) do
+  def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, explicit_mentions) do
-    map
+    explicit_to =
+      to
+      |> Enum.filter(fn x -> x in explicit_mentions end)
+
+    explicit_cc =
+      to
+      |> Enum.filter(fn x -> x not in explicit_mentions end)
+
+    final_cc =
+      (cc ++ explicit_cc)
+      |> Enum.uniq()
+
+    object
+    |> Map.put("to", explicit_to)
+    |> Map.put("cc", final_cc)
+  end
+
+  def fix_explicit_addressing(object, _explicit_mentions), do: object
+
+  # if directMessage flag is set to true, leave the addressing alone
+  def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
+
+  def fix_explicit_addressing(object) do
+    explicit_mentions =
+      object
+      |> Utils.determine_explicit_mentions()
+
+    explicit_mentions = explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+    object
+    |> fix_explicit_addressing(explicit_mentions)
+  end
+
+  def fix_addressing(object) do
+    object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
+    |> fix_explicit_addressing
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -141,7 +176,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object)
     case fetch_obj_helper(in_reply_to_id) do
       {:ok, replied_object} ->
         with %Activity{} = activity <-
-               Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do
+               Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
           object
           |> Map.put("inReplyTo", replied_object.data["id"])
           |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
@@ -334,7 +369,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
       Map.put(data, "actor", actor)
       |> fix_addressing
 
-    with nil <- Activity.get_create_activity_by_object_ap_id(object["id"]),
+    with nil <- Activity.get_create_by_object_ap_id(object["id"]),
          %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
@@ -348,6 +383,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
         additional:
           Map.take(data, [
             "cc",
+            "directMessage",
             "id"
           ])
       }
@@ -451,7 +487,8 @@ def handle_incoming(
     with actor <- get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
+         public <- ActivityPub.is_public?(data),
+         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
       {:ok, activity}
     else
       _e -> :error
@@ -863,15 +900,10 @@ defp user_upgrade_task(user) do
 
     maybe_retire_websub(user.ap_id)
 
-    # Only do this for recent activties, don't go through the whole db.
-    # Only look at the last 1000 activities.
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
-
     q =
       from(
         a in Activity,
         where: ^old_follower_address in a.recipients,
-        where: a.id > ^since,
         update: [
           set: [
             recipients:

lib/pleroma/web/activity_pub/utils.ex

@@ -25,6 +25,20 @@ def normalize_params(params) do
     Map.put(params, "actor", get_ap_id(params["actor"]))
   end
 
+  def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do
+    Map.put(object, "tag", [tag])
+    |> determine_explicit_mentions()
+  end
+
+  def determine_explicit_mentions(_), do: []
+
   defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll
   defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll
   defp recipient_in_collection(_, _), do: false
@@ -198,7 +212,7 @@ def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # Update activities that already had this. Could be done in a seperate process.
     # Alternatively, just don't do this and fetch the current object each time. Most
     # could probably be taken from cache.
-    relevant_activities = Activity.all_by_object_ap_id(id)
+    relevant_activities = Activity.get_all_create_by_object_ap_id(id)
 
     Enum.map(relevant_activities, fn activity ->
       new_activity_data = activity.data |> Map.put("object", object.data)
@@ -386,9 +400,10 @@ def get_existing_announce(actor, %{data: %{"id" => id}}) do
   """
   # for relayed messages, we only want to send to subscribers
   def make_announce_data(
-        %User{ap_id: ap_id, nickname: nil} = user,
+        %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        false
       ) do
     data = %{
       "type" => "Announce",
@@ -405,7 +420,8 @@ def make_announce_data(
   def make_announce_data(
         %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        true
       ) do
     data = %{
       "type" => "Announce",

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -160,7 +160,7 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => info.note_count,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
@@ -207,7 +207,7 @@ def render("inbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => -1,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do

lib/pleroma/web/common_api/common_api.ex

@@ -143,7 +143,7 @@ def post(user, %{"status" => status} = data) do
           actor: user,
           context: context,
           object: object,
-          additional: %{"cc" => cc}
+          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
         })
 
       res

lib/pleroma/web/common_api/utils.ex

@@ -14,13 +14,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_activity_by_object_ap_id(id)
+    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id(activity.data["object"])
       end
   end
 
@@ -116,16 +116,18 @@ def add_attachments(text, attachments) do
     Enum.join([text | attachment_text], "<br>")
   end
 
+  def format_input(text, mentions, tags, format, options \\ [])
+
   @doc """
   Formatting text to plain text.
   """
-  def format_input(text, mentions, tags, "text/plain") do
+  def format_input(text, mentions, tags, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
     |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
     |> Formatter.add_links()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end
@@ -133,24 +135,24 @@ def format_input(text, mentions, tags, "text/plain") do
   @doc """
   Formatting text to html.
   """
-  def format_input(text, mentions, _tags, "text/html") do
+  def format_input(text, mentions, _tags, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.finalize()
   end
 
   @doc """
   Formatting text to markdown.
   """
-  def format_input(text, mentions, tags, "text/markdown") do
+  def format_input(text, mentions, tags, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(mentions)
     |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -377,7 +377,7 @@ def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -386,7 +386,7 @@ def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def fav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -395,7 +395,7 @@ def fav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def unfav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _, _, %{data: %{"id" => id}}} <- CommonAPI.unfavorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -743,8 +743,7 @@ def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
         with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
-             %Activity{} = activity <-
+             %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
-               Activity.get_create_activity_by_object_ap_id(object.data["id"]),
              true <- ActivityPub.visible_for_user?(activity, user) do
           [activity]
         else
@@ -771,7 +770,7 @@ def status_search(user, query) do
   end
 
   def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
     statuses = status_search(user, query)
 
@@ -795,7 +794,7 @@ def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
   end
 
   def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
     statuses = status_search(user, query)
 
@@ -816,7 +815,7 @@ def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
   end
 
   def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
     res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
 
@@ -1138,7 +1137,7 @@ def empty_object(conn, _) do
 
   def render_notification(user, %{id: id, activity: activity, inserted_at: created_at} = _params) do
     actor = User.get_cached_by_ap_id(activity.data["actor"])
-    parent_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     mastodon_type = Activity.mastodon_notification_type(activity)
 
     response = %{

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -25,7 +25,7 @@ defp get_replied_to_activities(activities) do
         nil
     end)
     |> Enum.filter(& &1)
-    |> Activity.create_activity_by_object_id_query()
+    |> Activity.create_by_object_ap_id()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
       Map.put(acc, activity.data["object"]["id"], activity)
@@ -64,7 +64,7 @@ def render(
     user = get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
 
-    reblogged = Activity.get_create_activity_by_object_ap_id(object)
+    reblogged = Activity.get_create_by_object_ap_id(object)
     reblogged = render("status.json", Map.put(opts, :activity, reblogged))
 
     mentions =
@@ -209,7 +209,7 @@ def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
 
   def get_reply_to(%{data: %{"object" => object}}, _) do
     if object["inReplyTo"] && object["inReplyTo"] != "" do
-      Activity.get_create_activity_by_object_ap_id(object["inReplyTo"])
+      Activity.get_create_by_object_ap_id(object["inReplyTo"])
     else
       nil
     end
@@ -231,6 +231,9 @@ def get_visibility(object) do
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"
 
+      length(cc) > 0 ->
+        "private"
+
       true ->
         "direct"
     end

lib/pleroma/web/oauth/authorization.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:token, :string)
     field(:valid_until, :naive_datetime)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/token.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:valid_until, :naive_datetime)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/ostatus/activity_representer.ex

@@ -183,7 +183,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho
     _in_reply_to = get_in_reply_to(activity.data)
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
-    retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
 
     retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -86,7 +86,7 @@ def add_external_url(note, entry) do
   end
 
   def fetch_replied_to_activity(entry, inReplyTo) do
-    with %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(inReplyTo) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(inReplyTo) do
       activity
     else
       _e ->
@@ -103,7 +103,7 @@ def fetch_replied_to_activity(entry, inReplyTo) do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_activity_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),

lib/pleroma/web/ostatus/ostatus.ex

@@ -148,7 +148,7 @@ def get_or_try_fetching(entry) do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     else
       _ ->

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -93,8 +93,7 @@ def object(conn, %{"uuid" => uuid}) do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <-
+           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
-             {:activity, Activity.get_create_activity_by_object_ap_id(id)},
            {_, true} <- {:public?, ActivityPub.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do

lib/pleroma/web/push/subscription.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do
   alias Pleroma.Web.Push.Subscription
 
   schema "push_subscriptions" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
     field(:endpoint, :string)
     field(:key_p256dh, :string)

lib/pleroma/web/router.ex

@@ -107,6 +107,11 @@ defmodule Pleroma.Web.Router do
     get("/captcha", UtilController, :captcha)
   end
 
+  scope "/api/pleroma", Pleroma.Web do
+    pipe_through(:pleroma_api)
+    post("/uploader_callback/:upload_path", UploaderController, :callback)
+  end
+
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
     delete("/user", AdminAPIController, :user_delete)

lib/pleroma/web/streamer.ex

@@ -205,6 +205,15 @@ def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = ite
     end)
   end
 
+  def push_to_socket(topics, topic, %Activity{id: id, data: %{"type" => "Delete"}}) do
+    Enum.each(topics[topic] || [], fn socket ->
+      send(
+        socket.transport_pid,
+        {:text, %{event: "delete", payload: to_string(id)} |> Jason.encode!()}
+      )
+    end)
+  end
+
   def push_to_socket(topics, topic, item) do
     Enum.each(topics[topic] || [], fn socket ->
       # Get the current user so we have up-to-date blocks etc.

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -70,14 +70,14 @@ def unblock(%User{} = blocker, params) do
 
   def repeat(%User{} = user, ap_id_or_id) do
     with {:ok, _announce, %{data: %{"id" => id}}} <- CommonAPI.repeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
 
   def unrepeat(%User{} = user, ap_id_or_id) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
@@ -92,14 +92,14 @@ def unpin(%User{} = user, ap_id_or_id) do
 
   def fav(%User{} = user, ap_id_or_id) do
     with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
 
   def unfav(%User{} = user, ap_id_or_id) do
     with {:ok, _unfav, _fav, %{data: %{"id" => id}}} <- CommonAPI.unfavorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -265,8 +265,6 @@ def fetch_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    id = String.to_integer(id)
-
     with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
@@ -330,54 +328,57 @@ def upload_json(%{assigns: %{user: user}} = conn, %{"media" => media}) do
   end
 
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_activity_by_object_ap_id(id)
+    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
 
     if activity.data["type"] == "Create" do
       activity
     else
-      Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+      Activity.get_create_by_object_ap_id(activity.data["object"])
     end
   end
 
   def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.fav(user, id) do
-         {:ok, activity} <- TwitterAPI.fav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unfav(user, id) do
-         {:ok, activity} <- TwitterAPI.unfav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.repeat(user, id) do
-         {:ok, activity} <- TwitterAPI.repeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
-         {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.pin(user, id) do
-         {:ok, activity} <- TwitterAPI.pin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -388,8 +389,7 @@ def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def unpin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unpin(user, id) do
-         {:ok, activity} <- TwitterAPI.unpin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -556,7 +556,6 @@ def friend_requests(conn, params) do
 
   def approve_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          {:ok, follower} <- User.maybe_follow(follower, followed),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
@@ -578,7 +577,6 @@ def approve_friend_request(conn, %{"user_id" => uid} = _params) do
 
   def deny_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
@@ -675,7 +673,7 @@ def search(%{assigns: %{user: user}} = conn, %{"q" => _query} = params) do
   end
 
   def search_user(%{assigns: %{user: user}} = conn, %{"query" => query}) do
-    users = User.search(query, true)
+    users = User.search(query, true, user)
 
     conn
     |> put_view(UserView)

lib/pleroma/web/twitter_api/views/activity_view.ex

@@ -168,7 +168,7 @@ def render("activity.json", %{activity: %{data: %{"type" => "Follow"}} = activit
   def render("activity.json", %{activity: %{data: %{"type" => "Announce"}} = activity} = opts) do
     user = get_user(activity.data["actor"], opts)
     created_at = activity.data["published"] |> Utils.date_to_asctime()
-    announced_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    announced_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
 
     text = "#{user.nickname} retweeted a status."
 
@@ -192,7 +192,7 @@ def render("activity.json", %{activity: %{data: %{"type" => "Announce"}} = activ
 
   def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity} = opts) do
     user = get_user(activity.data["actor"], opts)
-    liked_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    liked_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     liked_activity_id = if liked_activity, do: liked_activity.id, else: nil
 
     created_at =

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -108,6 +108,7 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
       "locked" => user.info.locked,
       "default_scope" => user.info.default_scope,
       "no_rich_text" => user.info.no_rich_text,
+      "hide_network" => user.info.hide_network,
       "fields" => fields,
 
       # Pleroma extension

lib/pleroma/web/uploader_controller.ex

@@ -0,0 +1,25 @@
+defmodule Pleroma.Web.UploaderController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Uploaders.Uploader
+
+  def callback(conn, params = %{"upload_path" => upload_path}) do
+    process_callback(conn, :global.whereis_name({Uploader, upload_path}), params)
+  end
+
+  def callbacks(conn, _) do
+    send_resp(conn, 400, "bad request")
+  end
+
+  defp process_callback(conn, pid, params) when is_pid(pid) do
+    send(pid, {Uploader, self(), conn, params})
+
+    receive do
+      {Uploader, conn} -> conn
+    end
+  end
+
+  defp process_callback(conn, _, _) do
+    send_resp(conn, 400, "bad request")
+  end
+end

lib/pleroma/web/websub/websub_client_subscription.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
     field(:state, :string)
     field(:subscribers, {:array, :string}, default: [])
     field(:hub, :string)
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
 
     timestamps()
   end

priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs

@@ -0,0 +1,125 @@
+defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do
+  use Ecto.Migration
+  alias Pleroma.Clippy
+  require Integer
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  # This migrates from int serial IDs to custom Flake:
+  #   1- create a temporary uuid column
+  #   2- fill this column with compatibility ids (see below)
+  #   3- remove pkeys constraints
+  #   4- update relation pkeys with the new ids
+  #   5- rename the temporary column to id
+  #   6- re-create the constraints
+  def change do
+    # Old serial int ids are transformed to 128bits with extra padding.
+    # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility
+    # with previously issued ids.
+    #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+    #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+
+    clippy = start_clippy_heartbeats()
+
+    # Lock both tables to avoid a running server to meddling with our transaction
+    execute "LOCK TABLE activities;"
+    execute "LOCK TABLE users;"
+
+    execute """
+      ALTER TABLE activities
+      DROP CONSTRAINT activities_pkey CASCADE,
+      ALTER COLUMN id DROP default,
+      ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid),
+      ADD PRIMARY KEY (id);
+    """
+
+    execute """
+    ALTER TABLE users
+    DROP CONSTRAINT users_pkey CASCADE,
+    ALTER COLUMN id DROP default,
+    ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid),
+    ADD PRIMARY KEY (id);
+    """
+
+    execute "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);"
+
+    # Fkeys:
+    # Activities - Referenced by:
+    #   TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE
+    # Users - Referenced by:
+    #  TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+
+    execute """
+    ALTER TABLE notifications
+    ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid),
+    ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;
+    """
+
+    for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do
+      execute """
+      ALTER TABLE #{table}
+      ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid),
+      ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
+      """
+    end
+
+    flush()
+
+    stop_clippy_heartbeats(clippy)
+  end
+
+  defp start_clippy_heartbeats() do
+    count = from(a in "activities", select: count(a.id)) |> Repo.one!
+
+    if count > 5000 do
+      heartbeat_interval = :timer.minutes(2) + :timer.seconds(30)
+      all_tips = Clippy.tips() ++ [
+        "The migration is still running, maybe it's time for another “tea”?",
+        "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!",
+        "Nothing and everything.\n\nI still work.",
+        "Pleroma runs on a Raspberry Pi!\n\n  … but this migration will take forever if you\nactually run on a raspberry pi",
+        "Status? Stati? Post? Note? Toot?\nRepeat? Reboost? Boost? Retweet? Retoot??\n\nI-I'm confused.",
+      ]
+
+      heartbeat = fn(heartbeat, runs, all_tips, tips) ->
+        tips = if Integer.is_even(runs) do
+          tips = if tips == [], do: all_tips, else: tips
+          [tip | tips] = Enum.shuffle(tips)
+          Clippy.puts(tip)
+          tips
+        else
+          IO.puts "\n -- #{DateTime.to_string(DateTime.utc_now())} Migration still running, please wait…\n"
+          tips
+        end
+        :timer.sleep(heartbeat_interval)
+        heartbeat.(heartbeat, runs + 1, all_tips, tips)
+      end
+
+      Clippy.puts [
+        [:red, :bright, "It looks like you are running an older instance!"],
+        [""],
+        [:bright, "This migration may take a long time", :reset, " -- so you probably should"],
+        ["go drink a cofe, or a tea, or a beer, a whiskey, a vodka,"],
+        ["while it runs to deal with your temporary fediverse pause!"]
+      ]
+      :timer.sleep(heartbeat_interval)
+      spawn_link(fn() -> heartbeat.(heartbeat, 1, all_tips, []) end)
+    end
+  end
+
+  defp stop_clippy_heartbeats(pid) do
+    if pid do
+      Process.unlink(pid)
+      Process.exit(pid, :kill)
+      Clippy.puts [[:green, :bright, "Hurray!!", "", "", "Migration completed!"]]
+    end
+  end
+
+end

priv/repo/migrations/20190115085500_create_user_fts_index.exs

@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.CreateUserFtsIndex do
+  use Ecto.Migration
+
+  def change do
+    create index(
+             :users,
+             [
+               """
+               (setweight(to_tsvector('simple', regexp_replace(nickname, '\\W', ' ', 'g')), 'A') ||
+               setweight(to_tsvector('simple', regexp_replace(coalesce(name, ''), '\\W', ' ', 'g')), 'B'))
+               """
+             ],
+             name: :users_fts_index,
+             using: :gin
+           )
+  end
+end

priv/repo/migrations/20190118074940_fix_user_trigram_index.exs

@@ -0,0 +1,22 @@
+defmodule Pleroma.Repo.Migrations.FixUserTrigramIndex do
+  use Ecto.Migration
+
+  def up do
+    drop_if_exists(index(:users, [], name: :users_trigram_index))
+
+    create(
+      index(:users, ["(trim(nickname || ' ' || coalesce(name, ''))) gist_trgm_ops"],
+        name: :users_trigram_index,
+        using: :gist
+      )
+    )
+  end
+
+  def down do
+    drop_if_exists(index(:users, [], name: :users_trigram_index))
+
+    create(
+      index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist)
+    )
+  end
+end

priv/repo/migrations/20190122153157_update_activity_visibility.exs

@@ -0,0 +1,36 @@
+defmodule Pleroma.Repo.Migrations.UpdateActivityVisibility do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def up do
+    definition = """
+    create or replace function activity_visibility(actor varchar, recipients varchar[], data jsonb) returns varchar as $$
+    DECLARE
+      fa varchar;
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+    BEGIN
+      SELECT COALESCE(users.follower_address, '') into fa from users where users.ap_id = actor;
+
+      IF data->'to' ? public THEN
+        RETURN 'public';
+      ELSIF data->'cc' ? public THEN
+        RETURN 'unlisted';
+      ELSIF ARRAY[fa] && recipients THEN
+        RETURN 'private';
+      ELSIF not(ARRAY[fa, public] && recipients) THEN
+        RETURN 'direct';
+      ELSE
+        RETURN 'unknown';
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE SECURITY DEFINER;
+    """
+
+    execute(definition)
+
+  end
+
+  def down do
+
+  end
+end

priv/repo/migrations/20190123125839_fix_info_ids.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.FixInfoIds do
+  use Ecto.Migration
+
+  def change do
+    execute(
+      "update users set info = jsonb_set(info, '{id}', to_jsonb(uuid_generate_v4())) where info->'id' is null;"
+    )
+  end
+end

priv/repo/migrations/20190124131141_update_activity_visibility_again.exs

@@ -0,0 +1,37 @@
+defmodule Pleroma.Repo.Migrations.UpdateActivityVisibilityAgain do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def up do
+    definition = """
+    create or replace function activity_visibility(actor varchar, recipients varchar[], data jsonb) returns varchar as $$
+    DECLARE
+      fa varchar;
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+    BEGIN
+      SELECT COALESCE(users.follower_address, '') into fa from public.users where users.ap_id = actor;
+
+      IF data->'to' ? public THEN
+        RETURN 'public';
+      ELSIF data->'cc' ? public THEN
+        RETURN 'unlisted';
+      ELSIF ARRAY[fa] && recipients THEN
+        RETURN 'private';
+      ELSIF not(ARRAY[fa, public] && recipients) THEN
+        RETURN 'direct';
+      ELSE
+        RETURN 'unknown';
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE SECURITY DEFINER;
+    """
+
+    execute(definition)
+
+  end
+
+  def down do
+
+  end
+
+end

priv/static/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.c52cbb57296d5c682ff405d562e83a9b.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.e833e1c75fbc9f2b69b4.js></script><script type=text/javascript src=/static/js/vendor.b6e63c523d95d763c254.js></script><script type=text/javascript src=/static/js/app.1c83eacd8eddeef56c69.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.3d3e30a9afb8c41739656f496e8c79e6.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.6802874440f1854a6844.js></script><script type=text/javascript src=/static/js/vendor.61fd03d8471aaadcf63c.js></script><script type=text/javascript src=/static/js/app.5ed04a69c4a6e8b1f455.js></script></body></html>

priv/static/schemas/litepub-0.1.jsonld

@@ -17,7 +17,9 @@
             "toot": "http://joinmastodon.org/ns#",
             "totalItems": "as:totalItems",
             "value": "schema:value",
-            "sensitive": "as:sensitive"
+            "sensitive": "as:sensitive",
+            "litepub": "http://litepub.social/ns#",
+            "directMessage": "litepub:directMessage"
         }
     ]
 }

priv/static/static/config.json

@@ -18,5 +18,6 @@
   "hideUserStats": false,
   "loginMethod": "password",
   "webPushNotifications": false,
-  "noAttachmentLinks": false
+  "noAttachmentLinks": false,
+  "nsfwCensorImage": ""
 }

priv/static/static/js/manifest.6802874440f1854a6844.js

@@ -0,0 +1,2 @@
+!function(e){function t(a){if(r[a])return r[a].exports;var n=r[a]={exports:{},id:a,loaded:!1};return e[a].call(n.exports,n,n.exports,t),n.loaded=!0,n.exports}var a=window.webpackJsonp;window.webpackJsonp=function(c,o){for(var p,l,s=0,i=[];s<c.length;s++)l=c[s],n[l]&&i.push.apply(i,n[l]),n[l]=0;for(p in o)Object.prototype.hasOwnProperty.call(o,p)&&(e[p]=o[p]);for(a&&a(c,o);i.length;)i.shift().call(null,t);if(o[0])return r[0]=0,t(0)};var r={},n={0:0};t.e=function(e,a){if(0===n[e])return a.call(null,t);if(void 0!==n[e])n[e].push(a);else{n[e]=[a];var r=document.getElementsByTagName("head")[0],c=document.createElement("script");c.type="text/javascript",c.charset="utf-8",c.async=!0,c.src=t.p+"static/js/"+e+"."+{1:"61fd03d8471aaadcf63c",2:"5ed04a69c4a6e8b1f455"}[e]+".js",r.appendChild(c)}},t.m=e,t.c=r,t.p="/"}([]);
+//# sourceMappingURL=manifest.6802874440f1854a6844.js.map

priv/static/static/js/manifest.e833e1c75fbc9f2b69b4.js

@@ -1,2 +0,0 @@
-!function(e){function t(r){if(n[r])return n[r].exports;var a=n[r]={exports:{},id:r,loaded:!1};return e[r].call(a.exports,a,a.exports,t),a.loaded=!0,a.exports}var r=window.webpackJsonp;window.webpackJsonp=function(c,o){for(var p,l,s=0,d=[];s<c.length;s++)l=c[s],a[l]&&d.push.apply(d,a[l]),a[l]=0;for(p in o)Object.prototype.hasOwnProperty.call(o,p)&&(e[p]=o[p]);for(r&&r(c,o);d.length;)d.shift().call(null,t);if(o[0])return n[0]=0,t(0)};var n={},a={0:0};t.e=function(e,r){if(0===a[e])return r.call(null,t);if(void 0!==a[e])a[e].push(r);else{a[e]=[r];var n=document.getElementsByTagName("head")[0],c=document.createElement("script");c.type="text/javascript",c.charset="utf-8",c.async=!0,c.src=t.p+"static/js/"+e+"."+{1:"b6e63c523d95d763c254",2:"1c83eacd8eddeef56c69"}[e]+".js",n.appendChild(c)}},t.m=e,t.c=n,t.p="/"}([]);
-//# sourceMappingURL=manifest.e833e1c75fbc9f2b69b4.js.map

priv/static/sw.js

@@ -1,4 +1,4 @@
-var serviceWorkerOption = {"assets":["/static/img/nsfw.50fd83c.png","/static/js/manifest.e833e1c75fbc9f2b69b4.js","/static/js/vendor.b6e63c523d95d763c254.js","/static/js/app.1c83eacd8eddeef56c69.js","/static/css/app.c52cbb57296d5c682ff405d562e83a9b.css"]};
+var serviceWorkerOption = {"assets":["/static/img/nsfw.50fd83c.png","/static/js/manifest.6802874440f1854a6844.js","/static/js/vendor.61fd03d8471aaadcf63c.js","/static/js/app.5ed04a69c4a6e8b1f455.js","/static/css/app.3d3e30a9afb8c41739656f496e8c79e6.css"]};
         
         !function(e){function n(r){if(t[r])return t[r].exports;var o=t[r]={exports:{},id:r,loaded:!1};return e[r].call(o.exports,o,o.exports,n),o.loaded=!0,o.exports}var t={};return n.m=e,n.c=t,n.p="/",n(0)}([function(e,n,t){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(){return u.default.getItem("vuex-lz").then(function(e){return e.config.webPushNotifications})}function i(){return clients.matchAll({includeUncontrolled:!0}).then(function(e){return e.filter(function(e){var n=e.type;return"window"===n})})}var a=t(1),u=r(a);self.addEventListener("push",function(e){e.data&&e.waitUntil(o().then(function(n){return n&&i().then(function(n){var t=e.data.json();if(0===n.length)return self.registration.showNotification(t.title,t)})}))}),self.addEventListener("notificationclick",function(e){e.notification.close(),e.waitUntil(i().then(function(e){for(var n=0;n<e.length;n++){var t=e[n];if("/"===t.url&&"focus"in t)return t.focus()}if(clients.openWindow)return clients.openWindow("/")}))})},function(e,n){/*!
 	    localForage -- Offline Storage, Improved

test/activity_test.exs

@@ -16,7 +16,7 @@ test "returns an activity by it's AP id" do
 
   test "returns activities by it's objects AP ids" do
     activity = insert(:note_activity)
-    [found_activity] = Activity.all_by_object_ap_id(activity.data["object"]["id"])
+    [found_activity] = Activity.get_all_create_by_object_ap_id(activity.data["object"]["id"])
 
     assert activity == found_activity
   end
@@ -24,7 +24,7 @@ test "returns activities by it's objects AP ids" do
   test "returns the activity that created an object" do
     activity = insert(:note_activity)
 
-    found_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]["id"])
+    found_activity = Activity.get_create_by_object_ap_id(activity.data["object"]["id"])
 
     assert activity == found_activity
   end

test/flake_id_test.exs

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeIdTest do
+  use Pleroma.DataCase
+  import Kernel, except: [to_string: 1]
+  import Pleroma.FlakeId
+
+  describe "fake flakes (compatibility with older serial integers)" do
+    test "from_string/1" do
+      fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
+      assert from_string("42") == fake_flake
+    end
+
+    test "zero or -1 is a null flake" do
+      fake_flake = <<0::integer-size(128)>>
+      assert from_string("0") == fake_flake
+      assert from_string("-1") == fake_flake
+    end
+
+    test "to_string/1" do
+      fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
+      assert to_string(fake_flake) == "42"
+    end
+  end
+
+  test "ecto type behaviour" do
+    flake = <<0, 0, 1, 104, 80, 229, 2, 235, 140, 22, 69, 201, 53, 210, 0, 0>>
+    flake_s = "9eoozpwTul5mjSEDRI"
+
+    assert cast(flake) == {:ok, flake_s}
+    assert cast(flake_s) == {:ok, flake_s}
+
+    assert load(flake) == {:ok, flake_s}
+    assert load(flake_s) == {:ok, flake_s}
+
+    assert dump(flake_s) == {:ok, flake}
+    assert dump(flake) == {:ok, flake}
+  end
+end

test/formatter_test.exs

@@ -124,7 +124,7 @@ test "turning urls into links" do
   end
 
   describe "add_user_links" do
-    test "gives a replacement for user links" do
+    test "gives a replacement for user links, using local nicknames in user links text" do
       text = "@gsimg According to @archa_eme_, that is @daggsy. Also hello @archaeme@archae.me"
       gsimg = insert(:user, %{nickname: "gsimg"})
 

test/integration/mastodon_websocket_test.exs

@@ -66,13 +66,10 @@ test "receives well formatted events" do
     assert json["payload"]
     assert {:ok, json} = Jason.decode(json["payload"])
 
-    # Note: we remove the "statuses_count" from this result as it changes in the meantime
-
     view_json =
       Pleroma.Web.MastodonAPI.StatusView.render("status.json", activity: activity, for: nil)
       |> Jason.encode!()
       |> Jason.decode!()
-      |> put_in(["account", "statuses_count"], 0)
 
     assert json == view_json
   end

test/user_test.exs

@@ -775,14 +775,61 @@ test "User.delete() plugs any possible zombie objects" do
   end
 
   describe "User.search" do
-    test "finds a user, ranking by similarity" do
+    test "finds a user by full or partial nickname" do
-      _user = insert(:user, %{name: "lain"})
+      user = insert(:user, %{nickname: "john"})
-      _user_two = insert(:user, %{name: "ean"})
-      _user_three = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"})
-      user_four = insert(:user, %{nickname: "lain@pleroma.soykaf.com"})
 
-      assert user_four ==
+      Enum.each(["john", "jo", "j"], fn query ->
-               User.search("lain@ple") |> List.first() |> Map.put(:search_distance, nil)
+        assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil)
+      end)
+    end
+
+    test "finds a user by full or partial name" do
+      user = insert(:user, %{name: "John Doe"})
+
+      Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query ->
+        assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil)
+      end)
+    end
+
+    test "finds users, preferring nickname matches over name matches" do
+      u1 = insert(:user, %{name: "lain", nickname: "nick1"})
+      u2 = insert(:user, %{nickname: "lain", name: "nick1"})
+
+      assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id)
+    end
+
+    test "finds users, considering density of matched tokens" do
+      u1 = insert(:user, %{name: "Bar Bar plus Word Word"})
+      u2 = insert(:user, %{name: "Word Word Bar Bar Bar"})
+
+      assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id)
+    end
+
+    test "finds users, ranking by similarity" do
+      u1 = insert(:user, %{name: "lain"})
+      _u2 = insert(:user, %{name: "ean"})
+      u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"})
+      u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"})
+
+      assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple"), & &1.id)
+    end
+
+    test "finds users, handling misspelled requests" do
+      u1 = insert(:user, %{name: "lain"})
+
+      assert [u1.id] == Enum.map(User.search("laiin"), & &1.id)
+    end
+
+    test "finds users, boosting ranks of friends and followers" do
+      u1 = insert(:user)
+      u2 = insert(:user, %{name: "Doe"})
+      follower = insert(:user, %{name: "Doe"})
+      friend = insert(:user, %{name: "Doe"})
+
+      {:ok, follower} = User.follow(follower, u1)
+      {:ok, u1} = User.follow(u1, friend)
+
+      assert [friend.id, follower.id, u2.id] == Enum.map(User.search("doe", false, u1), & &1.id)
     end
 
     test "finds a user whose name is nil" do
@@ -792,7 +839,15 @@ test "finds a user whose name is nil" do
       assert user_two ==
                User.search("lain@pleroma.soykaf.com")
                |> List.first()
-               |> Map.put(:search_distance, nil)
+               |> Map.put(:search_rank, nil)
+    end
+
+    test "does not yield false-positive matches" do
+      insert(:user, %{name: "John Doe"})
+
+      Enum.each(["mary", "a", ""], fn query ->
+        assert [] == User.search(query)
+      end)
     end
   end
 
@@ -874,4 +929,19 @@ test "returns true when the account is unauthenticated and being viewed by a pri
       Pleroma.Config.put([:instance, :account_activation_required], false)
     end
   end
+
+  describe "parse_bio/2" do
+    test "preserves hosts in user links text" do
+      remote_user = insert(:user, local: false, nickname: "nick@domain.com")
+      user = insert(:user)
+      bio = "A.k.a. @nick@domain.com"
+
+      expected_text =
+        "A.k.a. <span class='h-card'><a data-user='#{remote_user.id}' class='u-url mention' href='#{
+          remote_user.ap_id
+        }'>" <> "@<span>nick@domain.com</span></a></span>"
+
+      assert expected_text == User.parse_bio(bio, user)
+    end
+  end
 end

test/web/activity_pub/activity_pub_test.exs

@@ -216,7 +216,7 @@ test "doesn't return blocked activities" do
 
     {:ok, user} = User.block(user, %{ap_id: activity_three.data["actor"]})
     {:ok, _announce, %{data: %{"id" => id}}} = CommonAPI.repeat(activity_three.id, booster)
-    %Activity{} = boost_activity = Activity.get_create_activity_by_object_ap_id(id)
+    %Activity{} = boost_activity = Activity.get_create_by_object_ap_id(id)
     activity_three = Repo.get(Activity, activity_three.id)
 
     activities = ActivityPub.fetch_activities([], %{"blocking_user" => user})
@@ -330,7 +330,7 @@ test "adds a like activity to the db" do
       assert like_activity == same_like_activity
       assert object.data["likes"] == [user.ap_id]
 
-      [note_activity] = Activity.all_by_object_ap_id(object.data["id"])
+      [note_activity] = Activity.get_all_create_by_object_ap_id(object.data["id"])
       assert note_activity.data["object"]["like_count"] == 1
 
       {:ok, _like_activity, object} = ActivityPub.like(user_two, object)
@@ -445,7 +445,7 @@ test "it fetches an object" do
       {:ok, object} =
         ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
 
-      assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
+      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
       assert activity.data["id"]
 
       {:ok, object_again} =
@@ -459,7 +459,7 @@ test "it fetches an object" do
 
     test "it works with objects only available via Ostatus" do
       {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
-      assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
+      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
       assert activity.data["id"]
 
       {:ok, object_again} =

test/web/activity_pub/mrf/anti_followbot_policy_test.exs

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  alias Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy
+
+  describe "blocking based on attributes" do
+    test "matches followbots by nickname" do
+      actor = insert(:user, %{nickname: "followbot@example.com"})
+      target = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "type" => "Follow",
+        "actor" => actor.ap_id,
+        "object" => target.ap_id,
+        "id" => "https://example.com/activities/1234"
+      }
+
+      {:reject, nil} = AntiFollowbotPolicy.filter(message)
+    end
+
+    test "matches followbots by display name" do
+      actor = insert(:user, %{name: "Federation Bot"})
+      target = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "type" => "Follow",
+        "actor" => actor.ap_id,
+        "object" => target.ap_id,
+        "id" => "https://example.com/activities/1234"
+      }
+
+      {:reject, nil} = AntiFollowbotPolicy.filter(message)
+    end
+  end
+
+  test "it allows non-followbots" do
+    actor = insert(:user)
+    target = insert(:user)
+
+    message = %{
+      "@context" => "https://www.w3.org/ns/activitystreams",
+      "type" => "Follow",
+      "actor" => actor.ap_id,
+      "object" => target.ap_id,
+      "id" => "https://example.com/activities/1234"
+    }
+
+    {:ok, _} = AntiFollowbotPolicy.filter(message)
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -51,7 +51,7 @@ test "it fetches replied-to activities if we don't have them" do
       {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
 
       assert activity =
-               Activity.get_create_activity_by_object_ap_id(
+               Activity.get_create_by_object_ap_id(
                  "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
                )
 
@@ -162,6 +162,36 @@ test "it works for incoming notices with url not being a string (prismo)" do
       assert data["object"]["url"] == "https://prismo.news/posts/83"
     end
 
+    test "it cleans up incoming notices which are not really DMs" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      to = [user.ap_id, other_user.ap_id]
+
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+        |> Map.put("to", to)
+        |> Map.put("cc", [])
+
+      object =
+        data["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", [])
+
+      data = Map.put(data, "object", object)
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+      assert data["to"] == []
+      assert data["cc"] == to
+
+      object = data["object"]
+
+      assert object["to"] == []
+      assert object["cc"] == to
+    end
+
     test "it works for incoming follow requests" do
       user = insert(:user)
 
@@ -263,7 +293,7 @@ test "it works for incoming announces" do
       assert data["object"] ==
                "http://mastodon.example.org/users/admin/statuses/99541947525187367"
 
-      assert Activity.get_create_activity_by_object_ap_id(data["object"])
+      assert Activity.get_create_by_object_ap_id(data["object"])
     end
 
     test "it works for incoming announces with an existing activity" do
@@ -285,7 +315,23 @@ test "it works for incoming announces with an existing activity" do
 
       assert data["object"] == activity.data["object"]["id"]
 
-      assert Activity.get_create_activity_by_object_ap_id(data["object"]).id == activity.id
+      assert Activity.get_create_by_object_ap_id(data["object"]).id == activity.id
+    end
+
+    test "it does not clobber the addressing on announce activities" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "hey"})
+
+      data =
+        File.read!("test/fixtures/mastodon-announce.json")
+        |> Poison.decode!()
+        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"])
+        |> Map.put("cc", [])
+
+      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+      assert data["to"] == ["http://mastodon.example.org/users/admin/followers"]
     end
 
     test "it works for incoming update activities" do
@@ -856,6 +902,34 @@ test "it adds like collection to object" do
       assert modified["object"]["likes"]["type"] == "OrderedCollection"
       assert modified["object"]["likes"]["totalItems"] == 0
     end
+
+    test "the directMessage flag is present" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu :moominmamma:"})
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == false
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "@#{other_user.nickname} :moominmamma:"})
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == false
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" => "@#{other_user.nickname} :moominmamma:",
+          "visibility" => "direct"
+        })
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert modified["directMessage"] == true
+    end
   end
 
   describe "user upgrade" do

test/web/activity_pub/utils_test.exs

@@ -0,0 +1,57 @@
+defmodule Pleroma.Web.ActivityPub.UtilsTest do
+  use Pleroma.DataCase
+  alias Pleroma.Web.ActivityPub.Utils
+
+  describe "determine_explicit_mentions()" do
+    test "works with an object that has mentions" do
+      object = %{
+        "tag" => [
+          %{
+            "type" => "Mention",
+            "href" => "https://example.com/~alyssa",
+            "name" => "Alyssa P. Hacker"
+          }
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"]
+    end
+
+    test "works with an object that does not have mentions" do
+      object = %{
+        "tag" => [
+          %{"type" => "Hashtag", "href" => "https://example.com/tag/2hu", "name" => "2hu"}
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+
+    test "works with an object that has mentions and other tags" do
+      object = %{
+        "tag" => [
+          %{
+            "type" => "Mention",
+            "href" => "https://example.com/~alyssa",
+            "name" => "Alyssa P. Hacker"
+          },
+          %{"type" => "Hashtag", "href" => "https://example.com/tag/2hu", "name" => "2hu"}
+        ]
+      }
+
+      assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"]
+    end
+
+    test "works with an object that has no tags" do
+      object = %{}
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+
+    test "works with an object that has only IR tags" do
+      object = %{"tag" => ["2hu"]}
+
+      assert Utils.determine_explicit_mentions(object) == []
+    end
+  end
+end

test/web/common_api/common_api_test.exs

@@ -17,6 +17,13 @@ test "it de-duplicates tags" do
     assert activity.data["object"]["tag"] == ["2hu"]
   end
 
+  test "it adds emoji in the object" do
+    user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"})
+
+    assert activity.data["object"]["emoji"]["moominmamma"]
+  end
+
   test "it adds emoji when updating profiles" do
     user = insert(:user, %{name: ":karjalanpiirakka:"})
 

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
   alias Pleroma.Web.{OStatus, CommonAPI}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.MastodonAPI.FilterView
+  alias Ecto.Changeset
   import Pleroma.Factory
   import ExUnit.CaptureLog
   import Tesla.Mock
@@ -1483,6 +1484,16 @@ test "get instance information", %{conn: conn} do
 
     {:ok, _} = TwitterAPI.create_status(user, %{"status" => "cofe"})
 
+    # Stats should count users with missing or nil `info.deactivated` value
+    user = Repo.get(User, user.id)
+    info_change = Changeset.change(user.info, %{deactivated: nil})
+
+    {:ok, _user} =
+      user
+      |> Changeset.change()
+      |> Changeset.put_embed(:info, info_change)
+      |> User.update_and_set_cache()
+
     Pleroma.Stats.update_stats()
 
     conn = get(conn, "/api/v1/instance")

test/web/mastodon_api/status_view_test.exs

@@ -202,7 +202,7 @@ test "a peertube video" do
         "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
       )
 
-    %Activity{} = activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
+    %Activity{} = activity = Activity.get_create_by_object_ap_id(object.data["id"])
 
     represented = StatusView.render("status.json", %{for: user, activity: activity})
 

test/web/streamer_test.exs

@@ -6,7 +6,8 @@ defmodule Pleroma.Web.StreamerTest do
   use Pleroma.DataCase
 
   alias Pleroma.Web.Streamer
-  alias Pleroma.{List, User}
+  alias Pleroma.List
+  alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   import Pleroma.Factory
 
@@ -35,6 +36,28 @@ test "it sends to public" do
     Streamer.push_to_socket(topics, "public", activity)
 
     Task.await(task)
+
+    task =
+      Task.async(fn ->
+        assert_receive {:text, _}, 4_000
+      end)
+
+    fake_socket = %{
+      transport_pid: task.pid,
+      assigns: %{
+        user: user
+      }
+    }
+
+    {:ok, activity} = CommonAPI.delete(activity.id, other_user)
+
+    topics = %{
+      "public" => [fake_socket]
+    }
+
+    Streamer.push_to_socket(topics, "public", activity)
+
+    Task.await(task)
   end
 
   test "it doesn't send to blocked users" do

test/web/twitter_api/twitter_api_controller_test.exs

@@ -797,7 +797,7 @@ test "with credentials, invalid activity", %{conn: conn, user: current_user} do
         |> with_credentials(current_user.nickname, "test")
         |> post("/api/favorites/create/1.json")
 
-      assert json_response(conn, 500)
+      assert json_response(conn, 400)
     end
   end
 
@@ -1621,7 +1621,7 @@ test "it approves a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
@@ -1644,7 +1644,7 @@ test "it denies a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
@@ -1655,16 +1655,16 @@ test "it denies a friend request" do
   describe "GET /api/pleroma/search_user" do
     test "it returns users, ordered by similarity", %{conn: conn} do
       user = insert(:user, %{name: "eal"})
-      user_two = insert(:user, %{name: "ean"})
+      user_two = insert(:user, %{name: "eal me"})
-      user_three = insert(:user, %{name: "ebn"})
+      _user_three = insert(:user, %{name: "zzz"})
 
       resp =
         conn
-        |> get(twitter_api_search__path(conn, :search_user), query: "eal")
+        |> get(twitter_api_search__path(conn, :search_user), query: "eal me")
         |> json_response(200)
 
-      assert length(resp) == 3
+      assert length(resp) == 2
-      assert [user.id, user_two.id, user_three.id] == Enum.map(resp, fn %{"id" => id} -> id end)
+      assert [user_two.id, user.id] == Enum.map(resp, fn %{"id" => id} -> id end)
     end
   end
 

test/web/twitter_api/twitter_api_test.exs

@@ -451,7 +451,7 @@ test "fetches a user by uri" do
       assert represented["id"] == UserView.render("show.json", %{user: remote, for: user})["id"]
 
       # Also fetches the feed.
-      # assert Activity.get_create_activity_by_object_ap_id("tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status")
+      # assert Activity.get_create_by_object_ap_id("tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status")
     end
   end
 end

test/web/twitter_api/views/activity_view_test.exs

@@ -344,7 +344,7 @@ test "a peertube video" do
         "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
       )
 
-    %Activity{} = activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
+    %Activity{} = activity = Activity.get_create_by_object_ap_id(object.data["id"])
 
     result = ActivityView.render("activity.json", activity: activity)
 

test/web/twitter_api/views/user_view_test.exs

@@ -100,6 +100,7 @@ test "A user" do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -146,6 +147,7 @@ test "A user for a given other follower", %{user: user} do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -193,6 +195,7 @@ test "A user that follows you", %{user: user} do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -254,6 +257,7 @@ test "A blocked user for the blocker" do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,