Flake Ids for Users and Activities

lib/pleroma/PasswordResetToken.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do
   alias Pleroma.{User, PasswordResetToken, Repo}
 
   schema "password_reset_tokens" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:token, :string)
     field(:used, :boolean, default: false)
 

lib/pleroma/activity.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{

lib/pleroma/application.ex

@@ -99,6 +99,7 @@ def start(_type, _args) do
           ],
           id: :cachex_idem
         ),
+        worker(Pleroma.FlakeId, []),
         worker(Pleroma.Web.Federator.RetryQueue, []),
         worker(Pleroma.Web.Federator, []),
         worker(Pleroma.Stats, []),

lib/pleroma/filter.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Filter do
   alias Pleroma.{User, Repo}
 
   schema "filters" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -0,0 +1,181 @@
+defmodule Pleroma.FlakeId do
+  @moduledoc """
+  Flake is a decentralized, k-ordered id generation service.
+
+  Adapted from:
+
+  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
+  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
+  """
+
+  @type t :: binary
+
+  @behaviour Ecto.Type
+  use GenServer
+  require Logger
+  alias __MODULE__
+  import Kernel, except: [to_string: 1]
+
+  defstruct node: nil, time: 0, sq: 0
+
+  @doc "Converts a binary Flake to a String"
+  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
+    Kernel.to_string(id)
+  end
+
+  def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do
+    encode_base62(flake)
+  end
+
+  def to_string(s), do: s
+
+  def from_string(<<id::integer-size(64)>>) do
+    <<0::integer-size(64), id::integer-size(64)>>
+  end
+
+  for i <- [-1, 0] do
+    def from_string(unquote(i)), do: <<0::integer-size(128)>>
+    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
+  end
+
+  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
+    case Integer.parse(string) do
+      {id, _} -> <<0::integer-size(64), id::integer-size(64)>>
+      _ -> nil
+    end
+  end
+
+  def from_string(string) do
+    string |> decode_base62 |> from_integer
+  end
+
+  def to_integer(<<integer::integer-size(128)>>), do: integer
+
+  def from_integer(integer) do
+    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
+      <<integer::integer-size(128)>>
+  end
+
+  @doc "Generates a Flake"
+  @spec get :: binary
+  def get, do: to_string(:gen_server.call(:flake, :get))
+
+  # -- Ecto.Type API
+  @impl Ecto.Type
+  def type, do: :uuid
+
+  @impl Ecto.Type
+  def cast(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def load(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def dump(value) do
+    {:ok, FlakeId.from_string(value)}
+  end
+
+  def autogenerate(), do: get()
+
+  # -- GenServer API
+  def start_link do
+    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
+  end
+
+  @impl GenServer
+  def init([]) do
+    {:ok, %FlakeId{node: mac(), time: time()}}
+  end
+
+  @impl GenServer
+  def handle_call(:get, _from, state) do
+    {flake, new_state} = get(time(), state)
+    {:reply, flake, new_state}
+  end
+
+  # Matches when the calling time is the same as the state time. Incr. sq
+  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
+    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Matches when the times are different, reset sq
+  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
+    new_state = %FlakeId{time: newtime, node: node, sq: 0}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Error when clock is running backwards
+  defp get(newtime, %FlakeId{time: time}) when newtime < time do
+    {:error, :clock_running_backwards}
+  end
+
+  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
+    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
+  end
+
+  defp nthchar_base62(n) when n <= 9, do: ?0 + n
+  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
+  defp nthchar_base62(n), do: ?a + n - 36
+
+  defp encode_base62(<<integer::integer-size(128)>>) do
+    integer
+    |> encode_base62([])
+    |> List.to_string()
+  end
+
+  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
+  defp encode_base62(int, []) when int == 0, do: '0'
+  defp encode_base62(int, acc) when int == 0, do: acc
+
+  defp encode_base62(int, acc) do
+    r = rem(int, 62)
+    id = div(int, 62)
+    acc = [nthchar_base62(r) | acc]
+    encode_base62(id, acc)
+  end
+
+  defp decode_base62(s) do
+    decode_base62(String.to_charlist(s), 0)
+  end
+
+  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
+    do: decode_base62(cs, 62 * acc + (c - ?0))
+
+  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
+    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
+
+  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
+    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
+
+  defp decode_base62([], acc), do: acc
+
+  defp time do
+    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
+    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
+  end
+
+  defp mac do
+    {:ok, addresses} = :inet.getifaddrs()
+
+    ifaces_with_mac =
+      Enum.reduce(addresses, [], fn {iface, attrs}, acc ->
+        if attrs[:hwaddr], do: [iface | acc], else: acc
+      end)
+
+    iface = Enum.at(ifaces_with_mac, :rand.uniform(length(ifaces_with_mac)) - 1)
+    mac(iface)
+  end
+
+  defp mac(name) do
+    {:ok, addresses} = :inet.getifaddrs()
+    proplist = :proplists.get_value(name, addresses)
+    hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6)
+    <<worker::integer-size(48)>> = :binary.list_to_bin(hwaddr)
+    worker
+  end
+end

lib/pleroma/list.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.List do
   alias Pleroma.{User, Repo, Activity}
 
   schema "lists" do
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
 

lib/pleroma/notification.ex

@@ -9,8 +9,8 @@ defmodule Pleroma.Notification do
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
-    belongs_to(:activity, Pleroma.Activity)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:activity, Activity, type: Pleroma.FlakeId)
 
     timestamps()
   end
@@ -96,7 +96,7 @@ def dismiss(%{id: user_id} = _user, id) do
     end
   end
 
-  def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
       when type in ["Create", "Like", "Announce", "Follow"] do
     users = get_notified_from_activity(activity)
 

lib/pleroma/user.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.User do
 
   @type t :: %__MODULE__{}
 
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -900,15 +900,10 @@ defp user_upgrade_task(user) do
 
     maybe_retire_websub(user.ap_id)
 
-    # Only do this for recent activties, don't go through the whole db.
-    # Only look at the last 1000 activities.
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
-
     q =
       from(
         a in Activity,
         where: ^old_follower_address in a.recipients,
-        where: a.id > ^since,
         update: [
           set: [
             recipients:

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -160,7 +160,7 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => info.note_count,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
@@ -207,7 +207,7 @@ def render("inbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => -1,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do

lib/pleroma/web/oauth/authorization.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:token, :string)
     field(:valid_until, :naive_datetime)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/token.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:valid_until, :naive_datetime)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/push/subscription.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do
   alias Pleroma.Web.Push.Subscription
 
   schema "push_subscriptions" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
     field(:endpoint, :string)
     field(:key_p256dh, :string)

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -265,8 +265,6 @@ def fetch_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    id = String.to_integer(id)
-
     with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
@@ -340,38 +338,42 @@ def get_by_id_or_ap_id(id) do
   end
 
   def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.fav(user, id) do
+    with {:ok, activity} <- TwitterAPI.fav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.unfav(user, id) do
+    with {:ok, activity} <- TwitterAPI.unfav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.repeat(user, id) do
+    with {:ok, activity} <- TwitterAPI.repeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
+    with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
@@ -556,7 +558,6 @@ def friend_requests(conn, params) do
 
   def approve_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          {:ok, follower} <- User.maybe_follow(follower, followed),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
@@ -578,7 +579,6 @@ def approve_friend_request(conn, %{"user_id" => uid} = _params) do
 
   def deny_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),

lib/pleroma/web/websub/websub_client_subscription.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
     field(:state, :string)
     field(:subscribers, {:array, :string}, default: [])
     field(:hub, :string)
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
 
     timestamps()
   end

priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs

@@ -0,0 +1,52 @@
+defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do
+  use Ecto.Migration
+
+  # This migrates from int serial IDs to custom Flake:
+  #   1- create a temporary uuid column
+  #   2- fill this column with compatibility ids (see below)
+  #   3- remove pkeys constraints
+  #   4- update relation pkeys with the new ids
+  #   5- rename the temporary column to id
+  #   6- re-create the constraints
+  def change do
+    # Old serial int ids are transformed to 128bits with extra padding.
+    # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility
+    # with previously issued ids.
+    #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+    #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+
+    execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;"
+    execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;"
+
+    execute "ALTER TABLE activities ALTER COLUMN id DROP default;"
+    execute "ALTER TABLE users ALTER COLUMN id DROP default;"
+
+    execute "ALTER TABLE activities ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+    execute "ALTER TABLE users ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+
+    execute "ALTER TABLE activities ADD PRIMARY KEY (id);"
+    execute "ALTER TABLE users ADD PRIMARY KEY (id);"
+
+    # Fkeys:
+    # Activities - Referenced by:
+    #   TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE
+    # Users - Referenced by:
+    #  TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+
+    execute "ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid);"
+    execute "ALTER TABLE notifications ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;"
+
+    for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do
+      execute "ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid);"
+      execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;"
+    end
+
+  end
+end

test/web/twitter_api/twitter_api_controller_test.exs

@@ -797,7 +797,7 @@ test "with credentials, invalid activity", %{conn: conn, user: current_user} do
         |> with_credentials(current_user.nickname, "test")
         |> post("/api/favorites/create/1.json")
 
-      assert json_response(conn, 500)
+      assert json_response(conn, 400)
     end
   end
 
@@ -1621,7 +1621,7 @@ test "it approves a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
@@ -1644,7 +1644,7 @@ test "it denies a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]