From 28d77e373cbaf0908f86973a873c9bfd6c3221cb Mon Sep 17 00:00:00 2001 From: href Date: Wed, 9 Jan 2019 16:08:24 +0100 Subject: [PATCH 01/12] Flake Ids for Users and Activities --- lib/pleroma/PasswordResetToken.ex | 2 +- lib/pleroma/activity.ex | 1 + lib/pleroma/application.ex | 1 + lib/pleroma/filter.ex | 2 +- lib/pleroma/flake_id.ex | 181 ++++++++++++++++++ lib/pleroma/list.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/user.ex | 2 + .../web/activity_pub/transmogrifier.ex | 5 - .../web/activity_pub/views/user_view.ex | 4 +- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/token.ex | 2 +- lib/pleroma/web/push/subscription.ex | 2 +- .../web/twitter_api/twitter_api_controller.ex | 24 +-- .../web/websub/websub_client_subscription.ex | 2 +- ...18172826_users_and_activities_flake_id.exs | 52 +++++ .../twitter_api_controller_test.exs | 6 +- 17 files changed, 264 insertions(+), 32 deletions(-) create mode 100644 lib/pleroma/flake_id.ex create mode 100644 priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs diff --git a/lib/pleroma/PasswordResetToken.ex b/lib/pleroma/PasswordResetToken.ex index 1dccdadae..c3c0384d2 100644 --- a/lib/pleroma/PasswordResetToken.ex +++ b/lib/pleroma/PasswordResetToken.ex @@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do alias Pleroma.{User, PasswordResetToken, Repo} schema "password_reset_tokens" do - belongs_to(:user, User) + belongs_to(:user, User, type: Pleroma.FlakeId) field(:token, :string) field(:used, :boolean, default: false) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 8fd0311d2..d907791b5 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Activity do import Ecto.Query @type t :: %__MODULE__{} + @primary_key {:id, Pleroma.FlakeId, autogenerate: true} # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19 @mastodon_notification_types %{ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ad2797209..47c0e5b68 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -99,6 +99,7 @@ defmodule Pleroma.Application do ], id: :cachex_idem ), + worker(Pleroma.FlakeId, []), worker(Pleroma.Web.Federator.RetryQueue, []), worker(Pleroma.Web.Federator, []), worker(Pleroma.Stats, []), diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index df5374a5c..308bd70e1 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Filter do alias Pleroma.{User, Repo} schema "filters" do - belongs_to(:user, User) + belongs_to(:user, User, type: Pleroma.FlakeId) field(:filter_id, :integer) field(:hide, :boolean, default: false) field(:whole_word, :boolean, default: true) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex new file mode 100644 index 000000000..3c72807ca --- /dev/null +++ b/lib/pleroma/flake_id.ex @@ -0,0 +1,181 @@ +defmodule Pleroma.FlakeId do + @moduledoc """ + Flake is a decentralized, k-ordered id generation service. + + Adapted from: + + * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License, + * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0 + """ + + @type t :: binary + + @behaviour Ecto.Type + use GenServer + require Logger + alias __MODULE__ + import Kernel, except: [to_string: 1] + + defstruct node: nil, time: 0, sq: 0 + + @doc "Converts a binary Flake to a String" + def to_string(<<0::integer-size(64), id::integer-size(64)>>) do + Kernel.to_string(id) + end + + def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do + encode_base62(flake) + end + + def to_string(s), do: s + + def from_string(<>) do + <<0::integer-size(64), id::integer-size(64)>> + end + + for i <- [-1, 0] do + def from_string(unquote(i)), do: <<0::integer-size(128)>> + def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> + end + + def from_string(string) when is_binary(string) and byte_size(string) < 18 do + case Integer.parse(string) do + {id, _} -> <<0::integer-size(64), id::integer-size(64)>> + _ -> nil + end + end + + def from_string(string) do + string |> decode_base62 |> from_integer + end + + def to_integer(<>), do: integer + + def from_integer(integer) do + <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> = + <> + end + + @doc "Generates a Flake" + @spec get :: binary + def get, do: to_string(:gen_server.call(:flake, :get)) + + # -- Ecto.Type API + @impl Ecto.Type + def type, do: :uuid + + @impl Ecto.Type + def cast(value) do + {:ok, FlakeId.to_string(value)} + end + + @impl Ecto.Type + def load(value) do + {:ok, FlakeId.to_string(value)} + end + + @impl Ecto.Type + def dump(value) do + {:ok, FlakeId.from_string(value)} + end + + def autogenerate(), do: get() + + # -- GenServer API + def start_link do + :gen_server.start_link({:local, :flake}, __MODULE__, [], []) + end + + @impl GenServer + def init([]) do + {:ok, %FlakeId{node: mac(), time: time()}} + end + + @impl GenServer + def handle_call(:get, _from, state) do + {flake, new_state} = get(time(), state) + {:reply, flake, new_state} + end + + # Matches when the calling time is the same as the state time. Incr. sq + defp get(time, %FlakeId{time: time, node: node, sq: seq}) do + new_state = %FlakeId{time: time, node: node, sq: seq + 1} + {gen_flake(new_state), new_state} + end + + # Matches when the times are different, reset sq + defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do + new_state = %FlakeId{time: newtime, node: node, sq: 0} + {gen_flake(new_state), new_state} + end + + # Error when clock is running backwards + defp get(newtime, %FlakeId{time: time}) when newtime < time do + {:error, :clock_running_backwards} + end + + defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do + <> + end + + defp nthchar_base62(n) when n <= 9, do: ?0 + n + defp nthchar_base62(n) when n <= 35, do: ?A + n - 10 + defp nthchar_base62(n), do: ?a + n - 36 + + defp encode_base62(<>) do + integer + |> encode_base62([]) + |> List.to_string() + end + + defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc) + defp encode_base62(int, []) when int == 0, do: '0' + defp encode_base62(int, acc) when int == 0, do: acc + + defp encode_base62(int, acc) do + r = rem(int, 62) + id = div(int, 62) + acc = [nthchar_base62(r) | acc] + encode_base62(id, acc) + end + + defp decode_base62(s) do + decode_base62(String.to_charlist(s), 0) + end + + defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9, + do: decode_base62(cs, 62 * acc + (c - ?0)) + + defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z, + do: decode_base62(cs, 62 * acc + (c - ?A + 10)) + + defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z, + do: decode_base62(cs, 62 * acc + (c - ?a + 36)) + + defp decode_base62([], acc), do: acc + + defp time do + {mega_seconds, seconds, micro_seconds} = :erlang.timestamp() + 1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000) + end + + defp mac do + {:ok, addresses} = :inet.getifaddrs() + + ifaces_with_mac = + Enum.reduce(addresses, [], fn {iface, attrs}, acc -> + if attrs[:hwaddr], do: [iface | acc], else: acc + end) + + iface = Enum.at(ifaces_with_mac, :rand.uniform(length(ifaces_with_mac)) - 1) + mac(iface) + end + + defp mac(name) do + {:ok, addresses} = :inet.getifaddrs() + proplist = :proplists.get_value(name, addresses) + hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6) + <> = :binary.list_to_bin(hwaddr) + worker + end +end diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index a75dc006e..ca66c6916 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -8,7 +8,7 @@ defmodule Pleroma.List do alias Pleroma.{User, Repo, Activity} schema "lists" do - belongs_to(:user, Pleroma.User) + belongs_to(:user, User, type: Pleroma.FlakeId) field(:title, :string) field(:following, {:array, :string}, default: []) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index c7d01f63b..2c8f60f19 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Notification do schema "notifications" do field(:seen, :boolean, default: false) - belongs_to(:user, Pleroma.User) - belongs_to(:activity, Pleroma.Activity) + belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:activity, Activity, type: Pleroma.FlakeId) timestamps() end @@ -96,7 +96,7 @@ defmodule Pleroma.Notification do end end - def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity) + def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity) when type in ["Create", "Like", "Announce", "Follow"] do users = get_notified_from_activity(activity) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 18137106e..b006f9f19 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -17,6 +17,8 @@ defmodule Pleroma.User do @type t :: %__MODULE__{} + @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5d3feccfe..e646de608 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -900,15 +900,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do maybe_retire_websub(user.ap_id) - # Only do this for recent activties, don't go through the whole db. - # Only look at the last 1000 activities. - since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000 - q = from( a in Activity, where: ^old_follower_address in a.recipients, - where: a.id > ^since, update: [ set: [ recipients: diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index fe8248107..dcf681b6d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -160,7 +160,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "partOf" => iri, "totalItems" => info.note_count, "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id - 1}" + "next" => "#{iri}?max_id=#{min_id}" } if max_qid == nil do @@ -207,7 +207,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "partOf" => iri, "totalItems" => -1, "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id - 1}" + "next" => "#{iri}?max_id=#{min_id}" } if max_qid == nil do diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index cc4b74bc5..f8c65602d 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do field(:token, :string) field(:valid_until, :naive_datetime) field(:used, :boolean, default: false) - belongs_to(:user, Pleroma.User) + belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index f0ebc63f6..4e01b123b 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:token, :string) field(:refresh_token, :string) field(:valid_until, :naive_datetime) - belongs_to(:user, Pleroma.User) + belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/push/subscription.ex b/lib/pleroma/web/push/subscription.ex index 82b30950c..bd9d9f3a7 100644 --- a/lib/pleroma/web/push/subscription.ex +++ b/lib/pleroma/web/push/subscription.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do alias Pleroma.Web.Push.Subscription schema "push_subscriptions" do - belongs_to(:user, User) + belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:token, Token) field(:endpoint, :string) field(:key_p256dh, :string) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index ede079963..43f8d64d0 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -265,8 +265,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do - id = String.to_integer(id) - with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id), activities <- ActivityPub.fetch_activities_for_context(context, %{ @@ -340,38 +338,42 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.fav(user, id) do + with {:ok, activity} <- TwitterAPI.fav(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) + else + _ -> json_reply(conn, 400, Jason.encode!(%{})) end end def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.unfav(user, id) do + with {:ok, activity} <- TwitterAPI.unfav(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) + else + _ -> json_reply(conn, 400, Jason.encode!(%{})) end end def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.repeat(user, id) do + with {:ok, activity} <- TwitterAPI.repeat(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) + else + _ -> json_reply(conn, 400, Jason.encode!(%{})) end end def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.unrepeat(user, id) do + with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) + else + _ -> json_reply(conn, 400, Jason.encode!(%{})) end end @@ -556,7 +558,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do def approve_friend_request(conn, %{"user_id" => uid} = _params) do with followed <- conn.assigns[:user], - uid when is_number(uid) <- String.to_integer(uid), %User{} = follower <- Repo.get(User, uid), {:ok, follower} <- User.maybe_follow(follower, followed), %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), @@ -578,7 +579,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do def deny_friend_request(conn, %{"user_id" => uid} = _params) do with followed <- conn.assigns[:user], - uid when is_number(uid) <- String.to_integer(uid), %User{} = follower <- Repo.get(User, uid), %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 105b0069f..969ee0684 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do field(:state, :string) field(:subscribers, {:array, :string}, default: []) field(:hub, :string) - belongs_to(:user, User) + belongs_to(:user, User, type: Pleroma.FlakeId) timestamps() end diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs new file mode 100644 index 000000000..6e5dfaa77 --- /dev/null +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -0,0 +1,52 @@ +defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do + use Ecto.Migration + + # This migrates from int serial IDs to custom Flake: + # 1- create a temporary uuid column + # 2- fill this column with compatibility ids (see below) + # 3- remove pkeys constraints + # 4- update relation pkeys with the new ids + # 5- rename the temporary column to id + # 6- re-create the constraints + def change do + # Old serial int ids are transformed to 128bits with extra padding. + # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility + # with previously issued ids. + #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + + execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;" + execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;" + + execute "ALTER TABLE activities ALTER COLUMN id DROP default;" + execute "ALTER TABLE users ALTER COLUMN id DROP default;" + + execute "ALTER TABLE activities ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + execute "ALTER TABLE users ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + + execute "ALTER TABLE activities ADD PRIMARY KEY (id);" + execute "ALTER TABLE users ADD PRIMARY KEY (id);" + + # Fkeys: + # Activities - Referenced by: + # TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE + # Users - Referenced by: + # TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + # TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + # TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + # TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) + # TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) + # TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) + # TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + # TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) + + execute "ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid);" + execute "ALTER TABLE notifications ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;" + + for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do + execute "ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid);" + execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;" + end + + end +end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index f22cdd870..863abd10f 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -797,7 +797,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> with_credentials(current_user.nickname, "test") |> post("/api/favorites/create/1.json") - assert json_response(conn, 500) + assert json_response(conn, 400) end end @@ -1621,7 +1621,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn = build_conn() |> assign(:user, user) - |> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)}) + |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id}) assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] @@ -1644,7 +1644,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn = build_conn() |> assign(:user, user) - |> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)}) + |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id}) assert relationship = json_response(conn, 200) assert other_user.id == relationship["id"] From 9d63b27dcd61dff61b77b6df0ef8a4cf8d7c87e3 Mon Sep 17 00:00:00 2001 From: href Date: Tue, 15 Jan 2019 16:18:18 +0100 Subject: [PATCH 02/12] Test FlakeID old id compat & Ecto type --- lib/pleroma/flake_id.ex | 6 ++++++ test/flake_id_test.exs | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 test/flake_id_test.exs diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index 3c72807ca..f23c6d4b0 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.FlakeId do @moduledoc """ Flake is a decentralized, k-ordered id generation service. @@ -38,6 +42,8 @@ defmodule Pleroma.FlakeId do def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> end + def from_string(flake = <<_::integer-size(128)>>), do: flake + def from_string(string) when is_binary(string) and byte_size(string) < 18 do case Integer.parse(string) do {id, _} -> <<0::integer-size(64), id::integer-size(64)>> diff --git a/test/flake_id_test.exs b/test/flake_id_test.exs new file mode 100644 index 000000000..e480fbdf3 --- /dev/null +++ b/test/flake_id_test.exs @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.FlakeIdTest do + use Pleroma.DataCase + import Kernel, except: [to_string: 1] + import Pleroma.FlakeId + + describe "fake flakes (compatibility with older serial integers)" do + test "from_string/1" do + fake_flake = <<0::integer-size(64), 42::integer-size(64)>> + assert from_string("42") == fake_flake + end + + test "zero or -1 is a null flake" do + fake_flake = <<0::integer-size(128)>> + assert from_string("0") == fake_flake + assert from_string("-1") == fake_flake + end + + test "to_string/1" do + fake_flake = <<0::integer-size(64), 42::integer-size(64)>> + assert to_string(fake_flake) == "42" + end + end + + test "ecto type behaviour" do + flake = <<0, 0, 1, 104, 80, 229, 2, 235, 140, 22, 69, 201, 53, 210, 0, 0>> + flake_s = "9eoozpwTul5mjSEDRI" + + assert cast(flake) == {:ok, flake_s} + assert cast(flake_s) == {:ok, flake_s} + + assert load(flake) == {:ok, flake_s} + assert load(flake_s) == {:ok, flake_s} + + assert dump(flake_s) == {:ok, flake} + assert dump(flake) == {:ok, flake} + end +end From cdc5e6ff5ca97f9123998f8af5f7b9d89b485a0c Mon Sep 17 00:00:00 2001 From: href Date: Tue, 15 Jan 2019 16:39:23 +0100 Subject: [PATCH 03/12] ActivityPub: restrict_since/restrict_max: ignore empty param --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 82fffd324..85fa83e2b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -410,6 +410,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Enum.reverse() end + defp restrict_since(query, %{"since_id" => ""}), do: query + defp restrict_since(query, %{"since_id" => since_id}) do from(activity in query, where: activity.id > ^since_id) end @@ -465,6 +467,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_local(query, _), do: query + defp restrict_max(query, %{"max_id" => ""}), do: query + defp restrict_max(query, %{"max_id" => max_id}) do from(activity in query, where: activity.id < ^max_id) end From 422e60ad7693ecc06f5fe6dfd65a61caf338e6b6 Mon Sep 17 00:00:00 2001 From: href Date: Tue, 15 Jan 2019 16:42:33 +0100 Subject: [PATCH 04/12] 2019 --- lib/pleroma/flake_id.ex | 2 +- test/flake_id_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index f23c6d4b0..af04fc6a4 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.FlakeId do diff --git a/test/flake_id_test.exs b/test/flake_id_test.exs index e480fbdf3..8e969fd1c 100644 --- a/test/flake_id_test.exs +++ b/test/flake_id_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.FlakeIdTest do From e36a434b990cfdd650aed1710c55b1f62bd93ccb Mon Sep 17 00:00:00 2001 From: href Date: Wed, 16 Jan 2019 10:01:15 +0100 Subject: [PATCH 05/12] FlakeId.from_string/1: remove old, un-needed conversion --- lib/pleroma/flake_id.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index af04fc6a4..6b83ee890 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -33,10 +33,6 @@ defmodule Pleroma.FlakeId do def to_string(s), do: s - def from_string(<>) do - <<0::integer-size(64), id::integer-size(64)>> - end - for i <- [-1, 0] do def from_string(unquote(i)), do: <<0::integer-size(128)>> def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> From 973c9eed18b2e69c6dd43eacaa040cfeabc772bc Mon Sep 17 00:00:00 2001 From: href Date: Wed, 16 Jan 2019 16:15:46 +0100 Subject: [PATCH 06/12] Treat User.Info pinned ids as strings --- lib/pleroma/user/info.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fb1791c20..c6c923aac 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -31,7 +31,7 @@ defmodule Pleroma.User.Info do field(:hub, :string, default: nil) field(:salmon, :string, default: nil) field(:hide_network, :boolean, default: false) - field(:pinned_activities, {:array, :integer}, default: []) + field(:pinned_activities, {:array, :string}, default: []) # Found in the wild # ap_id -> Where is this used? diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 43f8d64d0..65ae7aabf 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -378,8 +378,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.pin(user, id) do + with {:ok, activity} <- TwitterAPI.pin(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) @@ -390,8 +389,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def unpin(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, - {:ok, activity} <- TwitterAPI.unpin(user, id) do + with {:ok, activity} <- TwitterAPI.unpin(user, id) do conn |> put_view(ActivityView) |> render("activity.json", %{activity: activity, for: user}) From 465fb4327dab5eccdb8b88d6da5670007bd7110e Mon Sep 17 00:00:00 2001 From: href Date: Mon, 21 Jan 2019 13:10:48 +0100 Subject: [PATCH 07/12] Lock activities/users table during flake migration. --- .../20181218172826_users_and_activities_flake_id.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 6e5dfaa77..39d45f7e8 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -15,6 +15,10 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + # Lock both tables to avoid a running server to meddling with our transaction + execute "LOCK TABLE activities;" + execute "LOCK TABLE users;" + execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;" execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;" From a92c43bc4be914ac9f8118cae18bc82e2b9d1664 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 23 Jan 2019 11:21:52 +0100 Subject: [PATCH 08/12] Clippy! --- lib/pleroma/clippy.ex | 144 ++++++++++++++++++ ...18172826_users_and_activities_flake_id.exs | 55 +++++++ 2 files changed, 199 insertions(+) create mode 100644 lib/pleroma/clippy.ex diff --git a/lib/pleroma/clippy.ex b/lib/pleroma/clippy.ex new file mode 100644 index 000000000..5e82ed8e2 --- /dev/null +++ b/lib/pleroma/clippy.ex @@ -0,0 +1,144 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Clippy do + @moduledoc false + # No software is complete until they have a Clippy implementation. + # A ballmer peak _may_ be required to change this module. + + def tip() do + tips() + |> Enum.random() + |> puts() + end + + def tips() do + host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + + [ + "“πλήρωμα” is “pleroma” in greek", + "For an extended Pleroma Clippy Experience, use the “Redmond” themes in Pleroma FE settings", + "Staff accounts and MRF policies of Pleroma instances are disclosed on the NodeInfo endpoints for easy transparency!\n +- https://catgirl.science/misc/nodeinfo.lua?#{host} +- https://fediverse.network/#{host}/federation", + "Pleroma can federate to the Dark Web!\n +- Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor) +- i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation", + "Lists of Pleroma instances are available at:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma", + "Pleroma uses the LitePub protocol - https://litepub.social", + "To receive more federated posts, subscribe to relays!\n +- How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment +- Relays: https://fediverse.network/activityrelay" + ] + end + + @spec puts(String.t() | [[IO.ANSI.ansicode() | String.t(), ...], ...]) :: nil + def puts(text_or_lines) do + import IO.ANSI + + lines = + if is_binary(text_or_lines) do + String.split(text_or_lines, ~r/\n/) + else + text_or_lines + end + + longest_line_size = + lines + |> Enum.map(&charlist_count_text/1) + |> Enum.sort(&>=/2) + |> List.first() + + pad_text = longest_line_size + + pad = + for(_ <- 1..pad_text, do: "_") + |> Enum.join("") + + pad_spaces = + for(_ <- 1..pad_text, do: " ") + |> Enum.join("") + + spaces = " " + + pre_lines = [ + " / \\#{spaces} _#{pad}___", + " | |#{spaces} / #{pad_spaces} \\" + ] + + for l <- pre_lines do + IO.puts(l) + end + + clippy_lines = [ + " #{bright()}@ @#{reset()}#{spaces} ", + " || ||#{spaces}", + " || || <--", + " |\\_/| ", + " \\___/ " + ] + + noclippy_line = " " + + env = %{ + max_size: pad_text, + pad: pad, + pad_spaces: pad_spaces, + spaces: spaces, + pre_lines: pre_lines, + noclippy_line: noclippy_line + } + + clippy_line(lines, clippy_lines, env) + rescue + e -> + IO.puts("(Clippy crashed, sorry: #{inspect(e)})") + IO.puts(text_or_lines) + end + + defp clippy_line([line | lines], [prefix | clippy_lines], env) do + IO.puts([prefix <> "| ", rpad_line(line, env.max_size)]) + clippy_line(lines, clippy_lines, env) + end + + # more text lines but clippy's complete + defp clippy_line([line | lines], [], env) do + IO.puts([env.noclippy_line, "| ", rpad_line(line, env.max_size)]) + + if lines == [] do + IO.puts(env.noclippy_line <> "\\_#{env.pad}___/") + end + + clippy_line(lines, [], env) + end + + # no more text lines but clippy's not complete + defp clippy_line([], [clippy | clippy_lines], env) do + if env.pad do + IO.puts(clippy <> "\\_#{env.pad}___/") + clippy_line([], clippy_lines, %{env | pad: nil}) + else + IO.puts(clippy) + clippy_line([], clippy_lines, env) + end + end + + defp clippy_line(_, _, _) do + end + + defp rpad_line(line, max) do + pad = max - (charlist_count_text(line) - 2) + pads = Enum.join(for(_ <- 1..pad, do: " ")) + [IO.ANSI.format(line), pads <> " |"] + end + + defp charlist_count_text(line) do + if is_list(line) do + text = Enum.join(Enum.filter(line, &is_binary/1)) + String.length(text) + else + String.length(line) + end + end +end diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 39d45f7e8..69a27e0c8 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -1,5 +1,9 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do use Ecto.Migration + alias Pleroma.Clippy + require Integer + import Ecto.Query + alias Pleroma.Repo # This migrates from int serial IDs to custom Flake: # 1- create a temporary uuid column @@ -15,6 +19,8 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + clippy = start_clippy_heartbeats() + # Lock both tables to avoid a running server to meddling with our transaction execute "LOCK TABLE activities;" execute "LOCK TABLE users;" @@ -52,5 +58,54 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;" end + stop_clippy_heartbeats(clippy) end + + defp start_clippy_heartbeats() do + count = from(a in "activities", select: count(a.id)) |> Repo.one! + + pid = if count > 5000 do + heartbeat_interval = :timer.minutes(2) + :timer.seconds(30) + all_tips = Clippy.tips() ++ [ + "The migration is still running, maybe it's time for another tea?", + "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!", + "Nothing and everything.\n\nI still work.", + "Pleroma runs on a Raspberry Pi!\n\n … but this migration will take forever if you\nactually run on a raspberry pi", + "Status? Stati? Post? Note? Toot?\nRepeat? Reboost? Boost? Retweet? Retoot??\n\nI-I'm confused.", + ] + + heartbeat = fn(heartbeat, runs, all_tips, tips) -> + tips = if Integer.is_even(runs) do + tips = if tips == [], do: all_tips, else: tips + [tip | tips] = Enum.shuffle(tips) + Clippy.puts(tip) + tips + else + IO.puts "\n -- #{DateTime.to_string(DateTime.utc_now())} Migration still running, please wait…\n" + tips + end + :timer.sleep(heartbeat_interval) + heartbeat.(heartbeat, runs + 1, all_tips, tips) + end + + Clippy.puts [ + [:red, :bright, "It looks like you are running an older instance!"], + [""], + [:bright, "This migration may take a long time", :reset, " -- so you probably should"], + ["go drink a coffee, or a tea, or a beer, a whiskey, a vodka,"], + ["while it runs to deal with your temporary fediverse pause!"] + ] + :timer.sleep(heartbeat_interval) + spawn_link(fn() -> heartbeat.(heartbeat, 1, all_tips, []) end) + end + end + + defp stop_clippy_heartbeats(pid) do + if pid do + Process.unlink(pid) + Process.exit(pid, :kill) + Clippy.puts [[:green, :bright, "Hurray!!", "", "", "Migration completed!"]] + end + end + end From e67bd93d9036770359da731b1ddad6f3da4fef66 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 23 Jan 2019 11:22:31 +0100 Subject: [PATCH 09/12] Flake: migrate pinned_activities in jsonb --- .../migrations/20181218172826_users_and_activities_flake_id.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 69a27e0c8..70ec58cd2 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -37,6 +37,8 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do execute "ALTER TABLE activities ADD PRIMARY KEY (id);" execute "ALTER TABLE users ADD PRIMARY KEY (id);" + execute "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);" + # Fkeys: # Activities - Referenced by: # TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE From 9ba6e1fabed3a583ba09c730efb144939a3a15a8 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 23 Jan 2019 12:09:21 +0100 Subject: [PATCH 10/12] FlakeId: chain alter statements --- ...18172826_users_and_activities_flake_id.exs | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 70ec58cd2..848d507e5 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -25,17 +25,21 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do execute "LOCK TABLE activities;" execute "LOCK TABLE users;" - execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;" - execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;" + execute """ + ALTER TABLE activities + DROP CONSTRAINT activities_pkey CASCADE, + ALTER COLUMN id DROP default, + ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid), + ADD PRIMARY KEY (id); + """ - execute "ALTER TABLE activities ALTER COLUMN id DROP default;" - execute "ALTER TABLE users ALTER COLUMN id DROP default;" - - execute "ALTER TABLE activities ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" - execute "ALTER TABLE users ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" - - execute "ALTER TABLE activities ADD PRIMARY KEY (id);" - execute "ALTER TABLE users ADD PRIMARY KEY (id);" + execute """ + ALTER TABLE users + DROP CONSTRAINT users_pkey CASCADE, + ALTER COLUMN id DROP default, + ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid), + ADD PRIMARY KEY (id); + """ execute "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);" @@ -52,12 +56,18 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do # TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE # TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) - execute "ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid);" - execute "ALTER TABLE notifications ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;" + execute """ + ALTER TABLE notifications + ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid), + ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE; + """ for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do - execute "ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid);" - execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;" + execute """ + ALTER TABLE #{table} + ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid), + ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + """ end stop_clippy_heartbeats(clippy) From be43aa2875e08cd48b10fc0157d71239f098d1e4 Mon Sep 17 00:00:00 2001 From: href Date: Thu, 24 Jan 2019 13:08:27 +0100 Subject: [PATCH 11/12] FlakeId: ignore null mac for workerid, use first mac --- lib/pleroma/flake_id.ex | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index 6b83ee890..26399ae05 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -161,23 +161,23 @@ defmodule Pleroma.FlakeId do 1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000) end - defp mac do + def mac do {:ok, addresses} = :inet.getifaddrs() - ifaces_with_mac = - Enum.reduce(addresses, [], fn {iface, attrs}, acc -> - if attrs[:hwaddr], do: [iface | acc], else: acc + macids = + Enum.reduce(addresses, [], fn {_iface, attrs}, acc -> + case attrs[:hwaddr] do + [0, 0, 0 | _] -> acc + mac when is_list(mac) -> [mac_to_worker_id(mac) | acc] + _ -> acc + end end) - iface = Enum.at(ifaces_with_mac, :rand.uniform(length(ifaces_with_mac)) - 1) - mac(iface) + List.first(macids) end - defp mac(name) do - {:ok, addresses} = :inet.getifaddrs() - proplist = :proplists.get_value(name, addresses) - hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6) - <> = :binary.list_to_bin(hwaddr) + def mac_to_worker_id(mac) do + <> = :binary.list_to_bin(mac) worker end end From a3ba72d97849ba12a5ea4008bd0b97e80bdd588b Mon Sep 17 00:00:00 2001 From: href Date: Thu, 24 Jan 2019 16:15:13 +0100 Subject: [PATCH 12/12] Fix clippy with one/five lines --- lib/pleroma/clippy.ex | 13 ++++++++++++- ...20181218172826_users_and_activities_flake_id.exs | 8 +++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/clippy.ex b/lib/pleroma/clippy.ex index 5e82ed8e2..4e9bdbe19 100644 --- a/lib/pleroma/clippy.ex +++ b/lib/pleroma/clippy.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Clippy do "Pleroma can federate to the Dark Web!\n - Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor) - i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation", - "Lists of Pleroma instances are available at:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma", + "Lists of Pleroma instances:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma", "Pleroma uses the LitePub protocol - https://litepub.social", "To receive more federated posts, subscribe to relays!\n - How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment @@ -90,6 +90,17 @@ defmodule Pleroma.Clippy do noclippy_line: noclippy_line } + # surrond one/five line clippy with blank lines around to not fuck up the layout + # + # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched + # features anyway? + lines = + if length(lines) == 1 or length(lines) == 5 do + [""] ++ lines ++ [""] + else + lines + end + clippy_line(lines, clippy_lines, env) rescue e -> diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 848d507e5..47d2d02da 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -70,16 +70,18 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do """ end + flush() + stop_clippy_heartbeats(clippy) end defp start_clippy_heartbeats() do count = from(a in "activities", select: count(a.id)) |> Repo.one! - pid = if count > 5000 do + if count > 5000 do heartbeat_interval = :timer.minutes(2) + :timer.seconds(30) all_tips = Clippy.tips() ++ [ - "The migration is still running, maybe it's time for another tea?", + "The migration is still running, maybe it's time for another “tea”?", "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!", "Nothing and everything.\n\nI still work.", "Pleroma runs on a Raspberry Pi!\n\n … but this migration will take forever if you\nactually run on a raspberry pi", @@ -104,7 +106,7 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do [:red, :bright, "It looks like you are running an older instance!"], [""], [:bright, "This migration may take a long time", :reset, " -- so you probably should"], - ["go drink a coffee, or a tea, or a beer, a whiskey, a vodka,"], + ["go drink a cofe, or a tea, or a beer, a whiskey, a vodka,"], ["while it runs to deal with your temporary fediverse pause!"] ] :timer.sleep(heartbeat_interval)