Merge pull request 'SQL optimisations' (#227) from i-hate-sql into develop

Reviewed-on: #227

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## Unreleased
+
+## Changes
+- Follows no longer override domain blocks, a domain block is final
+- Deletes are now the lowest priority to publish and will be handled after creates
+
 ## 2022.10
 
 ### Added

config/config.exs

@@ -569,7 +569,10 @@
     mute_expire: 5,
     search_indexing: 10
   ],
-  plugins: [Oban.Plugins.Pruner],
+  plugins: [
+    Oban.Plugins.Pruner,
+    {Oban.Plugins.Reindexer, schedule: "@weekly"}
+  ],
   crontab: [
     {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
     {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}

lib/mix/tasks/pleroma/user.ex

@@ -538,6 +538,12 @@ def run(["fix_follow_state", local_user, remote_user]) do
     end
   end
 
+  def run(["convert_id", id]) do
+    {:ok, uuid} = FlakeId.Ecto.Type.dump(id)
+    {:ok, raw_id} = Ecto.UUID.load(uuid)
+    shell_info(raw_id)
+  end
+
   defp refetch_public_keys(query) do
     query
     |> Pleroma.Repo.chunk_stream(50, :batches)

lib/pleroma/activity.ex

@@ -368,9 +368,15 @@ def following_requests_for_actor(%User{ap_id: ap_id}) do
   end
 
   def restrict_deactivated_users(query) do
-    deactivated_users_query = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id)
-
-    from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
+    query
+    |> join(
+      :inner_lateral,
+      [activity],
+      active in fragment(
+        "SELECT is_active from users WHERE ap_id = ? AND is_active = TRUE",
+        activity.actor
+      )
+    )
   end
 
   defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch

lib/pleroma/following_relationship.ex

@@ -240,30 +240,6 @@ def find(following_relationships, follower, following) do
     end)
   end
 
-  @doc """
-  For a query with joined activity,
-  keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user.
-  """
-  def keep_following_or_not_domain_blocked(query, user) do
-    where(
-      query,
-      [_, activity],
-      fragment(
-        # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)"
-        """
-        NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR
-          ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
-            ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)
-        """,
-        activity.actor,
-        ^user.domain_blocks,
-        activity.actor,
-        ^User.binary_id(user.id),
-        ^accept_state_code()
-      )
-    )
-  end
-
   defp validate_not_self_relationship(%Changeset{} = changeset) do
     changeset
     |> validate_follower_id_following_id_inequality()

lib/pleroma/notification.ex

@@ -138,7 +138,24 @@ defp exclude_blocked(query, user, opts) do
 
     query
     |> where([n, a], a.actor not in ^blocked_ap_ids)
-    |> FollowingRelationship.keep_following_or_not_domain_blocked(user)
+    |> restrict_domain_blocked(user)
+  end
+
+  defp restrict_domain_blocked(query, user) do
+    where(
+      query,
+      [_, activity],
+      fragment(
+        # "(actor's domain NOT in domain_blocks)"
+        """
+        NOT (
+          substring(? from '.*://([^/]*)') = ANY(?)
+        )
+        """,
+        activity.actor,
+        ^user.domain_blocks
+      )
+    )
   end
 
   defp exclude_blockers(query, user) do

lib/pleroma/web/activity_pub/side_effects.ex

@@ -323,8 +323,6 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
       end
 
     if result == :ok do
-      Notification.create_notifications(object)
-
       # Only remove from index when deleting actual objects, not users or anything else
       with %Pleroma.Object{} <- deleted_object do
         Pleroma.Search.remove_from_index(deleted_object)

lib/pleroma/web/federator.ex

@@ -53,12 +53,19 @@ def publish(%{data: %{"object" => object}} = activity) when is_binary(object) do
 
   @impl true
   def publish(%{data: %{"object" => object}} = activity) when is_map(object) or is_list(object) do
-    PublisherWorker.enqueue("publish", %{
+    PublisherWorker.enqueue(
+      "publish",
+      %{
         "activity_id" => activity.id,
         "object_data" => Jason.encode!(object)
-    })
+      },
+      priority: publish_priority(activity)
+    )
   end
 
+  defp publish_priority(%{type: "Delete"}), do: 3
+  defp publish_priority(_), do: 0
+
   # Job Worker Callbacks
 
   @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}

mix.exs

@@ -120,7 +120,7 @@ defp deps do
       {:phoenix_pubsub, "~> 2.1"},
       {:phoenix_ecto, "~> 4.4"},
       {:ecto_enum, "~> 1.4"},
-      {:ecto_sql, "~> 3.8.3"},
+      {:ecto_sql, "~> 3.9.0"},
       {:postgrex, ">= 0.16.3"},
       {:oban, "~> 2.12.1"},
       {:gettext,

mix.lock

@@ -26,10 +26,10 @@
   "earmark": {:hex, :earmark, "1.4.26", "f0e3c3d5c278a6d448ad8c27ab0ecdec9c57a7710553138c56af220a6330a4fd", [:mix], [{:earmark_parser, "~> 1.4.26", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "e1231882b56bece0692af33f0959f06c9cd580c2dc2ecb1dc9f16f2750fa78c5"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"},
   "eblurhash": {:hex, :eblurhash, "1.2.2", "7da4255aaea984b31bb71155f673257353b0e0554d0d30dcf859547e74602582", [:rebar3], [], "hexpm", "8c20ca00904de023a835a9dcb7b7762fed32264c85a80c3cafa85288e405044c"},
-  "ecto": {:hex, :ecto, "3.8.4", "e06b8b87e62b27fea17fd2ff6041572ddd10339fd16cdf58446e402c6c90a74b", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f9244288b8d42db40515463a008cf3f4e0e564bb9c249fe87bf28a6d79fe82d4"},
+  "ecto": {:hex, :ecto, "3.9.1", "67173b1687afeb68ce805ee7420b4261649d5e2deed8fe5550df23bab0bc4396", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c80bb3d736648df790f7f92f81b36c922d9dd3203ca65be4ff01d067f54eb304"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.4", "5d43fd088d39a158c860b17e8d210669587f63ec89ea122a4654861c8c6e2db4", [:mix], [{:ecto_sql, "~> 3.4", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.15.7", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "311db02f1b772e3d0dc7f56a05044b5e1499d78ed6abf38885e1ca70059449e5"},
-  "ecto_sql": {:hex, :ecto_sql, "3.8.3", "a7d22c624202546a39d615ed7a6b784580391e65723f2d24f65941b4dd73d471", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.8.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "348cb17fb9e6daf6f251a87049eafcb57805e2892e5e6a0f5dea0985d367329b"},
+  "ecto_sql": {:hex, :ecto_sql, "3.9.0", "2bb21210a2a13317e098a420a8c1cc58b0c3421ab8e3acfa96417dab7817918c", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a8f3f720073b8b1ac4c978be25fa7960ed7fd44997420c304a4a2e200b596453"},
   "elasticsearch": {:git, "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", "6cd946f75f6ab9042521a009d1d32d29a90113ca", [ref: "main"]},
   "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
   "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
@@ -56,7 +56,7 @@
   "httpoison": {:hex, :httpoison, "1.8.1", "df030d96de89dad2e9983f92b0c506a642d4b1f4a819c96ff77d12796189c63e", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "35156a6d678d6d516b9229e208942c405cf21232edd632327ecfaf4fd03e79e0"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"},
-  "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"},
+  "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
   "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"},
   "jose": {:hex, :jose, "1.11.2", "f4c018ccf4fdce22c71e44d471f15f723cb3efab5d909ab2ba202b5bf35557b3", [:mix, :rebar3], [], "hexpm", "98143fbc48d55f3a18daba82d34fe48959d44538e9697c08f34200fa5f0947d2"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
@@ -94,7 +94,7 @@
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
-  "postgrex": {:hex, :postgrex, "0.16.3", "fac79a81a9a234b11c44235a4494d8565303fa4b9147acf57e48978a074971db", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "aeaae1d2d1322da4e5fe90d241b0a564ce03a3add09d7270fb85362166194590"},
+  "postgrex": {:hex, :postgrex, "0.16.5", "fcc4035cc90e23933c5d69a9cd686e329469446ef7abba2cf70f08e2c4b69810", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "edead639dc6e882618c01d8fc891214c481ab9a3788dfe38dd5e37fd1d5fb2e8"},
   "pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"},
   "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},

test/pleroma/notification_test.exs

@@ -1149,18 +1149,6 @@ test "it doesn't return notifications for domain-blocked non-followed user", %{u
       assert Notification.for_user(user) == []
     end
 
-    test "it returns notifications for domain-blocked but followed user" do
-      user = insert(:user)
-      blocked = insert(:user, ap_id: "http://some-domain.com")
-
-      {:ok, user} = User.block_domain(user, "some-domain.com")
-      {:ok, _, _} = User.follow(user, blocked)
-
-      {:ok, _activity} = CommonAPI.post(blocked, %{status: "hey @#{user.nickname}"})
-
-      assert length(Notification.for_user(user)) == 1
-    end
-
     test "it doesn't return notifications for muted thread", %{user: user} do
       another_user = insert(:user)