Merge branch 'websearch' into 'develop'

Use `websearch_to_tsvector` by default

See merge request pleroma/pleroma!3163

.gitlab-ci.yml

@@ -57,7 +57,7 @@ unit-testing:
     policy: pull
 
   services:
-  - name: postgres:9.6
+  - name: postgres:13
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Polls now always return a `voters_count`, even if they are single-choice.
 - Admin Emails: The ap id is used as the user link in emails now.
+- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
 
 ### Added
 

lib/pleroma/activity/search.ex

@@ -19,11 +19,18 @@ def search(user, search_query, options \\ []) do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
+    search_function =
+      if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+        :websearch
+      else
+        :plain
+      end
+
     Activity
     |> Activity.with_preloaded_object()
     |> Activity.restrict_deactivated_users()
     |> restrict_public()
-    |> query_with(index_type, search_query)
+    |> query_with(index_type, search_query, search_function)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
     |> maybe_restrict_blocked(user)
@@ -53,7 +60,7 @@ defp restrict_public(q) do
     )
   end
 
-  defp query_with(q, :gin, search_query) do
+  defp query_with(q, :gin, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -64,7 +71,18 @@ defp query_with(q, :gin, search_query) do
     )
   end
 
-  defp query_with(q, :rum, search_query) do
+  defp query_with(q, :gin, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+          o.data,
+          ^search_query
+        )
+    )
+  end
+
+  defp query_with(q, :rum, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -76,6 +94,18 @@ defp query_with(q, :rum, search_query) do
     )
   end
 
+  defp query_with(q, :rum, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ websearch_to_tsquery('english', ?)",
+          o.fts_content,
+          ^search_query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   defp maybe_restrict_local(q, user) do
     limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
 

lib/pleroma/application.ex

@@ -110,7 +110,28 @@ def start(_type, _args) do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    set_postgres_server_version()
+
+    result
+  end
+
+  defp set_postgres_server_version do
+    version =
+      with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
+           {num, _} <- Float.parse(version) do
+        num
+      else
+        e ->
+          Logger.warn(
+            "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
+          )
+
+          9.6
+      end
+
+    :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
   end
 
   def load_custom_modules do

test/pleroma/activity/search_test.exs

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.SearchTest do
+  alias Pleroma.Activity.Search
+  alias Pleroma.Web.CommonAPI
+  import Pleroma.Factory
+
+  use Pleroma.DataCase
+
+  test "it finds something" do
+    user = insert(:user)
+    {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+
+    [result] = Search.search(nil, "wednesday")
+
+    assert result.id == post.id
+  end
+
+  test "using plainto_tsquery on postgres < 11" do
+    old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
+    :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
+
+    user = insert(:user)
+    {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+    {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+    # plainto doesn't understand complex queries
+    assert [result] = Search.search(nil, "wednesday -dudes")
+
+    assert result.id == post.id
+
+    :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
+  end
+
+  test "using websearch_to_tsquery" do
+    user = insert(:user)
+    {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+    {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+    assert [result] = Search.search(nil, "wednesday -dudes")
+
+    assert result.id == other_post.id
+  end
+end

test/pleroma/web/mastodon_api/controllers/search_controller_test.exs

@@ -279,6 +279,9 @@ test "search", %{conn: conn} do
     end
 
     test "search fetches remote statuses and prefers them over other results", %{conn: conn} do
+      old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
+      :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
+
       capture_log(fn ->
         {:ok, %{id: activity_id}} =
           CommonAPI.post(insert(:user), %{
@@ -295,6 +298,8 @@ test "search fetches remote statuses and prefers them over other results", %{con
                  %{"id" => ^activity_id}
                ] = results["statuses"]
       end)
+
+      :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
     end
 
     test "search doesn't show statuses that it shouldn't", %{conn: conn} do