Merge branch 'websearch' into 'develop'

Use `websearch_to_tsvector` by default

See merge request 
This commit is contained in:
rinpatch 2020-11-23 14:40:33 +00:00
commit 12bdced845
6 changed files with 108 additions and 5 deletions
.gitlab-ci.ymlCHANGELOG.md
lib/pleroma
test/pleroma
activity
web/mastodon_api/controllers

View file

@ -57,7 +57,7 @@ unit-testing:
policy: pull
services:
- name: postgres:9.6
- name: postgres:13
alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script:

View file

@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Polls now always return a `voters_count`, even if they are single-choice.
- Admin Emails: The ap id is used as the user link in emails now.
- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
### Added

View file

@ -19,11 +19,18 @@ def search(user, search_query, options \\ []) do
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
search_function =
if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
:websearch
else
:plain
end
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public()
|> query_with(index_type, search_query)
|> query_with(index_type, search_query, search_function)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
@ -53,7 +60,7 @@ defp restrict_public(q) do
)
end
defp query_with(q, :gin, search_query) do
defp query_with(q, :gin, search_query, :plain) do
from([a, o] in q,
where:
fragment(
@ -64,7 +71,18 @@ defp query_with(q, :gin, search_query) do
)
end
defp query_with(q, :rum, search_query) do
defp query_with(q, :gin, search_query, :websearch) do
from([a, o] in q,
where:
fragment(
"to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
o.data,
^search_query
)
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
@ -76,6 +94,18 @@ defp query_with(q, :rum, search_query) do
)
end
defp query_with(q, :rum, search_query, :websearch) do
from([a, o] in q,
where:
fragment(
"? @@ websearch_to_tsquery('english', ?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)

View file

@ -110,7 +110,28 @@ def start(_type, _args) do
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
Supervisor.start_link(children, opts)
result = Supervisor.start_link(children, opts)
set_postgres_server_version()
result
end
defp set_postgres_server_version do
version =
with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
{num, _} <- Float.parse(version) do
num
else
e ->
Logger.warn(
"Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
)
9.6
end
:persistent_term.put({Pleroma.Repo, :postgres_version}, version)
end
def load_custom_modules do

View file

@ -0,0 +1,46 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Activity.SearchTest do
alias Pleroma.Activity.Search
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
use Pleroma.DataCase
test "it finds something" do
user = insert(:user)
{:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
[result] = Search.search(nil, "wednesday")
assert result.id == post.id
end
test "using plainto_tsquery on postgres < 11" do
old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
:persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
user = insert(:user)
{:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
{:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
# plainto doesn't understand complex queries
assert [result] = Search.search(nil, "wednesday -dudes")
assert result.id == post.id
:persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
end
test "using websearch_to_tsquery" do
user = insert(:user)
{:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
{:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
assert [result] = Search.search(nil, "wednesday -dudes")
assert result.id == other_post.id
end
end

View file

@ -279,6 +279,9 @@ test "search", %{conn: conn} do
end
test "search fetches remote statuses and prefers them over other results", %{conn: conn} do
old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
:persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
capture_log(fn ->
{:ok, %{id: activity_id}} =
CommonAPI.post(insert(:user), %{
@ -295,6 +298,8 @@ test "search fetches remote statuses and prefers them over other results", %{con
%{"id" => ^activity_id}
] = results["statuses"]
end)
:persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
end
test "search doesn't show statuses that it shouldn't", %{conn: conn} do