Preserve Meilisearch’s result ranking #772
6 changed files with 85 additions and 22 deletions
|
@ -14,6 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|||
## Added
|
||||
- Implement [FEP-67ff](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md) (federation documentation)
|
||||
|
||||
## Added
|
||||
- Meilisearch: it is now possible to use separate keys for search and admin actions
|
||||
|
||||
## Fixed
|
||||
- Meilisearch: order of results returned from our REST API now actually matches how Meilisearch ranks results
|
||||
|
||||
## 2024.04
|
||||
|
||||
## Added
|
||||
|
|
|
@ -33,6 +33,7 @@ indexes faster when it can process many posts in a single batch.
|
|||
> config :pleroma, Pleroma.Search.Meilisearch,
|
||||
> url: "http://127.0.0.1:7700/",
|
||||
> private_key: "private key",
|
||||
> search_key: "search key",
|
||||
> initial_indexing_chunk_size: 100_000
|
||||
|
||||
Information about setting up meilisearch can be found in the
|
||||
|
@ -45,7 +46,7 @@ is hardly usable on a somewhat big instance.
|
|||
### Private key authentication (optional)
|
||||
|
||||
To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
|
||||
you have to get the _private key_, which is actually used for authentication.
|
||||
you have to get the _private key_ and possibly _search key_, which are actually used for authentication.
|
||||
|
||||
=== "OTP"
|
||||
```sh
|
||||
|
@ -57,7 +58,11 @@ you have to get the _private key_, which is actually used for authentication.
|
|||
mix pleroma.search.meilisearch show-keys <your master key here>
|
||||
```
|
||||
|
||||
You will see a "Default Admin API Key", this is the key you actually put into your configuration file.
|
||||
You will see a "Default Admin API Key", this is the key you actually put into
|
||||
Oneric marked this conversation as resolved
Outdated
|
||||
your configuration file as `private_key`. You should also see a
|
||||
"Default Search API key", put this into your config as `search_key`.
|
||||
If your version of Meilisearch only showed the former,
|
||||
just leave `search_key` completely unset in Akkoma's config.
|
||||
|
||||
### Initial indexing
|
||||
|
||||
|
|
|
@ -126,7 +126,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
|
|||
decoded = Jason.decode!(result.body)
|
||||
|
||||
if decoded["results"] do
|
||||
Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
|
||||
Enum.each(decoded["results"], fn
|
||||
%{"name" => name, "key" => key} ->
|
||||
IO.puts("#{name}: #{key}")
|
||||
|
||||
%{"description" => desc, "key" => key} ->
|
||||
IO.puts("#{desc}: #{key}")
|
||||
end)
|
||||
else
|
||||
|
|
|
@ -258,6 +258,27 @@ defmodule Pleroma.Activity do
|
|||
|
||||
def get_create_by_object_ap_id(_), do: nil
|
||||
|
||||
@doc """
|
||||
Accepts a list of `ap__id`.
|
||||
Returns a query yielding Create activities for the given objects,
|
||||
in the same order as they were specified in the input list.
|
||||
"""
|
||||
@spec get_presorted_create_by_object_ap_id([String.t()]) :: Ecto.Queryable.t()
|
||||
def get_presorted_create_by_object_ap_id(ap_ids) do
|
||||
from(
|
||||
a in Activity,
|
||||
join:
|
||||
ids in fragment(
|
||||
"SELECT * FROM UNNEST(?::text[]) WITH ORDINALITY AS ids(ap_id, ord)",
|
||||
^ap_ids
|
||||
),
|
||||
on:
|
||||
ids.ap_id == fragment("?->>'object'", a.data) and
|
||||
fragment("?->>'type'", a.data) == "Create",
|
||||
order_by: [asc: ids.ord]
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Accepts `ap_id` or list of `ap_id`.
|
||||
Returns a query.
|
||||
|
|
|
@ -5,15 +5,27 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
alias Pleroma.Activity
|
||||
|
||||
import Pleroma.Search.DatabaseSearch
|
||||
import Ecto.Query
|
||||
|
||||
@behaviour Pleroma.Search.SearchBackend
|
||||
|
||||
defp meili_headers do
|
||||
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
|
||||
defp meili_headers(key) do
|
||||
key_header =
|
||||
if is_nil(key), do: [], else: [{"Authorization", "Bearer #{key}"}]
|
||||
|
||||
[{"Content-Type", "application/json"}] ++
|
||||
if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
|
||||
[{"Content-Type", "application/json"} | key_header]
|
||||
end
|
||||
|
||||
defp meili_headers_admin do
|
||||
private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
|
||||
meili_headers(private_key)
|
||||
end
|
||||
|
||||
defp meili_headers_search do
|
||||
search_key =
|
||||
Pleroma.Config.get([Pleroma.Search.Meilisearch, :search_key]) ||
|
||||
Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
|
||||
|
||||
meili_headers(search_key)
|
||||
end
|
||||
|
||||
def meili_get(path) do
|
||||
|
@ -22,7 +34,7 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
result =
|
||||
Pleroma.HTTP.get(
|
||||
Path.join(endpoint, path),
|
||||
meili_headers()
|
||||
meili_headers_admin()
|
||||
)
|
||||
|
||||
with {:ok, res} <- result do
|
||||
|
@ -30,14 +42,14 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
end
|
||||
end
|
||||
|
||||
def meili_post(path, params) do
|
||||
defp meili_search(params) do
|
||||
endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
|
||||
|
||||
result =
|
||||
Pleroma.HTTP.post(
|
||||
Path.join(endpoint, path),
|
||||
Path.join(endpoint, "/indexes/objects/search"),
|
||||
Jason.encode!(params),
|
||||
meili_headers()
|
||||
meili_headers_search()
|
||||
)
|
||||
|
||||
with {:ok, res} <- result do
|
||||
|
@ -53,7 +65,7 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
:put,
|
||||
Path.join(endpoint, path),
|
||||
Jason.encode!(params),
|
||||
meili_headers(),
|
||||
meili_headers_admin(),
|
||||
[]
|
||||
)
|
||||
|
||||
|
@ -70,7 +82,7 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
:delete,
|
||||
Path.join(endpoint, path),
|
||||
"",
|
||||
meili_headers(),
|
||||
meili_headers_admin(),
|
||||
[]
|
||||
)
|
||||
end
|
||||
|
@ -81,25 +93,20 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
author = Keyword.get(options, :author)
|
||||
|
||||
res =
|
||||
meili_post(
|
||||
"/indexes/objects/search",
|
||||
%{q: query, offset: offset, limit: limit}
|
||||
)
|
||||
meili_search(%{q: query, offset: offset, limit: limit})
|
||||
|
||||
with {:ok, result} <- res do
|
||||
hits = result["hits"] |> Enum.map(& &1["ap"])
|
||||
|
||||
try do
|
||||
hits
|
||||
|> Activity.create_by_object_ap_id()
|
||||
|> Activity.with_preloaded_object()
|
||||
|> Activity.get_presorted_create_by_object_ap_id()
|
||||
|> Activity.with_preloaded_object()
|
||||
|> Activity.restrict_deactivated_users()
|
||||
|> maybe_restrict_local(user)
|
||||
|> maybe_restrict_author(author)
|
||||
|> maybe_restrict_blocked(user)
|
||||
|> maybe_fetch(user, query)
|
||||
|> order_by([object: obj], desc: obj.data["published"])
|
||||
|> Pleroma.Repo.all()
|
||||
rescue
|
||||
_ -> maybe_fetch([], user, query)
|
||||
|
|
|
@ -41,6 +41,26 @@ defmodule Pleroma.ActivityTest do
|
|||
assert activity == found_activity
|
||||
end
|
||||
|
||||
test "returns activities by object's AP id in requested presorted order" do
|
||||
a1 = insert(:note_activity)
|
||||
o1 = Object.normalize(a1, fetch: false).data["id"]
|
||||
|
||||
a2 = insert(:note_activity)
|
||||
o2 = Object.normalize(a2, fetch: false).data["id"]
|
||||
|
||||
a3 = insert(:note_activity)
|
||||
o3 = Object.normalize(a3, fetch: false).data["id"]
|
||||
|
||||
a4 = insert(:note_activity)
|
||||
o4 = Object.normalize(a4, fetch: false).data["id"]
|
||||
|
||||
found_activities =
|
||||
Activity.get_presorted_create_by_object_ap_id([o3, o2, o4, o1])
|
||||
|> Repo.all()
|
||||
|
||||
assert found_activities == [a3, a2, a4, a1]
|
||||
end
|
||||
|
||||
test "preloading a bookmark" do
|
||||
user = insert(:user)
|
||||
user2 = insert(:user)
|
||||
|
|
Loading…
Reference in a new issue
Wondering if it would make more sense to limit the admin API key to just admin tasks and use a separate key for just searches done via our APIs..
repusehd, using separate search and admin keys is now possible and instead of changing documentation to match the output of our
show-keys
task,show-keys
is changed to display Meilisearch'sname
parameter if present instead ofdescription
.