forked from AkkomaGang/akkoma
Search through users and hashtags as well
This commit is contained in:
parent
8547cefaff
commit
c03e8d46e8
9 changed files with 280 additions and 71 deletions
|
@ -26,7 +26,7 @@ def run(["import", "activities" | _rest]) do
|
|||
def run(["import", "users" | _rest]) do
|
||||
start_pleroma()
|
||||
|
||||
from(u in User, where: not ilike(u.ap_id, "%/relay"))
|
||||
from(u in User, where: u.nickname not in ["internal.fetch", "relay"])
|
||||
|> get_all(:users)
|
||||
end
|
||||
|
||||
|
|
|
@ -6,7 +6,8 @@ def encode(%{actor_type: "Person"} = user) do
|
|||
timestamp: user.inserted_at,
|
||||
instance: URI.parse(user.ap_id).host,
|
||||
nickname: user.nickname,
|
||||
bio: user.bio
|
||||
bio: user.bio,
|
||||
display_name: user.name
|
||||
}
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,24 +1,32 @@
|
|||
defmodule Pleroma.Elasticsearch do
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Elasticsearch.DocumentMappings
|
||||
alias Pleroma.Config
|
||||
require Logger
|
||||
|
||||
defp url do
|
||||
Config.get([:elasticsearch, :url])
|
||||
end
|
||||
|
||||
def put_by_id(id) do
|
||||
defp enabled? do
|
||||
Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch
|
||||
end
|
||||
|
||||
def put_by_id(:activity, id) do
|
||||
id
|
||||
|> Activity.get_by_id_with_object()
|
||||
|> maybe_put_into_elasticsearch()
|
||||
end
|
||||
|
||||
def maybe_put_into_elasticsearch({:ok, activity}) do
|
||||
maybe_put_into_elasticsearch(activity)
|
||||
def maybe_put_into_elasticsearch({:ok, item}) do
|
||||
maybe_put_into_elasticsearch(item)
|
||||
end
|
||||
|
||||
def maybe_put_into_elasticsearch(%{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity) do
|
||||
if Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch do
|
||||
def maybe_put_into_elasticsearch(
|
||||
%{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity
|
||||
) do
|
||||
if enabled?() do
|
||||
actor = Pleroma.Activity.user_actor(activity)
|
||||
|
||||
activity
|
||||
|
@ -27,27 +35,48 @@ def maybe_put_into_elasticsearch(%{data: %{"type" => "Create"}, object: %{data:
|
|||
end
|
||||
end
|
||||
|
||||
def maybe_put_into_elasticsearch(%User{} = user) do
|
||||
if enabled?() do
|
||||
put(user)
|
||||
end
|
||||
end
|
||||
|
||||
def maybe_put_into_elasticsearch(_) do
|
||||
{:ok, :skipped}
|
||||
end
|
||||
|
||||
def put(%Activity{} = activity) do
|
||||
Elastix.Document.index(
|
||||
{:ok, _} = Elastix.Document.index(
|
||||
url(),
|
||||
"activities",
|
||||
"activity",
|
||||
DocumentMappings.Activity.id(activity),
|
||||
DocumentMappings.Activity.encode(activity)
|
||||
)
|
||||
{:ok, _} = bulk_post(
|
||||
activity.object.hashtags, :hashtags
|
||||
)
|
||||
end
|
||||
|
||||
def put(%User{} = user) do
|
||||
{:ok, _ } = Elastix.Document.index(
|
||||
url(),
|
||||
"users",
|
||||
"user",
|
||||
DocumentMappings.User.id(user),
|
||||
DocumentMappings.User.encode(user)
|
||||
)
|
||||
end
|
||||
|
||||
def bulk_post(data, :activities) do
|
||||
d =
|
||||
data
|
||||
|> Enum.filter(fn x ->
|
||||
t = x.object
|
||||
|> Map.get(:data, %{})
|
||||
|> Map.get("type", "")
|
||||
t =
|
||||
x.object
|
||||
|> Map.get(:data, %{})
|
||||
|> Map.get("type", "")
|
||||
|
||||
t == "Note"
|
||||
end)
|
||||
|> Enum.map(fn d ->
|
||||
|
@ -58,7 +87,7 @@ def bulk_post(data, :activities) do
|
|||
end)
|
||||
|> List.flatten()
|
||||
|
||||
Elastix.Bulk.post(
|
||||
{:ok, %{body: %{"errors" => false}}} = Elastix.Bulk.post(
|
||||
url(),
|
||||
d,
|
||||
index: "activities",
|
||||
|
@ -104,12 +133,54 @@ def bulk_post(data, :hashtags) do
|
|||
)
|
||||
end
|
||||
|
||||
def search_activities(q) do
|
||||
Elastix.Search.search(
|
||||
url(),
|
||||
"activities",
|
||||
["activity"],
|
||||
q
|
||||
)
|
||||
def search(:raw, index, type, q) do
|
||||
with {:ok, raw_results} <- Elastix.Search.search(url(), index, [type], q) do
|
||||
results =
|
||||
raw_results
|
||||
|> Map.get(:body, %{})
|
||||
|> Map.get("hits", %{})
|
||||
|> Map.get("hits", [])
|
||||
|
||||
{:ok, results}
|
||||
else
|
||||
{:error, e} ->
|
||||
Logger.error(e)
|
||||
{:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
def search(:activities, q) do
|
||||
with {:ok, results} <- search(:raw, "activities", "activity", q) do
|
||||
results
|
||||
|> Enum.map(fn result -> result["_id"] end)
|
||||
|> Pleroma.Activity.all_by_ids_with_object()
|
||||
else
|
||||
e ->
|
||||
Logger.error(e)
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
def search(:users, q) do
|
||||
with {:ok, results} <- search(:raw, "users", "user", q) do
|
||||
results
|
||||
|> Enum.map(fn result -> result["_id"] end)
|
||||
|> Pleroma.User.get_all_by_ids()
|
||||
else
|
||||
e ->
|
||||
Logger.error(e)
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
def search(:hashtags, q) do
|
||||
with {:ok, results} <- search(:raw, "hashtags", "hashtag", q) do
|
||||
results
|
||||
|> Enum.map(fn result -> result["_source"]["hashtag"] end)
|
||||
else
|
||||
e ->
|
||||
Logger.error(e)
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -2,79 +2,94 @@ defmodule Pleroma.Search.Elasticsearch do
|
|||
@behaviour Pleroma.Search
|
||||
|
||||
alias Pleroma.Web.MastodonAPI.StatusView
|
||||
alias Pleroma.Web.MastodonAPI.AccountView
|
||||
alias Pleroma.Web.ActivityPub.Visibility
|
||||
alias Pleroma.Search.Elasticsearch.Parsers
|
||||
alias Pleroma.Web.Endpoint
|
||||
|
||||
defp to_es(term) when is_binary(term) do
|
||||
defp es_query(:activity, query) do
|
||||
%{
|
||||
match: %{
|
||||
content: %{
|
||||
query: term,
|
||||
operator: "AND"
|
||||
query: %{
|
||||
bool: %{
|
||||
must: Parsers.Activity.parse(query)
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:quoted, term}), do: to_es(term)
|
||||
|
||||
defp to_es({:filter, ["hashtag", query]}) do
|
||||
defp es_query(:user, query) do
|
||||
%{
|
||||
term: %{
|
||||
hashtags: %{
|
||||
value: query
|
||||
query: %{
|
||||
bool: %{
|
||||
must: Parsers.User.parse(query)
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:filter, [field, query]}) do
|
||||
defp es_query(:hashtag, query) do
|
||||
%{
|
||||
term: %{
|
||||
field => %{
|
||||
value: query
|
||||
query: %{
|
||||
bool: %{
|
||||
must: Parsers.Hashtag.parse(query)
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp parse(query) do
|
||||
query
|
||||
|> SearchParser.parse!()
|
||||
|> Enum.map(&to_es/1)
|
||||
end
|
||||
|
||||
@impl Pleroma.Search
|
||||
def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) do
|
||||
q = %{
|
||||
query: %{
|
||||
bool: %{
|
||||
must: parse(String.trim(query))
|
||||
}
|
||||
}
|
||||
parsed_query =
|
||||
query
|
||||
|> String.trim()
|
||||
|> SearchParser.parse!()
|
||||
|
||||
activity_task =
|
||||
Task.async(fn ->
|
||||
q = es_query(:activity, parsed_query)
|
||||
|
||||
Pleroma.Elasticsearch.search(:activities, q)
|
||||
|> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
|
||||
end)
|
||||
|
||||
user_task =
|
||||
Task.async(fn ->
|
||||
q = es_query(:user, parsed_query)
|
||||
|
||||
Pleroma.Elasticsearch.search(:users, q)
|
||||
|> Enum.filter(fn x -> Pleroma.User.visible_for(x, user) == :visible end)
|
||||
end)
|
||||
|
||||
hashtag_task =
|
||||
Task.async(fn ->
|
||||
q = es_query(:hashtag, parsed_query)
|
||||
|
||||
Pleroma.Elasticsearch.search(:hashtags, q)
|
||||
end)
|
||||
|
||||
activity_results = Task.await(activity_task)
|
||||
user_results = Task.await(user_task)
|
||||
hashtag_results = Task.await(hashtag_task)
|
||||
|
||||
%{
|
||||
"accounts" =>
|
||||
AccountView.render("index.json",
|
||||
users: user_results,
|
||||
for: user
|
||||
),
|
||||
"hashtags" =>
|
||||
Enum.map(hashtag_results, fn x ->
|
||||
%{
|
||||
url: Endpoint.url() <> "/tag/" <> x,
|
||||
name: x
|
||||
}
|
||||
end),
|
||||
"statuses" =>
|
||||
StatusView.render("index.json",
|
||||
activities: activity_results,
|
||||
for: user,
|
||||
as: :activity
|
||||
)
|
||||
}
|
||||
|
||||
out = Pleroma.Elasticsearch.search_activities(q)
|
||||
|
||||
with {:ok, raw_results} <- out do
|
||||
results =
|
||||
raw_results
|
||||
|> Map.get(:body, %{})
|
||||
|> Map.get("hits", %{})
|
||||
|> Map.get("hits", [])
|
||||
|> Enum.map(fn result -> result["_id"] end)
|
||||
|> Pleroma.Activity.all_by_ids_with_object()
|
||||
|> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
|
||||
|
||||
%{
|
||||
"accounts" => [],
|
||||
"hashtags" => [],
|
||||
"statuses" =>
|
||||
StatusView.render("index.json",
|
||||
activities: results,
|
||||
for: user,
|
||||
as: :activity
|
||||
)
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
38
lib/pleroma/search/elasticsearch/activity_parser.ex
Normal file
38
lib/pleroma/search/elasticsearch/activity_parser.ex
Normal file
|
@ -0,0 +1,38 @@
|
|||
defmodule Pleroma.Search.Elasticsearch.Parsers.Activity do
|
||||
defp to_es(term) when is_binary(term) do
|
||||
%{
|
||||
match: %{
|
||||
content: %{
|
||||
query: term,
|
||||
operator: "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:quoted, term}), do: to_es(term)
|
||||
|
||||
defp to_es({:filter, ["hashtag", query]}) do
|
||||
%{
|
||||
term: %{
|
||||
hashtags: %{
|
||||
value: query
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:filter, [field, query]}) do
|
||||
%{
|
||||
term: %{
|
||||
field => %{
|
||||
value: query
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def parse(q) do
|
||||
Enum.map(q, &to_es/1)
|
||||
end
|
||||
end
|
30
lib/pleroma/search/elasticsearch/hashtag_parser.ex
Normal file
30
lib/pleroma/search/elasticsearch/hashtag_parser.ex
Normal file
|
@ -0,0 +1,30 @@
|
|||
defmodule Pleroma.Search.Elasticsearch.Parsers.Hashtag do
|
||||
defp to_es(term) when is_binary(term) do
|
||||
%{
|
||||
term: %{
|
||||
hashtag: %{
|
||||
value: String.downcase(term),
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:quoted, term}), do: to_es(term)
|
||||
|
||||
defp to_es({:filter, ["hashtag", query]}) do
|
||||
%{
|
||||
term: %{
|
||||
hashtag: %{
|
||||
value: String.downcase(query)
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:filter, _}), do: nil
|
||||
|
||||
def parse(q) do
|
||||
Enum.map(q, &to_es/1)
|
||||
|> Enum.filter(fn x -> x != nil end)
|
||||
end
|
||||
end
|
53
lib/pleroma/search/elasticsearch/user_paser.ex
Normal file
53
lib/pleroma/search/elasticsearch/user_paser.ex
Normal file
|
@ -0,0 +1,53 @@
|
|||
defmodule Pleroma.Search.Elasticsearch.Parsers.User do
|
||||
defp to_es(term) when is_binary(term) do
|
||||
%{
|
||||
bool: %{
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
%{
|
||||
match: %{
|
||||
bio: %{
|
||||
query: term,
|
||||
operator: "AND"
|
||||
}
|
||||
}
|
||||
},
|
||||
%{
|
||||
term: %{
|
||||
nickname: %{
|
||||
value: term
|
||||
}
|
||||
}
|
||||
},
|
||||
%{
|
||||
match: %{
|
||||
display_name: %{
|
||||
query: term,
|
||||
operator: "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:quoted, term}), do: to_es(term)
|
||||
|
||||
defp to_es({:filter, ["user", query]}) do
|
||||
%{
|
||||
term: %{
|
||||
nickname: %{
|
||||
value: query
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
defp to_es({:filter, _}), do: nil
|
||||
|
||||
def parse(q) do
|
||||
Enum.map(q, &to_es/1)
|
||||
|> Enum.filter(fn x -> x != nil end)
|
||||
end
|
||||
end
|
|
@ -1088,6 +1088,7 @@ def update_and_set_cache(struct, params) do
|
|||
|
||||
def update_and_set_cache(changeset) do
|
||||
with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
|
||||
Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user)
|
||||
set_cache(user)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -538,7 +538,7 @@ defp add_notifications(meta, notifications) do
|
|||
|
||||
@impl true
|
||||
def handle_after_transaction(%Pleroma.Activity{data: %{"type" => "Create"}} = activity) do
|
||||
Pleroma.Elasticsearch.put_by_id(activity.id)
|
||||
Pleroma.Elasticsearch.put_by_id(:activity, activity.id)
|
||||
end
|
||||
|
||||
def handle_after_transaction(%Pleroma.Activity{}) do
|
||||
|
|
Loading…
Reference in a new issue