feature/elasticsearch #1

Merged
floatingghost merged 41 commits from feature/elasticsearch into develop 2021-12-12 17:37:28 +00:00
5 changed files with 54 additions and 12 deletions
Showing only changes of commit 187c5417e6 - Show all commits

View file

@ -8,21 +8,28 @@ defmodule Mix.Tasks.Pleroma.Search do
import Ecto.Query import Ecto.Query
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Pagination alias Pleroma.Pagination
alias Pleroma.User
@shortdoc "Manages elasticsearch" @shortdoc "Manages elasticsearch"
def run(["import" | _rest]) do def run(["import", "activities" | _rest]) do
start_pleroma() start_pleroma()
from(a in Activity, where: not ilike(a.actor, "%/relay")) from(a in Activity, where: not ilike(a.actor, "%/relay"))
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> Activity.with_preloaded_object() |> Activity.with_preloaded_object()
|> Activity.with_preloaded_user_actor() |> Activity.with_preloaded_user_actor()
|> get_all |> get_all(:activities)
end end
defp get_all(query, max_id \\ nil) do def run(["import", "users" | _rest]) do
IO.puts(max_id) start_pleroma()
from(u in User, where: not ilike(u.ap_id, "%/relay"))
|> get_all(:users)
end
defp get_all(query, index, max_id \\ nil) do
params = %{limit: 2000} params = %{limit: 2000}
params = params =
@ -40,15 +47,9 @@ defp get_all(query, max_id \\ nil) do
:ok :ok
else else
res res
|> Enum.filter(fn x -> |> Pleroma.Elasticsearch.bulk_post(index)
t = x.object
|> Map.get(:data, %{})
|> Map.get("type", "")
t == "Note"
end)
|> Pleroma.Elasticsearch.bulk_post(:activities)
get_all(query, List.last(res).id) get_all(query, index, List.last(res).id)
end end
end end
end end

View file

@ -0,0 +1,12 @@
defmodule Pleroma.Elasticsearch.DocumentMappings.User do
def id(obj), do: obj.id
def encode(%{actor_type: "Person"} = user) do
%{
timestamp: user.inserted_at,
instance: URI.parse(user.ap_id).host,
nickname: user.nickname,
bio: user.bio
}
end
end

View file

@ -44,6 +44,12 @@ def put(%Activity{} = activity) do
def bulk_post(data, :activities) do def bulk_post(data, :activities) do
d = d =
data data
|> Enum.filter(fn x ->
t = x.object
|> Map.get(:data, %{})
|> Map.get("type", "")
t == "Note"
end)
|> Enum.map(fn d -> |> Enum.map(fn d ->
[ [
%{index: %{_id: DocumentMappings.Activity.id(d)}}, %{index: %{_id: DocumentMappings.Activity.id(d)}},
@ -60,6 +66,25 @@ def bulk_post(data, :activities) do
) )
end end
def bulk_post(data, :users) do
d =
data
|> Enum.map(fn d ->
[
%{index: %{_id: DocumentMappings.User.id(d)}},
DocumentMappings.User.encode(d)
]
end)
|> List.flatten()
Elastix.Bulk.post(
url(),
d,
index: "users",
type: "user"
)
end
def search_activities(q) do def search_activities(q) do
Elastix.Search.search( Elastix.Search.search(
url(), url(),

View file

@ -1,5 +1,9 @@
{ {
"properties": { "properties": {
"timestamp": {
"type": "date",
"index": true
},
"instance": { "instance": {
"type": "keyword" "type": "keyword"
}, },