feature/elasticsearch #1
5 changed files with 54 additions and 12 deletions
|
@ -8,21 +8,28 @@ defmodule Mix.Tasks.Pleroma.Search do
|
|||
import Ecto.Query
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Pagination
|
||||
alias Pleroma.User
|
||||
|
||||
@shortdoc "Manages elasticsearch"
|
||||
|
||||
def run(["import" | _rest]) do
|
||||
def run(["import", "activities" | _rest]) do
|
||||
start_pleroma()
|
||||
|
||||
from(a in Activity, where: not ilike(a.actor, "%/relay"))
|
||||
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|
||||
|> Activity.with_preloaded_object()
|
||||
|> Activity.with_preloaded_user_actor()
|
||||
|> get_all
|
||||
|> get_all(:activities)
|
||||
end
|
||||
|
||||
defp get_all(query, max_id \\ nil) do
|
||||
IO.puts(max_id)
|
||||
def run(["import", "users" | _rest]) do
|
||||
start_pleroma()
|
||||
|
||||
from(u in User, where: not ilike(u.ap_id, "%/relay"))
|
||||
|> get_all(:users)
|
||||
end
|
||||
|
||||
defp get_all(query, index, max_id \\ nil) do
|
||||
params = %{limit: 2000}
|
||||
|
||||
params =
|
||||
|
@ -40,15 +47,9 @@ defp get_all(query, max_id \\ nil) do
|
|||
:ok
|
||||
else
|
||||
res
|
||||
|> Enum.filter(fn x ->
|
||||
t = x.object
|
||||
|> Map.get(:data, %{})
|
||||
|> Map.get("type", "")
|
||||
t == "Note"
|
||||
end)
|
||||
|> Pleroma.Elasticsearch.bulk_post(:activities)
|
||||
|> Pleroma.Elasticsearch.bulk_post(index)
|
||||
|
||||
get_all(query, List.last(res).id)
|
||||
get_all(query, index, List.last(res).id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
12
lib/pleroma/elasticsearch/document_mappings/user.ex
Normal file
12
lib/pleroma/elasticsearch/document_mappings/user.ex
Normal file
|
@ -0,0 +1,12 @@
|
|||
defmodule Pleroma.Elasticsearch.DocumentMappings.User do
|
||||
def id(obj), do: obj.id
|
||||
|
||||
def encode(%{actor_type: "Person"} = user) do
|
||||
%{
|
||||
timestamp: user.inserted_at,
|
||||
instance: URI.parse(user.ap_id).host,
|
||||
nickname: user.nickname,
|
||||
bio: user.bio
|
||||
}
|
||||
end
|
||||
end
|
|
@ -44,6 +44,12 @@ def put(%Activity{} = activity) do
|
|||
def bulk_post(data, :activities) do
|
||||
d =
|
||||
data
|
||||
|> Enum.filter(fn x ->
|
||||
t = x.object
|
||||
|> Map.get(:data, %{})
|
||||
|> Map.get("type", "")
|
||||
t == "Note"
|
||||
end)
|
||||
|> Enum.map(fn d ->
|
||||
[
|
||||
%{index: %{_id: DocumentMappings.Activity.id(d)}},
|
||||
|
@ -60,6 +66,25 @@ def bulk_post(data, :activities) do
|
|||
)
|
||||
end
|
||||
|
||||
def bulk_post(data, :users) do
|
||||
d =
|
||||
data
|
||||
|> Enum.map(fn d ->
|
||||
[
|
||||
%{index: %{_id: DocumentMappings.User.id(d)}},
|
||||
DocumentMappings.User.encode(d)
|
||||
]
|
||||
end)
|
||||
|> List.flatten()
|
||||
|
||||
Elastix.Bulk.post(
|
||||
url(),
|
||||
d,
|
||||
index: "users",
|
||||
type: "user"
|
||||
)
|
||||
end
|
||||
|
||||
def search_activities(q) do
|
||||
Elastix.Search.search(
|
||||
url(),
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
{
|
||||
"properties": {
|
||||
"timestamp": {
|
||||
"type": "date",
|
||||
"index": true
|
||||
},
|
||||
"instance": {
|
||||
"type": "keyword"
|
||||
},
|
||||
|
|
Loading…
Reference in a new issue