feature/elasticsearch #1
5 changed files with 54 additions and 12 deletions
|
@ -8,21 +8,28 @@ defmodule Mix.Tasks.Pleroma.Search do
|
||||||
import Ecto.Query
|
import Ecto.Query
|
||||||
alias Pleroma.Activity
|
alias Pleroma.Activity
|
||||||
alias Pleroma.Pagination
|
alias Pleroma.Pagination
|
||||||
|
alias Pleroma.User
|
||||||
|
|
||||||
@shortdoc "Manages elasticsearch"
|
@shortdoc "Manages elasticsearch"
|
||||||
|
|
||||||
def run(["import" | _rest]) do
|
def run(["import", "activities" | _rest]) do
|
||||||
start_pleroma()
|
start_pleroma()
|
||||||
|
|
||||||
from(a in Activity, where: not ilike(a.actor, "%/relay"))
|
from(a in Activity, where: not ilike(a.actor, "%/relay"))
|
||||||
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|
||||||
|> Activity.with_preloaded_object()
|
|> Activity.with_preloaded_object()
|
||||||
|> Activity.with_preloaded_user_actor()
|
|> Activity.with_preloaded_user_actor()
|
||||||
|> get_all
|
|> get_all(:activities)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp get_all(query, max_id \\ nil) do
|
def run(["import", "users" | _rest]) do
|
||||||
IO.puts(max_id)
|
start_pleroma()
|
||||||
|
|
||||||
|
from(u in User, where: not ilike(u.ap_id, "%/relay"))
|
||||||
|
|> get_all(:users)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp get_all(query, index, max_id \\ nil) do
|
||||||
params = %{limit: 2000}
|
params = %{limit: 2000}
|
||||||
|
|
||||||
params =
|
params =
|
||||||
|
@ -40,15 +47,9 @@ defp get_all(query, max_id \\ nil) do
|
||||||
:ok
|
:ok
|
||||||
else
|
else
|
||||||
res
|
res
|
||||||
|> Enum.filter(fn x ->
|
|> Pleroma.Elasticsearch.bulk_post(index)
|
||||||
t = x.object
|
|
||||||
|> Map.get(:data, %{})
|
|
||||||
|> Map.get("type", "")
|
|
||||||
t == "Note"
|
|
||||||
end)
|
|
||||||
|> Pleroma.Elasticsearch.bulk_post(:activities)
|
|
||||||
|
|
||||||
get_all(query, List.last(res).id)
|
get_all(query, index, List.last(res).id)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
12
lib/pleroma/elasticsearch/document_mappings/user.ex
Normal file
12
lib/pleroma/elasticsearch/document_mappings/user.ex
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
defmodule Pleroma.Elasticsearch.DocumentMappings.User do
|
||||||
|
def id(obj), do: obj.id
|
||||||
|
|
||||||
|
def encode(%{actor_type: "Person"} = user) do
|
||||||
|
%{
|
||||||
|
timestamp: user.inserted_at,
|
||||||
|
instance: URI.parse(user.ap_id).host,
|
||||||
|
nickname: user.nickname,
|
||||||
|
bio: user.bio
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
|
@ -44,6 +44,12 @@ def put(%Activity{} = activity) do
|
||||||
def bulk_post(data, :activities) do
|
def bulk_post(data, :activities) do
|
||||||
d =
|
d =
|
||||||
data
|
data
|
||||||
|
|> Enum.filter(fn x ->
|
||||||
|
t = x.object
|
||||||
|
|> Map.get(:data, %{})
|
||||||
|
|> Map.get("type", "")
|
||||||
|
t == "Note"
|
||||||
|
end)
|
||||||
|> Enum.map(fn d ->
|
|> Enum.map(fn d ->
|
||||||
[
|
[
|
||||||
%{index: %{_id: DocumentMappings.Activity.id(d)}},
|
%{index: %{_id: DocumentMappings.Activity.id(d)}},
|
||||||
|
@ -60,6 +66,25 @@ def bulk_post(data, :activities) do
|
||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def bulk_post(data, :users) do
|
||||||
|
d =
|
||||||
|
data
|
||||||
|
|> Enum.map(fn d ->
|
||||||
|
[
|
||||||
|
%{index: %{_id: DocumentMappings.User.id(d)}},
|
||||||
|
DocumentMappings.User.encode(d)
|
||||||
|
]
|
||||||
|
end)
|
||||||
|
|> List.flatten()
|
||||||
|
|
||||||
|
Elastix.Bulk.post(
|
||||||
|
url(),
|
||||||
|
d,
|
||||||
|
index: "users",
|
||||||
|
type: "user"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
def search_activities(q) do
|
def search_activities(q) do
|
||||||
Elastix.Search.search(
|
Elastix.Search.search(
|
||||||
url(),
|
url(),
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
{
|
{
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"timestamp": {
|
||||||
|
"type": "date",
|
||||||
|
"index": true
|
||||||
|
},
|
||||||
"instance": {
|
"instance": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
},
|
},
|
||||||
|
|
Loading…
Reference in a new issue