From a85bf5929b151765c16ac6d8e9ff2bc78e664e55 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 14 Dec 2021 12:37:10 +0000 Subject: [PATCH] add hashtag indexing --- lib/mix/tasks/pleroma/search.ex | 11 ++++++++++- .../document_mappings/hashtag.ex | 10 ++++++++++ lib/pleroma/elasticsearch/store.ex | 19 +++++++++++++++++++ priv/es-mappings/hashtag.json | 4 ++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/elasticsearch/document_mappings/hashtag.ex diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex index 64396b92f..751e0ca11 100644 --- a/lib/mix/tasks/pleroma/search.ex +++ b/lib/mix/tasks/pleroma/search.ex @@ -9,6 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search do alias Pleroma.Activity alias Pleroma.Pagination alias Pleroma.User + alias Pleroma.Hashtag @shortdoc "Manages elasticsearch" @@ -29,8 +30,16 @@ defmodule Mix.Tasks.Pleroma.Search do |> get_all(:users) end + def run(["import", "hashtags" | _rest]) do + start_pleroma() + + from(h in Hashtag) + |> Pleroma.Repo.all() + |> Pleroma.Elasticsearch.bulk_post(:hashtags) + end + defp get_all(query, index, max_id \\ nil) do - params = %{limit: 2000} + params = %{limit: 1000} params = if max_id == nil do diff --git a/lib/pleroma/elasticsearch/document_mappings/hashtag.ex b/lib/pleroma/elasticsearch/document_mappings/hashtag.ex new file mode 100644 index 000000000..1c47d1451 --- /dev/null +++ b/lib/pleroma/elasticsearch/document_mappings/hashtag.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Elasticsearch.DocumentMappings.Hashtag do + def id(obj), do: obj.id + + def encode(hashtag) do + %{ + hashtag: hashtag.name, + timestamp: hashtag.inserted_at + } + end +end diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex index 776bad921..74c933038 100644 --- a/lib/pleroma/elasticsearch/store.ex +++ b/lib/pleroma/elasticsearch/store.ex @@ -85,6 +85,25 @@ defmodule Pleroma.Elasticsearch do ) end + def bulk_post(data, :hashtags) do + d = + data + |> Enum.map(fn d -> + [ + %{index: %{_id: DocumentMappings.Hashtag.id(d)}}, + DocumentMappings.Hashtag.encode(d) + ] + end) + |> List.flatten() + + Elastix.Bulk.post( + url(), + d, + index: "hashtags", + type: "hashtag" + ) + end + def search_activities(q) do Elastix.Search.search( url(), diff --git a/priv/es-mappings/hashtag.json b/priv/es-mappings/hashtag.json index 5330b8a2e..3a0ade8f8 100644 --- a/priv/es-mappings/hashtag.json +++ b/priv/es-mappings/hashtag.json @@ -1,5 +1,9 @@ { "properties": { + "timestamp": { + "type": "date", + "index": true + }, "hashtag": { "type": "text" }