From 0122512c2aa07a9d1dcb58e67a09513f3d687790 Mon Sep 17 00:00:00 2001 From: sadposter Date: Thu, 30 Jun 2022 19:41:28 +0100 Subject: [PATCH 1/2] fix import from live --- config/config.exs | 2 +- docs/configuration/search.md | 6 ++-- lib/mix/pleroma.ex | 13 ++++++++- lib/mix/tasks/pleroma/search.ex | 52 +++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search.ex diff --git a/config/config.exs b/config/config.exs index 727a2b0cb..b00fd18e3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -874,7 +874,7 @@ config :pleroma, Pleroma.Search.Elasticsearch.Cluster, settings: "priv/es-mappings/activity.json", store: Pleroma.Search.Elasticsearch.Store, sources: [Pleroma.Activity], - bulk_page_size: 5000, + bulk_page_size: 1000, bulk_wait_interval: 15_000 } } diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 7c1093ab9..e1f23b505 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -154,10 +154,10 @@ To start the initial indexing, run the `build` command: === "OTP" ```sh -./bin/pleroma_ctl search.elasticsearch index activities --cluster Pleroma.Search.Elasticsearch.Cluster +./bin/pleroma_ctl search import activities ``` === "From Source" ```sh -mix elasticsearch.build activities --cluster Pleroma.Search.Elasticsearch.Cluster -``` \ No newline at end of file +mix pleroma.search import activities +``` diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 2b6c7d6bb..36d3b4c6d 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -57,7 +57,8 @@ defmodule Mix.Pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + elasticsearch_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -136,4 +137,14 @@ defmodule Mix.Pleroma do end defp http_children(_), do: [] + + def elasticsearch_children do + config = Pleroma.Config.get([Pleroma.Search, :module]) + + if config == Pleroma.Search.Elasticsearch do + [Pleroma.Search.Elasticsearch.Cluster] + else + [] + end + end end diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex new file mode 100644 index 000000000..e28810f21 --- /dev/null +++ b/lib/mix/tasks/pleroma/search.ex @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search do + use Mix.Task + import Mix.Pleroma + import Ecto.Query + alias Pleroma.Activity + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Hashtag + + @shortdoc "Manages elasticsearch" + + def run(["import", "activities" | _rest]) do + start_pleroma() + + Elasticsearch.Index.Bulk.upload(Pleroma.Search.Elasticsearch.Cluster, + "activities", + Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities])) + #from(a in Activity, where: not ilike(a.actor, "%/relay")) + #|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + #|> Activity.with_preloaded_object() + #|> Activity.with_preloaded_user_actor() + #|> get_all(:activities) + end + + defp get_all(query, index, max_id \\ nil) do + params = %{limit: 1000} + + params = + if max_id == nil do + params + else + Map.put(params, :max_id, max_id) + end + + res = + query + |> Pagination.fetch_paginated(params) + + if res == [] do + :ok + else + res + |> Enum.map(fn x -> Pleroma.Search.Elasticsearch.add_to_index(x) end) + + get_all(query, index, List.last(res).id) + end + end +end -- 2.34.1 From 6c01646ebf5683c5c5aa19ab85c443f9863d341b Mon Sep 17 00:00:00 2001 From: sadposter Date: Thu, 30 Jun 2022 19:43:30 +0100 Subject: [PATCH 2/2] remove broken task --- lib/mix/pleroma.ex | 2 +- lib/mix/tasks/pleroma/search.ex | 42 +++---------------- lib/mix/tasks/pleroma/search/elasticsearch.ex | 9 ---- 3 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/search/elasticsearch.ex diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 36d3b4c6d..02c40850a 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -58,7 +58,7 @@ defmodule Mix.Pleroma do [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ http_children(adapter) ++ - elasticsearch_children() + elasticsearch_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex index e28810f21..102bc5b63 100644 --- a/lib/mix/tasks/pleroma/search.ex +++ b/lib/mix/tasks/pleroma/search.ex @@ -5,48 +5,16 @@ defmodule Mix.Tasks.Pleroma.Search do use Mix.Task import Mix.Pleroma - import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Hashtag @shortdoc "Manages elasticsearch" def run(["import", "activities" | _rest]) do start_pleroma() - Elasticsearch.Index.Bulk.upload(Pleroma.Search.Elasticsearch.Cluster, - "activities", - Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities])) - #from(a in Activity, where: not ilike(a.actor, "%/relay")) - #|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - #|> Activity.with_preloaded_object() - #|> Activity.with_preloaded_user_actor() - #|> get_all(:activities) - end - - defp get_all(query, index, max_id \\ nil) do - params = %{limit: 1000} - - params = - if max_id == nil do - params - else - Map.put(params, :max_id, max_id) - end - - res = - query - |> Pagination.fetch_paginated(params) - - if res == [] do - :ok - else - res - |> Enum.map(fn x -> Pleroma.Search.Elasticsearch.add_to_index(x) end) - - get_all(query, index, List.last(res).id) - end + Elasticsearch.Index.Bulk.upload( + Pleroma.Search.Elasticsearch.Cluster, + "activities", + Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities]) + ) end end diff --git a/lib/mix/tasks/pleroma/search/elasticsearch.ex b/lib/mix/tasks/pleroma/search/elasticsearch.ex deleted file mode 100644 index 1d7d7a29a..000000000 --- a/lib/mix/tasks/pleroma/search/elasticsearch.ex +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Mix.Tasks.Pleroma.Search.Elasticsearch do - alias Mix.Tasks.Elasticsearch.Build - import Mix.Pleroma - - def run(["index" | args]) do - start_pleroma() - Build.run(args) - end -end -- 2.34.1