From 0122512c2aa07a9d1dcb58e67a09513f3d687790 Mon Sep 17 00:00:00 2001
From: sadposter <hannah+pleroma@coffee-and-dreams.uk>
Date: Thu, 30 Jun 2022 19:41:28 +0100
Subject: [PATCH] fix import from live

---
 config/config.exs               |  2 +-
 docs/configuration/search.md    |  6 ++--
 lib/mix/pleroma.ex              | 13 ++++++++-
 lib/mix/tasks/pleroma/search.ex | 52 +++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 5 deletions(-)
 create mode 100644 lib/mix/tasks/pleroma/search.ex

diff --git a/config/config.exs b/config/config.exs
index 727a2b0cb..b00fd18e3 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -874,7 +874,7 @@ config :pleroma, Pleroma.Search.Elasticsearch.Cluster,
       settings: "priv/es-mappings/activity.json",
       store: Pleroma.Search.Elasticsearch.Store,
       sources: [Pleroma.Activity],
-      bulk_page_size: 5000,
+      bulk_page_size: 1000,
       bulk_wait_interval: 15_000
     }
   }
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 7c1093ab9..e1f23b505 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -154,10 +154,10 @@ To start the initial indexing, run the `build` command:
 
 === "OTP"
 ```sh
-./bin/pleroma_ctl search.elasticsearch index activities --cluster Pleroma.Search.Elasticsearch.Cluster
+./bin/pleroma_ctl search import activities
 ```
 
 === "From Source"
 ```sh
-mix elasticsearch.build activities --cluster Pleroma.Search.Elasticsearch.Cluster
-```
\ No newline at end of file
+mix pleroma.search import activities
+```
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index 2b6c7d6bb..36d3b4c6d 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -57,7 +57,8 @@ defmodule Mix.Pleroma do
         {Majic.Pool,
          [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
       ] ++
-        http_children(adapter)
+        http_children(adapter) ++
+	elasticsearch_children()
 
     cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
 
@@ -136,4 +137,14 @@ defmodule Mix.Pleroma do
   end
 
   defp http_children(_), do: []
+
+  def elasticsearch_children do
+    config = Pleroma.Config.get([Pleroma.Search, :module])
+
+    if config == Pleroma.Search.Elasticsearch do
+      [Pleroma.Search.Elasticsearch.Cluster]
+    else
+      []
+    end
+  end
 end
diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex
new file mode 100644
index 000000000..e28810f21
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search.ex
@@ -0,0 +1,52 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search do
+  use Mix.Task
+  import Mix.Pleroma
+  import Ecto.Query
+  alias Pleroma.Activity
+  alias Pleroma.Pagination
+  alias Pleroma.User
+  alias Pleroma.Hashtag
+
+  @shortdoc "Manages elasticsearch"
+
+  def run(["import", "activities" | _rest]) do
+    start_pleroma()
+
+    Elasticsearch.Index.Bulk.upload(Pleroma.Search.Elasticsearch.Cluster, 
+	"activities",
+	Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities]))
+    #from(a in Activity, where: not ilike(a.actor, "%/relay"))
+    #|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    #|> Activity.with_preloaded_object()
+    #|> Activity.with_preloaded_user_actor()
+    #|> get_all(:activities)
+  end
+
+  defp get_all(query, index, max_id \\ nil) do
+    params = %{limit: 1000}
+
+    params =
+      if max_id == nil do
+        params
+      else
+        Map.put(params, :max_id, max_id)
+      end
+
+    res =
+      query
+      |> Pagination.fetch_paginated(params)
+
+    if res == [] do
+      :ok
+    else
+      res
+      |> Enum.map(fn x -> Pleroma.Search.Elasticsearch.add_to_index(x) end)
+
+      get_all(query, index, List.last(res).id)
+    end
+  end
+end