config/config.exs

@@ -851,6 +851,8 @@ config :pleroma, ConcurrentLimiter, [
   {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
 ]
 
+config :pleroma, :search, provider: Pleroma.Search.Builtin
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

lib/mix/tasks/pleroma/activity.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Activity do
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Pagination
+  require Logger
+  import Mix.Pleroma
+  import Ecto.Query
+
+  def run(["get", id | _rest]) do
+    start_pleroma()
+
+    id
+    |> Activity.get_by_id()
+    |> IO.inspect()
+  end
+
+  def run(["delete_by_keyword", user, keyword | _rest]) do
+    start_pleroma()
+    u = User.get_by_nickname(user)
+
+    Activity
+    |> Activity.with_preloaded_object()
+    |> Activity.restrict_deactivated_users()
+    |> Activity.Queries.by_author(u)
+    |> query_with(keyword)
+    |> Pagination.fetch_paginated(
+      %{"offset" => 0, "limit" => 20, "skip_order" => false},
+      :offset
+    )
+    |> Enum.map(fn x -> CommonAPI.delete(x.id, u) end)
+    |> Enum.count()
+    |> IO.puts()
+  end
+
+  defp query_with(q, search_query) do
+    %{rows: [[tsc]]} =
+      Ecto.Adapters.SQL.query!(
+        Pleroma.Repo,
+        "select current_setting('default_text_search_config')::regconfig::oid;"
+      )
+
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
+          ^tsc,
+          o.data,
+          ^search_query
+        )
+    )
+  end
+end

lib/mix/tasks/pleroma/search.ex

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search do
+  use Mix.Task
+  import Mix.Pleroma
+  import Ecto.Query
+  alias Pleroma.Activity
+  alias Pleroma.Pagination
+  alias Pleroma.User
+  alias Pleroma.Hashtag
+
+  @shortdoc "Manages elasticsearch"
+
+  def run(["import", "activities" | _rest]) do
+    start_pleroma()
+
+    from(a in Activity, where: not ilike(a.actor, "%/relay"))
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> Activity.with_preloaded_object()
+    |> Activity.with_preloaded_user_actor()
+    |> get_all(:activities)
+  end
+
+  def run(["import", "users" | _rest]) do
+    start_pleroma()
+
+    from(u in User, where: u.nickname not in ["internal.fetch", "relay"])
+    |> get_all(:users)
+  end
+
+  def run(["import", "hashtags" | _rest]) do
+    start_pleroma()
+
+    from(h in Hashtag)
+    |> Pleroma.Repo.all()
+    |> Pleroma.Elasticsearch.bulk_post(:hashtags)
+  end
+
+  defp get_all(query, index, max_id \\ nil) do
+    params = %{limit: 1000}
+
+    params =
+      if max_id == nil do
+        params
+      else
+        Map.put(params, :max_id, max_id)
+      end
+
+    res =
+      query
+      |> Pagination.fetch_paginated(params)
+
+    if res == [] do
+      :ok
+    else
+      res
+      |> Pleroma.Elasticsearch.bulk_post(index)
+
+      get_all(query, index, List.last(res).id)
+    end
+  end
+end

lib/pleroma/elasticsearch/document_mappings/activity.ex

@@ -0,0 +1,15 @@
+defmodule Pleroma.Elasticsearch.DocumentMappings.Activity do
+  alias Pleroma.Object
+
+  def id(obj), do: obj.id
+
+  def encode(%{object: %{data: %{"type" => "Note"}}} = activity) do
+    %{
+      _timestamp: activity.inserted_at,
+      user: activity.user_actor.nickname,
+      content: activity.object.data["content"],
+      instance: URI.parse(activity.user_actor.ap_id).host,
+      hashtags: Object.hashtags(activity.object)
+    }
+  end
+end

lib/pleroma/elasticsearch/document_mappings/hashtag.ex

@@ -0,0 +1,17 @@
+defmodule Pleroma.Elasticsearch.DocumentMappings.Hashtag do
+  def id(obj), do: obj.id
+
+  def encode(%{timestamp: _} = hashtag) do
+    %{
+      hashtag: hashtag.name,
+      timestamp: hashtag.timestamp
+    }
+  end
+
+  def encode(hashtag) do
+    %{
+      hashtag: hashtag.name,
+      timestamp: hashtag.inserted_at
+    }
+  end
+end

lib/pleroma/elasticsearch/document_mappings/user.ex

@@ -0,0 +1,13 @@
+defmodule Pleroma.Elasticsearch.DocumentMappings.User do
+  def id(obj), do: obj.id
+
+  def encode(%{actor_type: "Person"} = user) do
+    %{
+      timestamp: user.inserted_at,
+      instance: URI.parse(user.ap_id).host,
+      nickname: user.nickname,
+      bio: user.bio,
+      display_name: user.name
+    }
+  end
+end

lib/pleroma/elasticsearch/store.ex

@@ -0,0 +1,254 @@
+defmodule Pleroma.Elasticsearch do
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Object
+  alias Pleroma.Elasticsearch.DocumentMappings
+  alias Pleroma.Config
+  require Logger
+
+  defp url do
+    Config.get([:elasticsearch, :url])
+  end
+
+  defp enabled? do
+    Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch
+  end
+
+  def delete_by_id(:activity, id) do
+    if enabled?() do
+      Elastix.Document.delete(url(), "activities", "activity", id)
+    end
+  end
+
+  def put_by_id(:activity, id) do
+    id
+    |> Activity.get_by_id_with_object()
+    |> maybe_put_into_elasticsearch()
+  end
+
+  def maybe_put_into_elasticsearch({:ok, item}) do
+    maybe_put_into_elasticsearch(item)
+  end
+
+  def maybe_put_into_elasticsearch(
+        %{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity
+      ) do
+    if enabled?() do
+      actor = Pleroma.Activity.user_actor(activity)
+
+      activity
+      |> Map.put(:user_actor, actor)
+      |> put()
+    end
+  end
+
+  def maybe_put_into_elasticsearch(%User{actor_type: "Person"} = user) do
+    if enabled?() do
+      put(user)
+    end
+  end
+
+  def maybe_put_into_elasticsearch(_) do
+    {:ok, :skipped}
+  end
+
+  def maybe_bulk_post(data, type) do
+    if enabled?() do
+      bulk_post(data, type)
+    end
+  end
+
+  def put(%Activity{} = activity) do
+    with {:ok, _} <-
+           Elastix.Document.index(
+             url(),
+             "activities",
+             "activity",
+             DocumentMappings.Activity.id(activity),
+             DocumentMappings.Activity.encode(activity)
+           ) do
+      activity
+      |> Map.get(:object)
+      |> Object.hashtags()
+      |> Enum.map(fn x ->
+        %{id: x, name: x, timestamp: DateTime.to_iso8601(DateTime.utc_now())}
+      end)
+      |> bulk_post(:hashtags)
+    else
+      {:error, %{reason: err}} ->
+        Logger.error("Could not put activity: #{err}")
+        :skipped
+    end
+  end
+
+  def put(%User{} = user) do
+    with {:ok, _} <-
+           Elastix.Document.index(
+             url(),
+             "users",
+             "user",
+             DocumentMappings.User.id(user),
+             DocumentMappings.User.encode(user)
+           ) do
+      :ok
+    else
+      {:error, %{reason: err}} ->
+        Logger.error("Could not put user: #{err}")
+        :skipped
+    end
+  end
+
+  def bulk_post(data, :activities) do
+    d =
+      data
+      |> Enum.filter(fn x ->
+        t =
+          x.object
+          |> Map.get(:data, %{})
+          |> Map.get("type", "")
+
+        t == "Note"
+      end)
+      |> Enum.map(fn d ->
+        [
+          %{index: %{_id: DocumentMappings.Activity.id(d)}},
+          DocumentMappings.Activity.encode(d)
+        ]
+      end)
+      |> List.flatten()
+
+    with {:ok, %{body: %{"errors" => false}}} <-
+           Elastix.Bulk.post(
+             url(),
+             d,
+             index: "activities",
+             type: "activity"
+           ) do
+      :ok
+    else
+      {:error, %{reason: err}} ->
+        Logger.error("Could not bulk put activity: #{err}")
+        :skipped
+
+      {:ok, %{body: body}} ->
+        IO.inspect(body)
+        :skipped
+    end
+  end
+
+  def bulk_post(data, :users) do
+    d =
+      data
+      |> Enum.map(fn d ->
+        [
+          %{index: %{_id: DocumentMappings.User.id(d)}},
+          DocumentMappings.User.encode(d)
+        ]
+      end)
+      |> List.flatten()
+
+    with {:ok, %{body: %{"errors" => false}}} <-
+           Elastix.Bulk.post(
+             url(),
+             d,
+             index: "users",
+             type: "user"
+           ) do
+      :ok
+    else
+      {:error, %{reason: err}} ->
+        Logger.error("Could not bulk put users: #{err}")
+        :skipped
+
+      {:ok, %{body: body}} ->
+        IO.inspect(body)
+        :skipped
+    end
+  end
+
+  def bulk_post(data, :hashtags) when is_list(data) do
+    d =
+      data
+      |> Enum.map(fn d ->
+        [
+          %{index: %{_id: DocumentMappings.Hashtag.id(d)}},
+          DocumentMappings.Hashtag.encode(d)
+        ]
+      end)
+      |> List.flatten()
+
+    with {:ok, %{body: %{"errors" => false}}} <-
+           Elastix.Bulk.post(
+             url(),
+             d,
+             index: "hashtags",
+             type: "hashtag"
+           ) do
+      :ok
+    else
+      {:error, %{reason: err}} ->
+        Logger.error("Could not bulk put hashtags: #{err}")
+        :skipped
+
+      {:ok, %{body: body}} ->
+        IO.inspect(body)
+        :skipped
+    end
+  end
+
+  def bulk_post(_, :hashtags), do: {:ok, nil}
+
+  def search(_, _, _, :skip), do: []
+
+  def search(:raw, index, type, q) do
+    with {:ok, raw_results} <- Elastix.Search.search(url(), index, [type], q) do
+      results =
+        raw_results
+        |> Map.get(:body, %{})
+        |> Map.get("hits", %{})
+        |> Map.get("hits", [])
+
+      {:ok, results}
+    else
+      {:error, e} ->
+        Logger.error(e)
+        {:error, e}
+    end
+  end
+
+  def search(:activities, q) do
+    with {:ok, results} <- search(:raw, "activities", "activity", q) do
+      results
+      |> Enum.map(fn result -> result["_id"] end)
+      |> Pleroma.Activity.all_by_ids_with_object()
+      |> Enum.sort(&(&1.inserted_at >= &2.inserted_at))
+    else
+      e ->
+        Logger.error(e)
+        []
+    end
+  end
+
+  def search(:users, q) do
+    with {:ok, results} <- search(:raw, "users", "user", q) do
+      results
+      |> Enum.map(fn result -> result["_id"] end)
+      |> Pleroma.User.get_all_by_ids()
+    else
+      e ->
+        Logger.error(e)
+        []
+    end
+  end
+
+  def search(:hashtags, q) do
+    with {:ok, results} <- search(:raw, "hashtags", "hashtag", q) do
+      results
+      |> Enum.map(fn result -> result["_source"]["hashtag"] end)
+    else
+      e ->
+        Logger.error(e)
+        []
+    end
+  end
+end

lib/pleroma/search.ex

@@ -0,0 +1,12 @@
+defmodule Pleroma.Search do
+  @type search_map :: %{
+          statuses: [map],
+          accounts: [map],
+          hashtags: [map]
+        }
+
+  @doc """
+  Searches for stuff
+  """
+  @callback search(map, map, keyword) :: search_map
+end

lib/pleroma/search/builtin.ex

@@ -0,0 +1,137 @@
+defmodule Pleroma.Search.Builtin do
+  @behaviour Pleroma.Search
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Activity
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.Endpoint
+
+  require Logger
+
+  @impl Pleroma.Search
+  def search(_conn, %{q: query} = params, options) do
+    version = Keyword.get(options, :version)
+    timeout = Keyword.get(Repo.config(), :timeout, 15_000)
+    default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}
+
+    default_values
+    |> Enum.map(fn {resource, default_value} ->
+      if params[:type] in [nil, resource] do
+        {resource, fn -> resource_search(version, resource, query, options) end}
+      else
+        {resource, fn -> default_value end}
+      end
+    end)
+    |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end,
+      timeout: timeout,
+      on_timeout: :kill_task
+    )
+    |> Enum.reduce(default_values, fn
+      {:ok, {resource, result}}, acc ->
+        Map.put(acc, resource, result)
+
+      _error, acc ->
+        acc
+    end)
+  end
+
+  defp resource_search(_, "accounts", query, options) do
+    accounts = with_fallback(fn -> User.search(query, options) end)
+
+    AccountView.render("index.json",
+      users: accounts,
+      for: options[:for_user],
+      embed_relationships: options[:embed_relationships]
+    )
+  end
+
+  defp resource_search(_, "statuses", query, options) do
+    statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end)
+
+    StatusView.render("index.json",
+      activities: statuses,
+      for: options[:for_user],
+      as: :activity
+    )
+  end
+
+  defp resource_search(:v2, "hashtags", query, options) do
+    tags_path = Endpoint.url() <> "/tag/"
+
+    query
+    |> prepare_tags(options)
+    |> Enum.map(fn tag ->
+      %{name: tag, url: tags_path <> tag}
+    end)
+  end
+
+  defp resource_search(:v1, "hashtags", query, options) do
+    prepare_tags(query, options)
+  end
+
+  defp prepare_tags(query, options) do
+    tags =
+      query
+      |> preprocess_uri_query()
+      |> String.split(~r/[^#\w]+/u, trim: true)
+      |> Enum.uniq_by(&String.downcase/1)
+
+    explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end)
+
+    tags =
+      if Enum.any?(explicit_tags) do
+        explicit_tags
+      else
+        tags
+      end
+
+    tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
+
+    tags =
+      if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
+        add_joined_tag(tags)
+      else
+        tags
+      end
+
+    Pleroma.Pagination.paginate(tags, options)
+  end
+
+  # If `query` is a URI, returns last component of its path, otherwise returns `query`
+  defp preprocess_uri_query(query) do
+    if query =~ ~r/https?:\/\// do
+      query
+      |> String.trim_trailing("/")
+      |> URI.parse()
+      |> Map.get(:path)
+      |> String.split("/")
+      |> Enum.at(-1)
+    else
+      query
+    end
+  end
+
+  defp add_joined_tag(tags) do
+    tags
+    |> Kernel.++([joined_tag(tags)])
+    |> Enum.uniq_by(&String.downcase/1)
+  end
+
+  defp joined_tag(tags) do
+    tags
+    |> Enum.map(fn tag -> String.capitalize(tag) end)
+    |> Enum.join()
+  end
+
+  defp with_fallback(f, fallback \\ []) do
+    try do
+      f.()
+    rescue
+      error ->
+        Logger.error("#{__MODULE__} search error: #{inspect(error)}")
+        fallback
+    end
+  end
+end

lib/pleroma/search/elasticsearch.ex

@@ -0,0 +1,157 @@
+defmodule Pleroma.Search.Elasticsearch do
+  @behaviour Pleroma.Search
+
+  alias Pleroma.Activity
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Search.Elasticsearch.Parsers
+  alias Pleroma.Web.Endpoint
+
+  def es_query(:activity, query) do
+    must = Parsers.Activity.parse(query)
+
+    if must == [] do
+      :skip
+    else
+      %{
+        size: 50,
+        terminate_after: 50,
+        timeout: "5s",
+        sort: [
+          "_score",
+          %{_timestamp: %{order: "desc", format: "basic_date_time"}}
+        ],
+        query: %{
+          bool: %{
+            must: must
+          }
+        }
+      }
+    end
+  end
+
+  def es_query(:user, query) do
+    must = Parsers.User.parse(query)
+
+    if must == [] do
+      :skip
+    else
+      %{
+        size: 50,
+        terminate_after: 50,
+        timeout: "5s",
+        sort: [
+          "_score"
+        ],
+        query: %{
+          bool: %{
+            must: must
+          }
+        }
+      }
+    end
+  end
+
+  def es_query(:hashtag, query) do
+    must = Parsers.Hashtag.parse(query)
+
+    if must == [] do
+      :skip
+    else
+      %{
+        size: 50,
+        terminate_after: 50,
+        timeout: "5s",
+        sort: [
+          "_score"
+        ],
+        query: %{
+          bool: %{
+            must: Parsers.Hashtag.parse(query)
+          }
+        }
+      }
+    end
+  end
+
+  defp maybe_fetch(:activity, search_query) do
+    with true <- Regex.match?(~r/https?:/, search_query),
+         {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]) do
+      activity
+    else
+      _ -> nil
+    end
+  end
+
+  @impl Pleroma.Search
+  def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) do
+    parsed_query =
+      query
+      |> String.trim()
+      |> SearchParser.parse!()
+
+    activity_fetch_task =
+      Task.async(fn ->
+        maybe_fetch(:activity, String.trim(query))
+      end)
+
+    activity_task =
+      Task.async(fn ->
+        q = es_query(:activity, parsed_query)
+
+        Pleroma.Elasticsearch.search(:activities, q)
+        |> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
+      end)
+
+    user_task =
+      Task.async(fn ->
+        q = es_query(:user, parsed_query)
+
+        Pleroma.Elasticsearch.search(:users, q)
+        |> Enum.filter(fn x -> Pleroma.User.visible_for(x, user) == :visible end)
+      end)
+
+    hashtag_task =
+      Task.async(fn ->
+        q = es_query(:hashtag, parsed_query)
+
+        Pleroma.Elasticsearch.search(:hashtags, q)
+      end)
+
+    activity_results = Task.await(activity_task)
+    user_results = Task.await(user_task)
+    hashtag_results = Task.await(hashtag_task)
+    direct_activity = Task.await(activity_fetch_task)
+
+    activity_results =
+      if direct_activity == nil do
+        activity_results
+      else
+        [direct_activity | activity_results]
+      end
+
+    %{
+      "accounts" =>
+        AccountView.render("index.json",
+          users: user_results,
+          for: user
+        ),
+      "hashtags" =>
+        Enum.map(hashtag_results, fn x ->
+          %{
+            url: Endpoint.url() <> "/tag/" <> x,
+            name: x
+          }
+        end),
+      "statuses" =>
+        StatusView.render("index.json",
+          activities: activity_results,
+          for: user,
+          as: :activity
+        )
+    }
+  end
+end

lib/pleroma/search/elasticsearch/activity_parser.ex

@@ -0,0 +1,38 @@
+defmodule Pleroma.Search.Elasticsearch.Parsers.Activity do
+  defp to_es(term) when is_binary(term) do
+    %{
+      match: %{
+        content: %{
+          query: term,
+          operator: "AND"
+        }
+      }
+    }
+  end
+
+  defp to_es({:quoted, term}), do: to_es(term)
+
+  defp to_es({:filter, ["hashtag", query]}) do
+    %{
+      term: %{
+        hashtags: %{
+          value: query
+        }
+      }
+    }
+  end
+
+  defp to_es({:filter, [field, query]}) do
+    %{
+      term: %{
+        field => %{
+          value: query
+        }
+      }
+    }
+  end
+
+  def parse(q) do
+    Enum.map(q, &to_es/1)
+  end
+end

lib/pleroma/search/elasticsearch/hashtag_parser.ex

@@ -0,0 +1,30 @@
+defmodule Pleroma.Search.Elasticsearch.Parsers.Hashtag do
+  defp to_es(term) when is_binary(term) do
+    %{
+      term: %{
+        hashtag: %{
+          value: String.downcase(term)
+        }
+      }
+    }
+  end
+
+  defp to_es({:quoted, term}), do: to_es(term)
+
+  defp to_es({:filter, ["hashtag", query]}) do
+    %{
+      term: %{
+        hashtag: %{
+          value: String.downcase(query)
+        }
+      }
+    }
+  end
+
+  defp to_es({:filter, _}), do: nil
+
+  def parse(q) do
+    Enum.map(q, &to_es/1)
+    |> Enum.filter(fn x -> x != nil end)
+  end
+end

lib/pleroma/search/elasticsearch/user_paser.ex

@@ -0,0 +1,53 @@
+defmodule Pleroma.Search.Elasticsearch.Parsers.User do
+  defp to_es(term) when is_binary(term) do
+    %{
+      bool: %{
+        minimum_should_match: 1,
+        should: [
+          %{
+            match: %{
+              bio: %{
+                query: term,
+                operator: "AND"
+              }
+            }
+          },
+          %{
+            term: %{
+              nickname: %{
+                value: term
+              }
+            }
+          },
+          %{
+            match: %{
+              display_name: %{
+                query: term,
+                operator: "AND"
+              }
+            }
+          }
+        ]
+      }
+    }
+  end
+
+  defp to_es({:quoted, term}), do: to_es(term)
+
+  defp to_es({:filter, ["user", query]}) do
+    %{
+      term: %{
+        nickname: %{
+          value: query
+        }
+      }
+    }
+  end
+
+  defp to_es({:filter, _}), do: nil
+
+  def parse(q) do
+    Enum.map(q, &to_es/1)
+    |> Enum.filter(fn x -> x != nil end)
+  end
+end

lib/pleroma/user.ex

@@ -1088,6 +1088,7 @@ defmodule Pleroma.User do
 
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
+      Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user)
       set_cache(user)
     end
   end

lib/pleroma/web/activity_pub/pipeline.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
     case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do
       {:ok, {:ok, activity, meta}} ->
         side_effects().handle_after_transaction(meta)
+        side_effects().handle_after_transaction(activity)
         {:ok, activity, meta}
 
       {:ok, value} ->

lib/pleroma/web/activity_pub/side_effects.ex

@@ -537,6 +537,24 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   @impl true
+  def handle_after_transaction(%Pleroma.Activity{data: %{"type" => "Create"}} = activity) do
+    Pleroma.Elasticsearch.put_by_id(:activity, activity.id)
+  end
+
+  def handle_after_transaction(%Pleroma.Activity{
+        data: %{"type" => "Delete", "deleted_activity_id" => id}
+      }) do
+    Pleroma.Elasticsearch.delete_by_id(:activity, id)
+  end
+
+  def handle_after_transaction(%Pleroma.Activity{}) do
+    :ok
+  end
+
+  def handle_after_transaction(%Pleroma.Object{}) do
+    :ok
+  end
+
   def handle_after_transaction(meta) do
     meta
     |> send_notifications()

lib/pleroma/web/common_api.ex

@@ -397,7 +397,13 @@ defmodule Pleroma.Web.CommonAPI do
 
   def post(user, %{status: _} = data) do
     with {:ok, draft} <- ActivityDraft.create(user, data) do
-      ActivityPub.create(draft.changes, draft.preview?)
+      activity = ActivityPub.create(draft.changes, draft.preview?)
+
+      unless draft.preview? do
+        Pleroma.Elasticsearch.maybe_put_into_elasticsearch(activity)
+      end
+
+      activity
     end
   end
 

lib/pleroma/web/mastodon_api/controllers/search_controller.ex

@@ -5,13 +5,9 @@
 defmodule Pleroma.Web.MastodonAPI.SearchController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Activity
-  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ControllerHelper
-  alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Plugs.RateLimiter
 
@@ -43,34 +39,13 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
   def search2(conn, params), do: do_search(:v2, conn, params)
   def search(conn, params), do: do_search(:v1, conn, params)
 
-  defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do
-    query = String.trim(query)
-    options = search_options(params, user)
-    timeout = Keyword.get(Repo.config(), :timeout, 15_000)
-    default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}
+  defp do_search(version, %{assigns: %{user: user}} = conn, params) do
+    options =
+      search_options(params, user)
+      |> Keyword.put(:version, version)
 
-    result =
-      default_values
-      |> Enum.map(fn {resource, default_value} ->
-        if params[:type] in [nil, resource] do
-          {resource, fn -> resource_search(version, resource, query, options) end}
-        else
-          {resource, fn -> default_value end}
-        end
-      end)
-      |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end,
-        timeout: timeout,
-        on_timeout: :kill_task
-      )
-      |> Enum.reduce(default_values, fn
-        {:ok, {resource, result}}, acc ->
-          Map.put(acc, resource, result)
-
-        _error, acc ->
-          acc
-      end)
-
-    json(conn, result)
+    search_provider = Pleroma.Config.get([:search, :provider])
+    json(conn, search_provider.search(conn, params, options))
   end
 
   defp search_options(params, user) do
@@ -87,104 +62,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
     |> Enum.filter(&elem(&1, 1))
   end
 
-  defp resource_search(_, "accounts", query, options) do
-    accounts = with_fallback(fn -> User.search(query, options) end)
-
-    AccountView.render("index.json",
-      users: accounts,
-      for: options[:for_user],
-      embed_relationships: options[:embed_relationships]
-    )
-  end
-
-  defp resource_search(_, "statuses", query, options) do
-    statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end)
-
-    StatusView.render("index.json",
-      activities: statuses,
-      for: options[:for_user],
-      as: :activity
-    )
-  end
-
-  defp resource_search(:v2, "hashtags", query, options) do
-    tags_path = Endpoint.url() <> "/tag/"
-
-    query
-    |> prepare_tags(options)
-    |> Enum.map(fn tag ->
-      %{name: tag, url: tags_path <> tag}
-    end)
-  end
-
-  defp resource_search(:v1, "hashtags", query, options) do
-    prepare_tags(query, options)
-  end
-
-  defp prepare_tags(query, options) do
-    tags =
-      query
-      |> preprocess_uri_query()
-      |> String.split(~r/[^#\w]+/u, trim: true)
-      |> Enum.uniq_by(&String.downcase/1)
-
-    explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end)
-
-    tags =
-      if Enum.any?(explicit_tags) do
-        explicit_tags
-      else
-        tags
-      end
-
-    tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
-
-    tags =
-      if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
-        add_joined_tag(tags)
-      else
-        tags
-      end
-
-    Pleroma.Pagination.paginate(tags, options)
-  end
-
-  defp add_joined_tag(tags) do
-    tags
-    |> Kernel.++([joined_tag(tags)])
-    |> Enum.uniq_by(&String.downcase/1)
-  end
-
-  # If `query` is a URI, returns last component of its path, otherwise returns `query`
-  defp preprocess_uri_query(query) do
-    if query =~ ~r/https?:\/\// do
-      query
-      |> String.trim_trailing("/")
-      |> URI.parse()
-      |> Map.get(:path)
-      |> String.split("/")
-      |> Enum.at(-1)
-    else
-      query
-    end
-  end
-
-  defp joined_tag(tags) do
-    tags
-    |> Enum.map(fn tag -> String.capitalize(tag) end)
-    |> Enum.join()
-  end
-
-  defp with_fallback(f, fallback \\ []) do
-    try do
-      f.()
-    rescue
-      error ->
-        Logger.error("#{__MODULE__} search error: #{inspect(error)}")
-        fallback
-    end
-  end
-
   defp get_author(%{account_id: account_id}) when is_binary(account_id),
     do: User.get_cached_by_id(account_id)
 

mix.exs

@@ -197,6 +197,11 @@ defmodule Pleroma.Mixfile do
        ref: "289cda1b6d0d70ccb2ba508a2b0bd24638db2880"},
       {:eblurhash, "~> 1.1.0"},
       {:open_api_spex, "~> 3.10"},
+      {:elastix, ">= 0.0.0"},
+      {:search_parser,
+       git: "https://github.com/FloatingGhost/pleroma-contrib-search-parser.git",
+       ref: "08971a81e68686f9ac465cfb6661d51c5e4e1e7f"},
+      {:nimble_parsec, "~> 1.0", override: true},
 
       # indirect dependency version override
       {:plug, "~> 1.10.4", override: true},

mix.lock

@@ -34,6 +34,8 @@
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
   "ecto_sql": {:hex, :ecto_sql, "3.6.2", "9526b5f691701a5181427634c30655ac33d11e17e4069eff3ae1176c764e0ba3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ec9d7e6f742ea39b63aceaea9ac1d1773d574ea40df5a53ef8afbd9242fdb6b"},
   "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
+  "elasticsearch": {:hex, :elasticsearch, "1.0.1", "8339538d90af6b280f10ecd02b1eae372f09373e629b336a13461babf7366495", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "83e7d8b8bee3e7e19a06ab4d357d24845ac1da894e79678227fd52c0b7f71867"},
+  "elastix": {:hex, :elastix, "0.10.0", "7567da885677ba9deffc20063db5f3ca8cd10f23cff1ab3ed9c52b7063b7e340", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "5fb342ce068b20f7845f5dd198c2dc80d967deafaa940a6e51b846db82696d1d"},
   "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"},
   "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"},
   "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
@@ -81,7 +83,7 @@
   "mogrify": {:hex, :mogrify, "0.9.1", "a26f107c4987477769f272bd0f7e3ac4b7b75b11ba597fd001b877beffa9c068", [:mix], [], "hexpm", "134edf189337d2125c0948bf0c228fdeef975c594317452d536224069a5b7f05"},
   "mox": {:hex, :mox, "1.0.0", "4b3c7005173f47ff30641ba044eb0fe67287743eec9bd9545e37f3002b0a9f8b", [:mix], [], "hexpm", "201b0a20b7abdaaab083e9cf97884950f8a30a1350a1da403b3145e213c6f4df"},
   "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
-  "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm", "5c040b8469c1ff1b10093d3186e2e10dbe483cd73d79ec017993fb3985b8a9b3"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"},
   "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
   "oban": {:hex, :oban, "2.3.4", "ec7509b9af2524d55f529cb7aee93d36131ae0bf0f37706f65d2fe707f4d9fd8", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c70ca0434758fd1805422ea4446af5e910ddc697c0c861549c8f0eb0cfbd2fdf"},
@@ -112,6 +114,8 @@
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "recon": {:hex, :recon, "2.5.1", "430ffa60685ac1efdfb1fe4c97b8767c92d0d92e6e7c3e8621559ba77598678a", [:mix, :rebar3], [], "hexpm", "5721c6b6d50122d8f68cccac712caa1231f97894bab779eff5ff0f886cb44648"},
   "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8", [ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"]},
+  "retry": {:hex, :retry, "0.15.0", "ba6aaeba92905a396c18c299a07e638947b2ba781e914f803202bc1b9ae867c3", [:mix], [], "hexpm", "93d3310bce78c0a30cc94610684340a14adfc9136856a3f662e4d9ce6013c784"},
+  "search_parser": {:git, "https://github.com/FloatingGhost/pleroma-contrib-search-parser.git", "08971a81e68686f9ac465cfb6661d51c5e4e1e7f", [ref: "08971a81e68686f9ac465cfb6661d51c5e4e1e7f"]},
   "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
   "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"},
@@ -125,6 +129,7 @@
   "ueberauth": {:hex, :ueberauth, "0.6.3", "d42ace28b870e8072cf30e32e385579c57b9cc96ec74fa1f30f30da9c14f3cc0", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "afc293d8a1140d6591b53e3eaf415ca92842cb1d32fad3c450c6f045f7f91b60"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
+  "vex": {:hex, :vex, "0.9.0", "613ea5eb3055662e7178b83e25b2df0975f68c3d8bb67c1645f0573e1a78d606", [:mix], [], "hexpm", "c69fff44d5c8aa3f1faee71bba1dcab05dd36364c5a629df8bb11751240c857f"},
   "web_push_encryption": {:git, "https://github.com/lanodan/elixir-web-push-encryption.git", "026a043037a89db4da8f07560bc8f9c68bcf0cc0", [branch: "bugfix/otp-24"]},
   "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []},
 }

priv/es-mappings/activity.json

@@ -0,0 +1,21 @@
+{
+  "properties": {
+    "_timestamp": {
+      "type": "date",
+      "index": true
+    },
+    "instance": {
+      "type": "keyword"
+    },
+    "content": {
+      "type": "text"
+    },
+    "hashtags": {
+      "type": "keyword"
+    },
+    "user": {
+      "type": "text"
+    }
+  }
+}
+

priv/es-mappings/hashtag.json

@@ -0,0 +1,11 @@
+{
+  "properties": {
+    "timestamp": {
+      "type": "date",
+      "index": true
+    },
+    "hashtag": {
+      "type": "text"
+    }
+  }
+}

priv/es-mappings/user.json

@@ -0,0 +1,18 @@
+{
+  "properties": {
+    "timestamp": {
+      "type": "date",
+      "index": true
+    },
+    "instance": {
+      "type": "keyword"
+    },
+    "nickname": {
+      "type": "text"
+    },
+    "bio": {
+      "type": "text"
+    }
+  }
+}
+