diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml
index 335f3c8e8..28043aa65 100644
--- a/.woodpecker/.release.yml
+++ b/.woodpecker/.release.yml
@@ -16,7 +16,9 @@ pipeline:
glibc:
when:
event:
- - tag
+ - push
+ branch:
+ - develop
secrets:
- SCW_ACCESS_KEY
- SCW_SECRET_KEY
@@ -44,7 +46,9 @@ pipeline:
musl:
when:
event:
- - tag
+ - push
+ branch:
+ - develop
secrets:
- SCW_ACCESS_KEY
- SCW_SECRET_KEY
diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml
index cef743643..6724d363d 100644
--- a/.woodpecker/.test.yml
+++ b/.woodpecker/.test.yml
@@ -11,6 +11,7 @@ pipeline:
when:
event:
- push
+ - pull_request
environment:
MIX_ENV: test
commands:
@@ -25,6 +26,7 @@ pipeline:
when:
event:
- push
+ - pull_request
environment:
MIX_ENV: test
POSTGRES_DB: pleroma_test
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9513a54fa..2119c8e21 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Readded mastoFE
- Added support for custom emoji reactions
- Added `emoji_url` in notifications to allow for custom emoji rendering
+- Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field.
### Fixed
- Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies
@@ -116,6 +117,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available.
- AdminAPI: sort users so the newest are at the top.
- ActivityPub Client-to-Server(C2S): Limitation on the type of Activity/Object are lifted as they are now passed through ObjectValidators
+- MRF (`AntiFollowbotPolicy`): Bot accounts are now also considered followbots. Users can still allow bots to follow them by first following the bot.
### Added
diff --git a/config/config.exs b/config/config.exs
index 15bcfb2b9..8de699e3c 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -568,7 +568,8 @@
remote_fetcher: 2,
attachments_cleanup: 1,
new_users_digest: 1,
- mute_expire: 5
+ mute_expire: 5,
+ search_indexing: 10
],
plugins: [Oban.Plugins.Pruner],
crontab: [
@@ -579,7 +580,8 @@
config :pleroma, :workers,
retries: [
federator_incoming: 5,
- federator_outgoing: 5
+ federator_outgoing: 5,
+ search_indexing: 2
]
config :pleroma, Pleroma.Formatter,
@@ -846,17 +848,32 @@
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
- {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
+ {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
+ {Pleroma.Search, [max_running: 30, max_waiting: 50]}
]
-config :pleroma, :search, provider: Pleroma.Search.Builtin
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
-config :pleroma, :telemetry,
- slow_queries_logging: [
- enabled: false,
- min_duration: 500_000,
- exclude_sources: [nil, "oban_jobs"]
- ]
+config :pleroma, Pleroma.Search.Meilisearch,
+ url: "http://127.0.0.1:7700/",
+ private_key: nil,
+ initial_indexing_chunk_size: 100_000
+
+config :pleroma, Pleroma.Search.Elasticsearch.Cluster,
+ url: "http://localhost:9200",
+ username: "elastic",
+ password: "changeme",
+ api: Elasticsearch.API.HTTP,
+ json_library: Jason,
+ indexes: %{
+ activities: %{
+ settings: "priv/es-mappings/activity.json",
+ store: Pleroma.Search.Elasticsearch.Store,
+ sources: [Pleroma.Activity],
+ bulk_page_size: 1000,
+ bulk_wait_interval: 15_000
+ }
+ }
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
diff --git a/config/description.exs b/config/description.exs
index 4ca79ad51..9401bed5c 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3358,5 +3358,133 @@
]
}
]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Search,
+ type: :group,
+ description: "General search settings.",
+ children: [
+ %{
+ key: :module,
+ type: :keyword,
+ description: "Selected search module.",
+ suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Search.Meilisearch,
+ type: :group,
+ description: "Meilisearch settings.",
+ children: [
+ %{
+ key: :url,
+ type: :string,
+ description: "Meilisearch URL.",
+ suggestion: ["http://127.0.0.1:7700/"]
+ },
+ %{
+ key: :private_key,
+ type: :string,
+ description:
+ "Private key for meilisearch authentication, or `nil` to disable private key authentication.",
+ suggestion: [nil]
+ },
+ %{
+ key: :initial_indexing_chunk_size,
+ type: :int,
+ description:
+ "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <>
+ " since there's a limit on maximum insert size",
+ suggestion: [100_000]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Search.Elasticsearch.Cluster,
+ type: :group,
+ description: "Elasticsearch settings.",
+ children: [
+ %{
+ key: :url,
+ type: :string,
+ description: "Elasticsearch URL.",
+ suggestion: ["http://127.0.0.1:9200/"]
+ },
+ %{
+ key: :username,
+ type: :string,
+ description: "Username to connect to ES. Set to nil if your cluster is unauthenticated.",
+ suggestion: ["elastic"]
+ },
+ %{
+ key: :password,
+ type: :string,
+ description: "Password to connect to ES. Set to nil if your cluster is unauthenticated.",
+ suggestion: ["changeme"]
+ },
+ %{
+ key: :api,
+ type: :module,
+ description:
+ "The API module used by Elasticsearch. Should always be Elasticsearch.API.HTTP",
+ suggestion: [Elasticsearch.API.HTTP]
+ },
+ %{
+ key: :json_library,
+ type: :module,
+ description:
+ "The JSON module used to encode/decode when communicating with Elasticsearch",
+ suggestion: [Jason]
+ },
+ %{
+ key: :indexes,
+ type: :map,
+ description: "The indices to set up in Elasticsearch",
+ children: [
+ %{
+ key: :activities,
+ type: :map,
+ description: "Config for the index to use for activities",
+ children: [
+ %{
+ key: :settings,
+ type: :string,
+ description:
+ "Path to the file containing index settings for the activities index. Should contain a mapping.",
+ suggestion: ["priv/es-mappings/activity.json"]
+ },
+ %{
+ key: :store,
+ type: :module,
+ description: "The internal store module",
+ suggestion: [Pleroma.Search.Elasticsearch.Store]
+ },
+ %{
+ key: :sources,
+ type: {:list, :module},
+ description: "The internal types to use for this index",
+ suggestion: [[Pleroma.Activity]]
+ },
+ %{
+ key: :bulk_page_size,
+ type: :int,
+ description: "Size for bulk put requests, mostly used on building the index",
+ suggestion: [5000]
+ },
+ %{
+ key: :bulk_wait_interval,
+ type: :int,
+ description: "Time to wait between bulk put requests (in ms)",
+ suggestion: [15_000]
+ }
+ ]
+ }
+ ]
+ }
+ ]
}
]
diff --git a/config/test.exs b/config/test.exs
index a5bf3a4d1..7fbababdf 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -134,6 +134,10 @@
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
+
+config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
+
# Reduce recompilation time
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
diff --git a/docs/administration/updating.md b/docs/administration/updating.md
index ef2c9218c..01d3b9b0e 100644
--- a/docs/administration/updating.md
+++ b/docs/administration/updating.md
@@ -17,11 +17,11 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
## For from source installations (using git)
1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
-2. Run `git pull`. This pulls the latest changes from upstream.
+2. Run `git pull` [^1]. This pulls the latest changes from upstream.
3. Run `mix deps.get` [^1]. This pulls in any new dependencies.
4. Stop the Pleroma service.
5. Run `mix ecto.migrate` [^1] [^2]. This task performs database migrations, if there were any.
6. Start the Pleroma service.
-[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command.
+[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `git` and `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command.
[^2]: Prefix with `MIX_ENV=prod` to run it using the production config file.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 3b8f8cc52..50281f451 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -125,6 +125,8 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
* `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
* `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
+ * `Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy`: Drops follow requests from followbots. Users can still allow bots to follow them by first following the bot.
+ * `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)).
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
* `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
new file mode 100644
index 000000000..e1f23b505
--- /dev/null
+++ b/docs/configuration/search.md
@@ -0,0 +1,163 @@
+# Configuring search
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Built-in search
+
+To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
+
+While it has no external dependencies, it has problems with performance and relevancy.
+
+## Meilisearch
+
+Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million
+posts while idle and up to 7G while indexing initially). The disk usage for this additional index is also
+around 4 gigabytes. Like [RUM](./cheatsheet.md#rum-indexing-for-full-text-search) indexes, it offers considerably
+higher performance and ordering by timestamp in a reasonable amount of time.
+Additionally, the search results seem to be more accurate.
+
+Due to high memory usage, it may be best to set it up on a different machine, if running pleroma on a low-resource
+computer, and use private key authentication to secure the remote search instance.
+
+To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
+
+You then need to set the address of the meilisearch instance, and optionally the private key for authentication. You might
+also want to change the `initial_indexing_chunk_size` to be smaller if you're server is not very powerful, but not higher than `100_000`,
+because meilisearch will refuse to process it if it's too big. However, in general you want this to be as big as possible, because meilisearch
+indexes faster when it can process many posts in a single batch.
+
+> config :pleroma, Pleroma.Search.Meilisearch,
+> url: "http://127.0.0.1:7700/",
+> private_key: "private key",
+> initial_indexing_chunk_size: 100_000
+
+Information about setting up meilisearch can be found in the
+[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
+You probably want to start it with `MEILI_NO_ANALYTICS=true` environment variable to disable analytics.
+At least version 0.25.0 is required, but you are strongly adviced to use at least 0.26.0, as it introduces
+the `--enable-auto-batching` option which drastically improves performance. Without this option, the search
+is hardly usable on a somewhat big instance.
+
+### Private key authentication (optional)
+
+To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
+you have to get the _private key_, which is actually used for authentication.
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch show-keys
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch show-keys
+ ```
+
+You will see a "Default Admin API Key", this is the key you actually put into your configuration file.
+
+### Initial indexing
+
+After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed. You'll only
+have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
+consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
+million posts while idle and up to 7G while indexing initially, but your experience may be different).
+
+The sequence of actions is as follows:
+
+1. First, change the configuration to use `Pleroma.Search.Meilisearch` as the search backend
+2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything
+3. Start the initial indexing process (as described below with `index`),
+ and wait until the task says it sent everything from the database to index
+4. Wait until everything is actually indexed (by checking with `stats` as described below),
+ at this point you don't have to do anything, just wait a while.
+
+To start the initial indexing, run the `index` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch index
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch index
+ ```
+
+This will show you the total amount of posts to index, and then show you the amount of posts indexed currently, until the numbers eventually
+become the same. The posts are indexed in big batches and meilisearch will take some time to actually index them, even after you have
+inserted all the posts into it. Depending on the amount of posts, this may be as long as several hours. To get information about the status
+of indexing and how many posts have actually been indexed, use the `stats` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch stats
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch stats
+ ```
+
+### Clearing the index
+
+In case you need to clear the index (for example, to re-index from scratch, if that needs to happen for some reason), you can
+use the `clear` command:
+
+=== "OTP"
+ ```sh
+ ./bin/pleroma_ctl search.meilisearch clear
+ ```
+
+=== "From Source"
+ ```sh
+ mix pleroma.search.meilisearch clear
+ ```
+
+This will clear **all** the posts from the search index. Note, that deleted posts are also removed from index by the instance itself, so
+there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information
+that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
+depends on the amount of text in posts.
+
+## Elasticsearch
+
+As with meilisearch, this can be rather memory-hungry, but it is very good at what it does.
+
+To use [elasticsearch](https://www.elastic.co/), set the search module to `Pleroma.Search.Elasticsearch`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.Elasticsearch
+
+You then need to set the URL and authentication credentials if relevant.
+
+> config :pleroma, Pleroma.Search.Elasticsearch.Cluster,
+> url: "http://127.0.0.1:9200/",
+> username: "elastic",
+> password: "changeme",
+
+### Initial indexing
+
+After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed. You'll only
+have to do it one time, but it might take a while, depending on the amount of posts your instance has seen.
+
+The sequence of actions is as follows:
+
+1. First, change the configuration to use `Pleroma.Search.Elasticsearch` as the search backend
+2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything
+3. Start the initial indexing process (as described below with `index`),
+ and wait until the task says it sent everything from the database to index
+4. Wait until the index tasks exits
+
+To start the initial indexing, run the `build` command:
+
+=== "OTP"
+```sh
+./bin/pleroma_ctl search import activities
+```
+
+=== "From Source"
+```sh
+mix pleroma.search import activities
+```
diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md
index 518aca114..def718b95 100644
--- a/docs/development/API/differences_in_mastoapi_responses.md
+++ b/docs/development/API/differences_in_mastoapi_responses.md
@@ -241,6 +241,7 @@ Additional parameters can be added to the JSON body/Form data:
- `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
- `actor_type` - the type of this account.
- `accepts_chat_messages` - if false, this account will reject all chat messages.
+- `language` - user's preferred language for receiving emails (digest, confirmation, etc.)
All images (avatar, banner and background) can be reset to the default by sending an empty string ("") instead of a file.
@@ -292,6 +293,7 @@ Has these additional parameters (which are the same as in Pleroma-API):
- `captcha_token`: optional, contains provider-specific captcha token
- `captcha_answer_data`: optional, contains provider-specific captcha data
- `token`: invite token required when the registrations aren't public.
+- `language`: optional, user's preferred language for receiving emails (digest, confirmation, etc.), default to the language set in the `userLanguage` cookies or `Accept-Language` header.
## Instance
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index 2b6c7d6bb..02c40850a 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -57,7 +57,8 @@ def start_pleroma do
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
- http_children(adapter)
+ http_children(adapter) ++
+ elasticsearch_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
@@ -136,4 +137,14 @@ defp http_children(Tesla.Adapter.Gun) do
end
defp http_children(_), do: []
+
+ def elasticsearch_children do
+ config = Pleroma.Config.get([Pleroma.Search, :module])
+
+ if config == Pleroma.Search.Elasticsearch do
+ [Pleroma.Search.Elasticsearch.Cluster]
+ else
+ []
+ end
+ end
end
diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex
index 1fd880eab..102bc5b63 100644
--- a/lib/mix/tasks/pleroma/search.ex
+++ b/lib/mix/tasks/pleroma/search.ex
@@ -5,60 +5,16 @@
defmodule Mix.Tasks.Pleroma.Search do
use Mix.Task
import Mix.Pleroma
- import Ecto.Query
- alias Pleroma.Activity
- alias Pleroma.Pagination
- alias Pleroma.User
- alias Pleroma.Hashtag
@shortdoc "Manages elasticsearch"
def run(["import", "activities" | _rest]) do
start_pleroma()
- from(a in Activity, where: not ilike(a.actor, "%/relay"))
- |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
- |> Activity.with_preloaded_object()
- |> Activity.with_preloaded_user_actor()
- |> get_all(:activities)
- end
-
- def run(["import", "users" | _rest]) do
- start_pleroma()
-
- from(u in User, where: u.nickname not in ["internal.fetch", "relay"])
- |> get_all(:users)
- end
-
- def run(["import", "hashtags" | _rest]) do
- start_pleroma()
-
- from(h in Hashtag)
- |> Pleroma.Repo.all()
- |> Pleroma.Elasticsearch.bulk_post(:hashtags)
- end
-
- defp get_all(query, index, max_id \\ nil) do
- params = %{limit: 1000}
-
- params =
- if max_id == nil do
- params
- else
- Map.put(params, :max_id, max_id)
- end
-
- res =
- query
- |> Pagination.fetch_paginated(params)
-
- if res == [] do
- :ok
- else
- res
- |> Pleroma.Elasticsearch.bulk_post(index)
-
- get_all(query, index, List.last(res).id)
- end
+ Elasticsearch.Index.Bulk.upload(
+ Pleroma.Search.Elasticsearch.Cluster,
+ "activities",
+ Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities])
+ )
end
end
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..d4a83c3cd
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -0,0 +1,144 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
+ require Pleroma.Constants
+
+ import Mix.Pleroma
+ import Ecto.Query
+
+ import Pleroma.Search.Meilisearch,
+ only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1]
+
+ def run(["index"]) do
+ start_pleroma()
+
+ meili_version =
+ (
+ {:ok, result} = meili_get("/version")
+
+ result["pkgVersion"]
+ )
+
+ # The ranking rule syntax was changed but nothing about that is mentioned in the changelog
+ if not Version.match?(meili_version, ">= 0.25.0") do
+ raise "Meilisearch <0.24.0 not supported"
+ end
+
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/ranking-rules",
+ [
+ "published:desc",
+ "words",
+ "exactness",
+ "proximity",
+ "typo",
+ "attribute",
+ "sort"
+ ]
+ )
+
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/searchable-attributes",
+ [
+ "content"
+ ]
+ )
+
+ IO.puts("Created indices. Starting to insert posts.")
+
+ chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size])
+
+ Pleroma.Repo.transaction(
+ fn ->
+ query =
+ from(Pleroma.Object,
+ # Only index public and unlisted posts which are notes and have some text
+ where:
+ fragment("data->>'type' = 'Note'") and
+ (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or
+ fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())),
+ order_by: [desc: fragment("data->'published'")]
+ )
+
+ count = query |> Pleroma.Repo.aggregate(:count, :data)
+ IO.puts("Entries to index: #{count}")
+
+ Pleroma.Repo.stream(
+ query,
+ timeout: :infinity
+ )
+ |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
+ |> Stream.filter(fn o -> not is_nil(o) end)
+ |> Stream.chunk_every(chunk_size)
+ |> Stream.transform(0, fn objects, acc ->
+ new_acc = acc + Enum.count(objects)
+
+ # Reset to the beginning of the line and rewrite it
+ IO.write("\r")
+ IO.write("Indexed #{new_acc} entries")
+
+ {[objects], new_acc}
+ end)
+ |> Stream.each(fn objects ->
+ result =
+ meili_put(
+ "/indexes/objects/documents",
+ objects
+ )
+
+ with {:ok, res} <- result do
+ if not Map.has_key?(res, "uid") do
+ IO.puts("\nFailed to index: #{inspect(result)}")
+ end
+ else
+ e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}")
+ end
+ end)
+ |> Stream.run()
+ end,
+ timeout: :infinity
+ )
+
+ IO.write("\n")
+ end
+
+ def run(["clear"]) do
+ start_pleroma()
+
+ meili_delete!("/indexes/objects/documents")
+ end
+
+ def run(["show-keys", master_key]) do
+ start_pleroma()
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.get(
+ Path.join(endpoint, "/keys"),
+ [{"Authorization", "Bearer #{master_key}"}]
+ )
+
+ decoded = Jason.decode!(result.body)
+
+ if decoded["results"] do
+ Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
+ IO.puts("#{desc}: #{key}")
+ end)
+ else
+ IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}")
+ end
+ end
+
+ def run(["stats"]) do
+ start_pleroma()
+
+ {:ok, result} = meili_get("/indexes/objects/stats")
+ IO.puts("Number of entries: #{result["numberOfDocuments"]}")
+ IO.puts("Indexing? #{result["isIndexing"]}")
+ end
+end
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 4106feef6..abfe778d2 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -367,7 +367,7 @@ def restrict_deactivated_users(query) do
from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
end
- defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
+ defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index d37454d2c..b709e737b 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -105,6 +105,7 @@ def start(_type, _args) do
{Oban, Config.get(Oban)},
Pleroma.Web.Endpoint
] ++
+ elasticsearch_children() ++
task_children(@mix_env) ++
dont_run_in_test(@mix_env) ++
shout_child(shout_enabled?())
@@ -303,11 +304,25 @@ defp http_children(Tesla.Adapter.Gun, _) do
defp http_children(_, _), do: []
+ def elasticsearch_children do
+ config = Config.get([Pleroma.Search, :module])
+
+ if config == Pleroma.Search.Elasticsearch do
+ [Pleroma.Search.Elasticsearch.Cluster]
+ else
+ []
+ end
+ end
+
@spec limiters_setup() :: :ok
def limiters_setup do
config = Config.get(ConcurrentLimiter, [])
- [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+ [
+ Pleroma.Web.RichMedia.Helpers,
+ Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
+ Pleroma.Search
+ ]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])
diff --git a/lib/pleroma/elasticsearch/document_mappings/activity.ex b/lib/pleroma/elasticsearch/document_mappings/activity.ex
deleted file mode 100644
index a028c6fad..000000000
--- a/lib/pleroma/elasticsearch/document_mappings/activity.ex
+++ /dev/null
@@ -1,19 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Elasticsearch.DocumentMappings.Activity do
- alias Pleroma.Object
-
- def id(obj), do: obj.id
-
- def encode(%{object: %{data: %{"type" => "Note"}}} = activity) do
- %{
- _timestamp: activity.inserted_at,
- user: activity.user_actor.nickname,
- content: activity.object.data["content"],
- instance: URI.parse(activity.user_actor.ap_id).host,
- hashtags: Object.hashtags(activity.object)
- }
- end
-end
diff --git a/lib/pleroma/elasticsearch/document_mappings/hashtag.ex b/lib/pleroma/elasticsearch/document_mappings/hashtag.ex
deleted file mode 100644
index 7391983f6..000000000
--- a/lib/pleroma/elasticsearch/document_mappings/hashtag.ex
+++ /dev/null
@@ -1,21 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Elasticsearch.DocumentMappings.Hashtag do
- def id(obj), do: obj.id
-
- def encode(%{timestamp: _} = hashtag) do
- %{
- hashtag: hashtag.name,
- timestamp: hashtag.timestamp
- }
- end
-
- def encode(hashtag) do
- %{
- hashtag: hashtag.name,
- timestamp: hashtag.inserted_at
- }
- end
-end
diff --git a/lib/pleroma/elasticsearch/document_mappings/user.ex b/lib/pleroma/elasticsearch/document_mappings/user.ex
deleted file mode 100644
index d5cfca656..000000000
--- a/lib/pleroma/elasticsearch/document_mappings/user.ex
+++ /dev/null
@@ -1,17 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Elasticsearch.DocumentMappings.User do
- def id(obj), do: obj.id
-
- def encode(%{actor_type: "Person"} = user) do
- %{
- timestamp: user.inserted_at,
- instance: URI.parse(user.ap_id).host,
- nickname: user.nickname,
- bio: user.bio,
- display_name: user.name
- }
- end
-end
diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex
deleted file mode 100644
index 98c88a7c7..000000000
--- a/lib/pleroma/elasticsearch/store.ex
+++ /dev/null
@@ -1,256 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Elasticsearch do
- alias Pleroma.Activity
- alias Pleroma.User
- alias Pleroma.Object
- alias Pleroma.Elasticsearch.DocumentMappings
- alias Pleroma.Config
- require Logger
-
- defp url do
- Config.get([:elasticsearch, :url])
- end
-
- defp enabled? do
- Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch
- end
-
- def delete_by_id(:activity, id) do
- if enabled?() do
- Elastix.Document.delete(url(), "activities", "activity", id)
- end
- end
-
- def put_by_id(:activity, id) do
- id
- |> Activity.get_by_id_with_object()
- |> maybe_put_into_elasticsearch()
- end
-
- def maybe_put_into_elasticsearch({:ok, item}) do
- maybe_put_into_elasticsearch(item)
- end
-
- def maybe_put_into_elasticsearch(
- %{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity
- ) do
- if enabled?() do
- actor = Pleroma.Activity.user_actor(activity)
-
- activity
- |> Map.put(:user_actor, actor)
- |> put()
- end
- end
-
- def maybe_put_into_elasticsearch(%User{actor_type: "Person"} = user) do
- if enabled?() do
- put(user)
- end
- end
-
- def maybe_put_into_elasticsearch(_) do
- {:ok, :skipped}
- end
-
- def maybe_bulk_post(data, type) do
- if enabled?() do
- bulk_post(data, type)
- end
- end
-
- def put(%Activity{} = activity) do
- with {:ok, _} <-
- Elastix.Document.index(
- url(),
- "activities",
- "activity",
- DocumentMappings.Activity.id(activity),
- DocumentMappings.Activity.encode(activity)
- ) do
- activity
- |> Map.get(:object)
- |> Object.hashtags()
- |> Enum.map(fn x ->
- %{id: x, name: x, timestamp: DateTime.to_iso8601(DateTime.utc_now())}
- end)
- |> bulk_post(:hashtags)
- else
- {:error, %{reason: err}} ->
- Logger.error("Could not put activity: #{err}")
- :skipped
- end
- end
-
- def put(%User{} = user) do
- with {:ok, _} <-
- Elastix.Document.index(
- url(),
- "users",
- "user",
- DocumentMappings.User.id(user),
- DocumentMappings.User.encode(user)
- ) do
- :ok
- else
- {:error, %{reason: err}} ->
- Logger.error("Could not put user: #{err}")
- :skipped
- end
- end
-
- def bulk_post(data, :activities) do
- d =
- data
- |> Enum.filter(fn x ->
- t =
- x.object
- |> Map.get(:data, %{})
- |> Map.get("type", "")
-
- t == "Note"
- end)
- |> Enum.map(fn d ->
- [
- %{index: %{_id: DocumentMappings.Activity.id(d)}},
- DocumentMappings.Activity.encode(d)
- ]
- end)
- |> List.flatten()
-
- with {:ok, %{body: %{"errors" => false}}} <-
- Elastix.Bulk.post(
- url(),
- d,
- index: "activities",
- type: "activity"
- ) do
- :ok
- else
- {:error, %{reason: err}} ->
- Logger.error("Could not bulk put activity: #{err}")
- :skipped
-
- {:ok, %{body: _}} ->
- :skipped
- end
- end
-
- def bulk_post(data, :users) do
- d =
- data
- |> Enum.filter(fn x -> x.actor_type == "Person" end)
- |> Enum.map(fn d ->
- [
- %{index: %{_id: DocumentMappings.User.id(d)}},
- DocumentMappings.User.encode(d)
- ]
- end)
- |> List.flatten()
-
- with {:ok, %{body: %{"errors" => false}}} <-
- Elastix.Bulk.post(
- url(),
- d,
- index: "users",
- type: "user"
- ) do
- :ok
- else
- {:error, %{reason: err}} ->
- Logger.error("Could not bulk put users: #{err}")
- :skipped
-
- {:ok, %{body: _}} ->
- :skipped
- end
- end
-
- def bulk_post(data, :hashtags) when is_list(data) do
- d =
- data
- |> Enum.map(fn d ->
- [
- %{index: %{_id: DocumentMappings.Hashtag.id(d)}},
- DocumentMappings.Hashtag.encode(d)
- ]
- end)
- |> List.flatten()
-
- with {:ok, %{body: %{"errors" => false}}} <-
- Elastix.Bulk.post(
- url(),
- d,
- index: "hashtags",
- type: "hashtag"
- ) do
- :ok
- else
- {:error, %{reason: err}} ->
- Logger.error("Could not bulk put hashtags: #{err}")
- :skipped
-
- {:ok, %{body: _}} ->
- :skipped
- end
- end
-
- def bulk_post(_, :hashtags), do: {:ok, nil}
-
- def search(_, _, _, :skip), do: []
-
- def search(:raw, index, type, q) do
- with {:ok, raw_results} <- Elastix.Search.search(url(), index, [type], q) do
- results =
- raw_results
- |> Map.get(:body, %{})
- |> Map.get("hits", %{})
- |> Map.get("hits", [])
-
- {:ok, results}
- else
- {:error, e} ->
- Logger.error(e)
- {:error, e}
- end
- end
-
- def search(:activities, q) do
- with {:ok, results} <- search(:raw, "activities", "activity", q) do
- results
- |> Enum.map(fn result -> result["_id"] end)
- |> Pleroma.Activity.all_by_ids_with_object()
- |> Enum.sort(&(&1.inserted_at >= &2.inserted_at))
- else
- e ->
- Logger.error(e)
- []
- end
- end
-
- def search(:users, q) do
- with {:ok, results} <- search(:raw, "users", "user", q) do
- results
- |> Enum.map(fn result -> result["_id"] end)
- |> Pleroma.User.get_all_by_ids()
- else
- e ->
- Logger.error(e)
- []
- end
- end
-
- def search(:hashtags, q) do
- with {:ok, results} <- search(:raw, "hashtags", "hashtag", q) do
- results
- |> Enum.map(fn result -> result["_source"]["hashtag"] end)
- else
- e ->
- Logger.error(e)
- []
- end
- end
-end
diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index e38c681ba..24adfabd7 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -5,9 +5,12 @@
defmodule Pleroma.Emails.UserEmail do
@moduledoc "User emails"
+ require Pleroma.Web.Gettext
+
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.Web.Endpoint
+ alias Pleroma.Web.Gettext
alias Pleroma.Web.Router
import Swoosh.Email
@@ -27,29 +30,75 @@ defp recipient(%User{} = user), do: recipient(user.email, user.name)
@spec welcome(User.t(), map()) :: Swoosh.Email.t()
def welcome(user, opts \\ %{}) do
- new()
- |> to(recipient(user))
- |> from(Map.get(opts, :sender, sender()))
- |> subject(Map.get(opts, :subject, "Welcome to #{instance_name()}!"))
- |> html_body(Map.get(opts, :html, "Welcome to #{instance_name()}!"))
- |> text_body(Map.get(opts, :text, "Welcome to #{instance_name()}!"))
+ Gettext.with_locale_or_default user.language do
+ new()
+ |> to(recipient(user))
+ |> from(Map.get(opts, :sender, sender()))
+ |> subject(
+ Map.get(
+ opts,
+ :subject,
+ Gettext.dpgettext(
+ "static_pages",
+ "welcome email subject",
+ "Welcome to %{instance_name}!",
+ instance_name: instance_name()
+ )
+ )
+ )
+ |> html_body(
+ Map.get(
+ opts,
+ :html,
+ Gettext.dpgettext(
+ "static_pages",
+ "welcome email html body",
+ "Welcome to %{instance_name}!",
+ instance_name: instance_name()
+ )
+ )
+ )
+ |> text_body(
+ Map.get(
+ opts,
+ :text,
+ Gettext.dpgettext(
+ "static_pages",
+ "welcome email text body",
+ "Welcome to %{instance_name}!",
+ instance_name: instance_name()
+ )
+ )
+ )
+ end
end
def password_reset_email(user, token) when is_binary(token) do
- password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token)
+ Gettext.with_locale_or_default user.language do
+ password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token)
- html_body = """
- Reset your password at #{instance_name()}
- Someone has requested password change for your account at #{instance_name()}.
- If it was you, visit the following link to proceed: reset password .
- If it was someone else, nothing to worry about: your data is secure and your password has not been changed.
- """
+ html_body =
+ Gettext.dpgettext(
+ "static_pages",
+ "password reset email body",
+ """
+ Reset your password at %{instance_name}
+ Someone has requested password change for your account at %{instance_name}.
+ If it was you, visit the following link to proceed: reset password .
+ If it was someone else, nothing to worry about: your data is secure and your password has not been changed.
+ """,
+ instance_name: instance_name(),
+ password_reset_url: password_reset_url
+ )
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("Password reset")
- |> html_body(html_body)
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext("static_pages", "password reset email subject", "Password reset")
+ )
+ |> html_body(html_body)
+ end
end
def user_invitation_email(
@@ -58,73 +107,136 @@ def user_invitation_email(
to_email,
to_name \\ nil
) do
- registration_url =
- Router.Helpers.redirect_url(
- Endpoint,
- :registration_page,
- user_invite_token.token
+ Gettext.with_locale_or_default user.language do
+ registration_url =
+ Router.Helpers.redirect_url(
+ Endpoint,
+ :registration_page,
+ user_invite_token.token
+ )
+
+ html_body =
+ Gettext.dpgettext(
+ "static_pages",
+ "user invitation email body",
+ """
+ You are invited to %{instance_name}
+ %{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.
+ Click the following link to register: accept invitation .
+ """,
+ instance_name: instance_name(),
+ inviter_name: user.name,
+ registration_url: registration_url
+ )
+
+ new()
+ |> to(recipient(to_email, to_name))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "user invitation email subject",
+ "Invitation to %{instance_name}",
+ instance_name: instance_name()
+ )
)
-
- html_body = """
- You are invited to #{instance_name()}
- #{user.name} invites you to join #{instance_name()}, an instance of Pleroma federated social networking platform.
- Click the following link to register: accept invitation .
- """
-
- new()
- |> to(recipient(to_email, to_name))
- |> from(sender())
- |> subject("Invitation to #{instance_name()}")
- |> html_body(html_body)
+ |> html_body(html_body)
+ end
end
def account_confirmation_email(user) do
- confirmation_url =
- Router.Helpers.confirm_email_url(
- Endpoint,
- :confirm_email,
- user.id,
- to_string(user.confirmation_token)
+ Gettext.with_locale_or_default user.language do
+ confirmation_url =
+ Router.Helpers.confirm_email_url(
+ Endpoint,
+ :confirm_email,
+ user.id,
+ to_string(user.confirmation_token)
+ )
+
+ html_body =
+ Gettext.dpgettext(
+ "static_pages",
+ "confirmation email body",
+ """
+ Thank you for registering on %{instance_name}
+ Email confirmation is required to activate the account.
+ Please click the following link to activate your account .
+ """,
+ instance_name: instance_name(),
+ confirmation_url: confirmation_url
+ )
+
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "confirmation email subject",
+ "%{instance_name} account confirmation",
+ instance_name: instance_name()
+ )
)
-
- html_body = """
- Thank you for registering on #{instance_name()}
- Email confirmation is required to activate the account.
- Please click the following link to activate your account .
- """
-
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("#{instance_name()} account confirmation")
- |> html_body(html_body)
+ |> html_body(html_body)
+ end
end
def approval_pending_email(user) do
- html_body = """
- Awaiting Approval
- Your account at #{instance_name()} is being reviewed by staff. You will receive another email once your account is approved.
- """
+ Gettext.with_locale_or_default user.language do
+ html_body =
+ Gettext.dpgettext(
+ "static_pages",
+ "approval pending email body",
+ """
+ Awaiting Approval
+ Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.
+ """,
+ instance_name: instance_name()
+ )
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("Your account is awaiting approval")
- |> html_body(html_body)
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "approval pending email subject",
+ "Your account is awaiting approval"
+ )
+ )
+ |> html_body(html_body)
+ end
end
def successful_registration_email(user) do
- html_body = """
- Hello @#{user.nickname},
- Your account at #{instance_name()} has been registered successfully.
- No further action is required to activate your account.
- """
+ Gettext.with_locale_or_default user.language do
+ html_body =
+ Gettext.dpgettext(
+ "static_pages",
+ "successful registration email body",
+ """
+ Hello @%{nickname},
+ Your account at %{instance_name} has been registered successfully.
+ No further action is required to activate your account.
+ """,
+ nickname: user.nickname,
+ instance_name: instance_name()
+ )
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("Account registered on #{instance_name()}")
- |> html_body(html_body)
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "successful registration email subject",
+ "Account registered on %{instance_name}",
+ instance_name: instance_name()
+ )
+ )
+ |> html_body(html_body)
+ end
end
@doc """
@@ -134,69 +246,78 @@ def successful_registration_email(user) do
"""
@spec digest_email(User.t()) :: Swoosh.Email.t() | nil
def digest_email(user) do
- notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
+ Gettext.with_locale_or_default user.language do
+ notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
- mentions =
- notifications
- |> Enum.filter(&(&1.activity.data["type"] == "Create"))
- |> Enum.map(fn notification ->
- object = Pleroma.Object.normalize(notification.activity, fetch: false)
+ mentions =
+ notifications
+ |> Enum.filter(&(&1.activity.data["type"] == "Create"))
+ |> Enum.map(fn notification ->
+ object = Pleroma.Object.normalize(notification.activity, fetch: false)
- if not is_nil(object) do
- object = update_in(object.data["content"], &format_links/1)
+ if not is_nil(object) do
+ object = update_in(object.data["content"], &format_links/1)
- %{
- data: notification,
- object: object,
- from: User.get_by_ap_id(notification.activity.actor)
- }
- end
- end)
- |> Enum.filter(& &1)
+ %{
+ data: notification,
+ object: object,
+ from: User.get_by_ap_id(notification.activity.actor)
+ }
+ end
+ end)
+ |> Enum.filter(& &1)
- followers =
- notifications
- |> Enum.filter(&(&1.activity.data["type"] == "Follow"))
- |> Enum.map(fn notification ->
- from = User.get_by_ap_id(notification.activity.actor)
+ followers =
+ notifications
+ |> Enum.filter(&(&1.activity.data["type"] == "Follow"))
+ |> Enum.map(fn notification ->
+ from = User.get_by_ap_id(notification.activity.actor)
- if not is_nil(from) do
- %{
- data: notification,
- object: Pleroma.Object.normalize(notification.activity, fetch: false),
- from: User.get_by_ap_id(notification.activity.actor)
- }
- end
- end)
- |> Enum.filter(& &1)
+ if not is_nil(from) do
+ %{
+ data: notification,
+ object: Pleroma.Object.normalize(notification.activity, fetch: false),
+ from: User.get_by_ap_id(notification.activity.actor)
+ }
+ end
+ end)
+ |> Enum.filter(& &1)
- unless Enum.empty?(mentions) do
- styling = Config.get([__MODULE__, :styling])
- logo = Config.get([__MODULE__, :logo])
+ unless Enum.empty?(mentions) do
+ styling = Config.get([__MODULE__, :styling])
+ logo = Config.get([__MODULE__, :logo])
- html_data = %{
- instance: instance_name(),
- user: user,
- mentions: mentions,
- followers: followers,
- unsubscribe_link: unsubscribe_url(user, "digest"),
- styling: styling
- }
+ html_data = %{
+ instance: instance_name(),
+ user: user,
+ mentions: mentions,
+ followers: followers,
+ unsubscribe_link: unsubscribe_url(user, "digest"),
+ styling: styling
+ }
- logo_path =
- if is_nil(logo) do
- Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg")
- else
- Path.join(Config.get([:instance, :static_dir]), logo)
- end
+ logo_path =
+ if is_nil(logo) do
+ Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg")
+ else
+ Path.join(Config.get([:instance, :static_dir]), logo)
+ end
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("Your digest from #{instance_name()}")
- |> put_layout(false)
- |> render_body("digest.html", html_data)
- |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline))
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "digest email subject",
+ "Your digest from %{instance_name}",
+ instance_name: instance_name()
+ )
+ )
+ |> put_layout(false)
+ |> render_body("digest.html", html_data)
+ |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline))
+ end
end
end
@@ -226,27 +347,47 @@ def unsubscribe_url(user, notifications_type) do
def backup_is_ready_email(backup, admin_user_id \\ nil) do
%{user: user} = Pleroma.Repo.preload(backup, :user)
- download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
- html_body =
- if is_nil(admin_user_id) do
- """
- You requested a full backup of your Pleroma account. It's ready for download:
- #{download_url}
- """
- else
- admin = Pleroma.Repo.get(User, admin_user_id)
+ Gettext.with_locale_or_default user.language do
+ download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
- """
- Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:
- #{download_url}
- """
- end
+ html_body =
+ if is_nil(admin_user_id) do
+ Gettext.dpgettext(
+ "static_pages",
+ "account archive email body - self-requested",
+ """
+ You requested a full backup of your Pleroma account. It's ready for download:
+ %{download_url}
+ """,
+ download_url: download_url
+ )
+ else
+ admin = Pleroma.Repo.get(User, admin_user_id)
- new()
- |> to(recipient(user))
- |> from(sender())
- |> subject("Your account archive is ready")
- |> html_body(html_body)
+ Gettext.dpgettext(
+ "static_pages",
+ "account archive email body - admin requested",
+ """
+ Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:
+ %{download_url}
+ """,
+ admin_nickname: admin.nickname,
+ download_url: download_url
+ )
+ end
+
+ new()
+ |> to(recipient(user))
+ |> from(sender())
+ |> subject(
+ Gettext.dpgettext(
+ "static_pages",
+ "account archive email subject",
+ "Your account archive is ready"
+ )
+ )
+ |> html_body(html_body)
+ end
end
end
diff --git a/lib/pleroma/emoji-test.txt b/lib/pleroma/emoji-test.txt
index d3c6d12bd..dd5493366 100644
--- a/lib/pleroma/emoji-test.txt
+++ b/lib/pleroma/emoji-test.txt
@@ -1,11 +1,11 @@
# emoji-test.txt
-# Date: 2020-09-12, 22:19:50 GMT
-# © 2020 Unicode®, Inc.
+# Date: 2021-08-26, 17:22:23 GMT
+# © 2021 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Emoji Keyboard/Display Test Data for UTS #51
-# Version: 13.1
+# Version: 14.0
#
# For documentation and usage, see http://www.unicode.org/reports/tr51
#
@@ -43,6 +43,7 @@
1F602 ; fully-qualified # 😂 E0.6 face with tears of joy
1F642 ; fully-qualified # 🙂 E1.0 slightly smiling face
1F643 ; fully-qualified # 🙃 E1.0 upside-down face
+1FAE0 ; fully-qualified # 🫠 E14.0 melting face
1F609 ; fully-qualified # 😉 E0.6 winking face
1F60A ; fully-qualified # 😊 E0.6 smiling face with smiling eyes
1F607 ; fully-qualified # 😇 E1.0 smiling face with halo
@@ -68,10 +69,13 @@
1F911 ; fully-qualified # 🤑 E1.0 money-mouth face
# subgroup: face-hand
-1F917 ; fully-qualified # 🤗 E1.0 hugging face
+1F917 ; fully-qualified # 🤗 E1.0 smiling face with open hands
1F92D ; fully-qualified # 🤭 E5.0 face with hand over mouth
+1FAE2 ; fully-qualified # 🫢 E14.0 face with open eyes and hand over mouth
+1FAE3 ; fully-qualified # 🫣 E14.0 face with peeking eye
1F92B ; fully-qualified # 🤫 E5.0 shushing face
1F914 ; fully-qualified # 🤔 E1.0 thinking face
+1FAE1 ; fully-qualified # 🫡 E14.0 saluting face
# subgroup: face-neutral-skeptical
1F910 ; fully-qualified # 🤐 E1.0 zipper-mouth face
@@ -79,6 +83,7 @@
1F610 ; fully-qualified # 😐 E0.7 neutral face
1F611 ; fully-qualified # 😑 E1.0 expressionless face
1F636 ; fully-qualified # 😶 E1.0 face without mouth
+1FAE5 ; fully-qualified # 🫥 E14.0 dotted line face
1F636 200D 1F32B FE0F ; fully-qualified # 😶🌫️ E13.1 face in clouds
1F636 200D 1F32B ; minimally-qualified # 😶🌫 E13.1 face in clouds
1F60F ; fully-qualified # 😏 E0.6 smirking face
@@ -105,7 +110,7 @@
1F975 ; fully-qualified # 🥵 E11.0 hot face
1F976 ; fully-qualified # 🥶 E11.0 cold face
1F974 ; fully-qualified # 🥴 E11.0 woozy face
-1F635 ; fully-qualified # 😵 E0.6 knocked-out face
+1F635 ; fully-qualified # 😵 E0.6 face with crossed-out eyes
1F635 200D 1F4AB ; fully-qualified # 😵💫 E13.1 face with spiral eyes
1F92F ; fully-qualified # 🤯 E5.0 exploding head
@@ -121,6 +126,7 @@
# subgroup: face-concerned
1F615 ; fully-qualified # 😕 E1.0 confused face
+1FAE4 ; fully-qualified # 🫤 E14.0 face with diagonal mouth
1F61F ; fully-qualified # 😟 E1.0 worried face
1F641 ; fully-qualified # 🙁 E1.0 slightly frowning face
2639 FE0F ; fully-qualified # ☹️ E0.7 frowning face
@@ -130,6 +136,7 @@
1F632 ; fully-qualified # 😲 E0.6 astonished face
1F633 ; fully-qualified # 😳 E0.6 flushed face
1F97A ; fully-qualified # 🥺 E11.0 pleading face
+1F979 ; fully-qualified # 🥹 E14.0 face holding back tears
1F626 ; fully-qualified # 😦 E1.0 frowning face with open mouth
1F627 ; fully-qualified # 😧 E1.0 anguished face
1F628 ; fully-qualified # 😨 E0.6 fearful face
@@ -232,8 +239,8 @@
1F4AD ; fully-qualified # 💭 E1.0 thought balloon
1F4A4 ; fully-qualified # 💤 E0.6 zzz
-# Smileys & Emotion subtotal: 170
-# Smileys & Emotion subtotal: 170 w/o modifiers
+# Smileys & Emotion subtotal: 177
+# Smileys & Emotion subtotal: 177 w/o modifiers
# group: People & Body
@@ -269,6 +276,30 @@
1F596 1F3FD ; fully-qualified # 🖖🏽 E1.0 vulcan salute: medium skin tone
1F596 1F3FE ; fully-qualified # 🖖🏾 E1.0 vulcan salute: medium-dark skin tone
1F596 1F3FF ; fully-qualified # 🖖🏿 E1.0 vulcan salute: dark skin tone
+1FAF1 ; fully-qualified # 🫱 E14.0 rightwards hand
+1FAF1 1F3FB ; fully-qualified # 🫱🏻 E14.0 rightwards hand: light skin tone
+1FAF1 1F3FC ; fully-qualified # 🫱🏼 E14.0 rightwards hand: medium-light skin tone
+1FAF1 1F3FD ; fully-qualified # 🫱🏽 E14.0 rightwards hand: medium skin tone
+1FAF1 1F3FE ; fully-qualified # 🫱🏾 E14.0 rightwards hand: medium-dark skin tone
+1FAF1 1F3FF ; fully-qualified # 🫱🏿 E14.0 rightwards hand: dark skin tone
+1FAF2 ; fully-qualified # 🫲 E14.0 leftwards hand
+1FAF2 1F3FB ; fully-qualified # 🫲🏻 E14.0 leftwards hand: light skin tone
+1FAF2 1F3FC ; fully-qualified # 🫲🏼 E14.0 leftwards hand: medium-light skin tone
+1FAF2 1F3FD ; fully-qualified # 🫲🏽 E14.0 leftwards hand: medium skin tone
+1FAF2 1F3FE ; fully-qualified # 🫲🏾 E14.0 leftwards hand: medium-dark skin tone
+1FAF2 1F3FF ; fully-qualified # 🫲🏿 E14.0 leftwards hand: dark skin tone
+1FAF3 ; fully-qualified # 🫳 E14.0 palm down hand
+1FAF3 1F3FB ; fully-qualified # 🫳🏻 E14.0 palm down hand: light skin tone
+1FAF3 1F3FC ; fully-qualified # 🫳🏼 E14.0 palm down hand: medium-light skin tone
+1FAF3 1F3FD ; fully-qualified # 🫳🏽 E14.0 palm down hand: medium skin tone
+1FAF3 1F3FE ; fully-qualified # 🫳🏾 E14.0 palm down hand: medium-dark skin tone
+1FAF3 1F3FF ; fully-qualified # 🫳🏿 E14.0 palm down hand: dark skin tone
+1FAF4 ; fully-qualified # 🫴 E14.0 palm up hand
+1FAF4 1F3FB ; fully-qualified # 🫴🏻 E14.0 palm up hand: light skin tone
+1FAF4 1F3FC ; fully-qualified # 🫴🏼 E14.0 palm up hand: medium-light skin tone
+1FAF4 1F3FD ; fully-qualified # 🫴🏽 E14.0 palm up hand: medium skin tone
+1FAF4 1F3FE ; fully-qualified # 🫴🏾 E14.0 palm up hand: medium-dark skin tone
+1FAF4 1F3FF ; fully-qualified # 🫴🏿 E14.0 palm up hand: dark skin tone
# subgroup: hand-fingers-partial
1F44C ; fully-qualified # 👌 E0.6 OK hand
@@ -302,6 +333,12 @@
1F91E 1F3FD ; fully-qualified # 🤞🏽 E3.0 crossed fingers: medium skin tone
1F91E 1F3FE ; fully-qualified # 🤞🏾 E3.0 crossed fingers: medium-dark skin tone
1F91E 1F3FF ; fully-qualified # 🤞🏿 E3.0 crossed fingers: dark skin tone
+1FAF0 ; fully-qualified # 🫰 E14.0 hand with index finger and thumb crossed
+1FAF0 1F3FB ; fully-qualified # 🫰🏻 E14.0 hand with index finger and thumb crossed: light skin tone
+1FAF0 1F3FC ; fully-qualified # 🫰🏼 E14.0 hand with index finger and thumb crossed: medium-light skin tone
+1FAF0 1F3FD ; fully-qualified # 🫰🏽 E14.0 hand with index finger and thumb crossed: medium skin tone
+1FAF0 1F3FE ; fully-qualified # 🫰🏾 E14.0 hand with index finger and thumb crossed: medium-dark skin tone
+1FAF0 1F3FF ; fully-qualified # 🫰🏿 E14.0 hand with index finger and thumb crossed: dark skin tone
1F91F ; fully-qualified # 🤟 E5.0 love-you gesture
1F91F 1F3FB ; fully-qualified # 🤟🏻 E5.0 love-you gesture: light skin tone
1F91F 1F3FC ; fully-qualified # 🤟🏼 E5.0 love-you gesture: medium-light skin tone
@@ -359,6 +396,12 @@
261D 1F3FD ; fully-qualified # ☝🏽 E1.0 index pointing up: medium skin tone
261D 1F3FE ; fully-qualified # ☝🏾 E1.0 index pointing up: medium-dark skin tone
261D 1F3FF ; fully-qualified # ☝🏿 E1.0 index pointing up: dark skin tone
+1FAF5 ; fully-qualified # 🫵 E14.0 index pointing at the viewer
+1FAF5 1F3FB ; fully-qualified # 🫵🏻 E14.0 index pointing at the viewer: light skin tone
+1FAF5 1F3FC ; fully-qualified # 🫵🏼 E14.0 index pointing at the viewer: medium-light skin tone
+1FAF5 1F3FD ; fully-qualified # 🫵🏽 E14.0 index pointing at the viewer: medium skin tone
+1FAF5 1F3FE ; fully-qualified # 🫵🏾 E14.0 index pointing at the viewer: medium-dark skin tone
+1FAF5 1F3FF ; fully-qualified # 🫵🏿 E14.0 index pointing at the viewer: dark skin tone
# subgroup: hand-fingers-closed
1F44D ; fully-qualified # 👍 E0.6 thumbs up
@@ -411,6 +454,12 @@
1F64C 1F3FD ; fully-qualified # 🙌🏽 E1.0 raising hands: medium skin tone
1F64C 1F3FE ; fully-qualified # 🙌🏾 E1.0 raising hands: medium-dark skin tone
1F64C 1F3FF ; fully-qualified # 🙌🏿 E1.0 raising hands: dark skin tone
+1FAF6 ; fully-qualified # 🫶 E14.0 heart hands
+1FAF6 1F3FB ; fully-qualified # 🫶🏻 E14.0 heart hands: light skin tone
+1FAF6 1F3FC ; fully-qualified # 🫶🏼 E14.0 heart hands: medium-light skin tone
+1FAF6 1F3FD ; fully-qualified # 🫶🏽 E14.0 heart hands: medium skin tone
+1FAF6 1F3FE ; fully-qualified # 🫶🏾 E14.0 heart hands: medium-dark skin tone
+1FAF6 1F3FF ; fully-qualified # 🫶🏿 E14.0 heart hands: dark skin tone
1F450 ; fully-qualified # 👐 E0.6 open hands
1F450 1F3FB ; fully-qualified # 👐🏻 E1.0 open hands: light skin tone
1F450 1F3FC ; fully-qualified # 👐🏼 E1.0 open hands: medium-light skin tone
@@ -424,6 +473,31 @@
1F932 1F3FE ; fully-qualified # 🤲🏾 E5.0 palms up together: medium-dark skin tone
1F932 1F3FF ; fully-qualified # 🤲🏿 E5.0 palms up together: dark skin tone
1F91D ; fully-qualified # 🤝 E3.0 handshake
+1F91D 1F3FB ; fully-qualified # 🤝🏻 E3.0 handshake: light skin tone
+1F91D 1F3FC ; fully-qualified # 🤝🏼 E3.0 handshake: medium-light skin tone
+1F91D 1F3FD ; fully-qualified # 🤝🏽 E3.0 handshake: medium skin tone
+1F91D 1F3FE ; fully-qualified # 🤝🏾 E3.0 handshake: medium-dark skin tone
+1F91D 1F3FF ; fully-qualified # 🤝🏿 E3.0 handshake: dark skin tone
+1FAF1 1F3FB 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏻🫲🏼 E14.0 handshake: light skin tone, medium-light skin tone
+1FAF1 1F3FB 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏻🫲🏽 E14.0 handshake: light skin tone, medium skin tone
+1FAF1 1F3FB 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏻🫲🏾 E14.0 handshake: light skin tone, medium-dark skin tone
+1FAF1 1F3FB 200D 1FAF2 1F3FF ; fully-qualified # 🫱🏻🫲🏿 E14.0 handshake: light skin tone, dark skin tone
+1FAF1 1F3FC 200D 1FAF2 1F3FB ; fully-qualified # 🫱🏼🫲🏻 E14.0 handshake: medium-light skin tone, light skin tone
+1FAF1 1F3FC 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏼🫲🏽 E14.0 handshake: medium-light skin tone, medium skin tone
+1FAF1 1F3FC 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏼🫲🏾 E14.0 handshake: medium-light skin tone, medium-dark skin tone
+1FAF1 1F3FC 200D 1FAF2 1F3FF ; fully-qualified # 🫱🏼🫲🏿 E14.0 handshake: medium-light skin tone, dark skin tone
+1FAF1 1F3FD 200D 1FAF2 1F3FB ; fully-qualified # 🫱🏽🫲🏻 E14.0 handshake: medium skin tone, light skin tone
+1FAF1 1F3FD 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏽🫲🏼 E14.0 handshake: medium skin tone, medium-light skin tone
+1FAF1 1F3FD 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏽🫲🏾 E14.0 handshake: medium skin tone, medium-dark skin tone
+1FAF1 1F3FD 200D 1FAF2 1F3FF ; fully-qualified # 🫱🏽🫲🏿 E14.0 handshake: medium skin tone, dark skin tone
+1FAF1 1F3FE 200D 1FAF2 1F3FB ; fully-qualified # 🫱🏾🫲🏻 E14.0 handshake: medium-dark skin tone, light skin tone
+1FAF1 1F3FE 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏾🫲🏼 E14.0 handshake: medium-dark skin tone, medium-light skin tone
+1FAF1 1F3FE 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏾🫲🏽 E14.0 handshake: medium-dark skin tone, medium skin tone
+1FAF1 1F3FE 200D 1FAF2 1F3FF ; fully-qualified # 🫱🏾🫲🏿 E14.0 handshake: medium-dark skin tone, dark skin tone
+1FAF1 1F3FF 200D 1FAF2 1F3FB ; fully-qualified # 🫱🏿🫲🏻 E14.0 handshake: dark skin tone, light skin tone
+1FAF1 1F3FF 200D 1FAF2 1F3FC ; fully-qualified # 🫱🏿🫲🏼 E14.0 handshake: dark skin tone, medium-light skin tone
+1FAF1 1F3FF 200D 1FAF2 1F3FD ; fully-qualified # 🫱🏿🫲🏽 E14.0 handshake: dark skin tone, medium skin tone
+1FAF1 1F3FF 200D 1FAF2 1F3FE ; fully-qualified # 🫱🏿🫲🏾 E14.0 handshake: dark skin tone, medium-dark skin tone
1F64F ; fully-qualified # 🙏 E0.6 folded hands
1F64F 1F3FB ; fully-qualified # 🙏🏻 E1.0 folded hands: light skin tone
1F64F 1F3FC ; fully-qualified # 🙏🏼 E1.0 folded hands: medium-light skin tone
@@ -501,6 +575,7 @@
1F441 ; unqualified # 👁 E0.7 eye
1F445 ; fully-qualified # 👅 E0.6 tongue
1F444 ; fully-qualified # 👄 E0.6 mouth
+1FAE6 ; fully-qualified # 🫦 E14.0 biting lip
# subgroup: person
1F476 ; fully-qualified # 👶 E0.6 baby
@@ -1472,6 +1547,12 @@
1F477 1F3FE 200D 2640 ; minimally-qualified # 👷🏾♀ E4.0 woman construction worker: medium-dark skin tone
1F477 1F3FF 200D 2640 FE0F ; fully-qualified # 👷🏿♀️ E4.0 woman construction worker: dark skin tone
1F477 1F3FF 200D 2640 ; minimally-qualified # 👷🏿♀ E4.0 woman construction worker: dark skin tone
+1FAC5 ; fully-qualified # 🫅 E14.0 person with crown
+1FAC5 1F3FB ; fully-qualified # 🫅🏻 E14.0 person with crown: light skin tone
+1FAC5 1F3FC ; fully-qualified # 🫅🏼 E14.0 person with crown: medium-light skin tone
+1FAC5 1F3FD ; fully-qualified # 🫅🏽 E14.0 person with crown: medium skin tone
+1FAC5 1F3FE ; fully-qualified # 🫅🏾 E14.0 person with crown: medium-dark skin tone
+1FAC5 1F3FF ; fully-qualified # 🫅🏿 E14.0 person with crown: dark skin tone
1F934 ; fully-qualified # 🤴 E3.0 prince
1F934 1F3FB ; fully-qualified # 🤴🏻 E3.0 prince: light skin tone
1F934 1F3FC ; fully-qualified # 🤴🏼 E3.0 prince: medium-light skin tone
@@ -1592,6 +1673,18 @@
1F930 1F3FD ; fully-qualified # 🤰🏽 E3.0 pregnant woman: medium skin tone
1F930 1F3FE ; fully-qualified # 🤰🏾 E3.0 pregnant woman: medium-dark skin tone
1F930 1F3FF ; fully-qualified # 🤰🏿 E3.0 pregnant woman: dark skin tone
+1FAC3 ; fully-qualified # 🫃 E14.0 pregnant man
+1FAC3 1F3FB ; fully-qualified # 🫃🏻 E14.0 pregnant man: light skin tone
+1FAC3 1F3FC ; fully-qualified # 🫃🏼 E14.0 pregnant man: medium-light skin tone
+1FAC3 1F3FD ; fully-qualified # 🫃🏽 E14.0 pregnant man: medium skin tone
+1FAC3 1F3FE ; fully-qualified # 🫃🏾 E14.0 pregnant man: medium-dark skin tone
+1FAC3 1F3FF ; fully-qualified # 🫃🏿 E14.0 pregnant man: dark skin tone
+1FAC4 ; fully-qualified # 🫄 E14.0 pregnant person
+1FAC4 1F3FB ; fully-qualified # 🫄🏻 E14.0 pregnant person: light skin tone
+1FAC4 1F3FC ; fully-qualified # 🫄🏼 E14.0 pregnant person: medium-light skin tone
+1FAC4 1F3FD ; fully-qualified # 🫄🏽 E14.0 pregnant person: medium skin tone
+1FAC4 1F3FE ; fully-qualified # 🫄🏾 E14.0 pregnant person: medium-dark skin tone
+1FAC4 1F3FF ; fully-qualified # 🫄🏿 E14.0 pregnant person: dark skin tone
1F931 ; fully-qualified # 🤱 E5.0 breast-feeding
1F931 1F3FB ; fully-qualified # 🤱🏻 E5.0 breast-feeding: light skin tone
1F931 1F3FC ; fully-qualified # 🤱🏼 E5.0 breast-feeding: medium-light skin tone
@@ -1862,6 +1955,7 @@
1F9DF 200D 2642 ; minimally-qualified # 🧟♂ E5.0 man zombie
1F9DF 200D 2640 FE0F ; fully-qualified # 🧟♀️ E5.0 woman zombie
1F9DF 200D 2640 ; minimally-qualified # 🧟♀ E5.0 woman zombie
+1F9CC ; fully-qualified # 🧌 E14.0 troll
# subgroup: person-activity
1F486 ; fully-qualified # 💆 E0.6 person getting massage
@@ -3168,8 +3262,8 @@
1FAC2 ; fully-qualified # 🫂 E13.0 people hugging
1F463 ; fully-qualified # 👣 E0.6 footprints
-# People & Body subtotal: 2899
-# People & Body subtotal: 494 w/o modifiers
+# People & Body subtotal: 2986
+# People & Body subtotal: 506 w/o modifiers
# group: Component
@@ -3304,6 +3398,7 @@
1F988 ; fully-qualified # 🦈 E3.0 shark
1F419 ; fully-qualified # 🐙 E0.6 octopus
1F41A ; fully-qualified # 🐚 E0.6 spiral shell
+1FAB8 ; fully-qualified # 🪸 E14.0 coral
# subgroup: animal-bug
1F40C ; fully-qualified # 🐌 E0.6 snail
@@ -3329,6 +3424,7 @@
1F490 ; fully-qualified # 💐 E0.6 bouquet
1F338 ; fully-qualified # 🌸 E0.6 cherry blossom
1F4AE ; fully-qualified # 💮 E0.6 white flower
+1FAB7 ; fully-qualified # 🪷 E14.0 lotus
1F3F5 FE0F ; fully-qualified # 🏵️ E0.7 rosette
1F3F5 ; unqualified # 🏵 E0.7 rosette
1F339 ; fully-qualified # 🌹 E0.6 rose
@@ -3353,9 +3449,11 @@
1F341 ; fully-qualified # 🍁 E0.6 maple leaf
1F342 ; fully-qualified # 🍂 E0.6 fallen leaf
1F343 ; fully-qualified # 🍃 E0.6 leaf fluttering in wind
+1FAB9 ; fully-qualified # 🪹 E14.0 empty nest
+1FABA ; fully-qualified # 🪺 E14.0 nest with eggs
-# Animals & Nature subtotal: 147
-# Animals & Nature subtotal: 147 w/o modifiers
+# Animals & Nature subtotal: 151
+# Animals & Nature subtotal: 151 w/o modifiers
# group: Food & Drink
@@ -3396,6 +3494,7 @@
1F9C5 ; fully-qualified # 🧅 E12.0 onion
1F344 ; fully-qualified # 🍄 E0.6 mushroom
1F95C ; fully-qualified # 🥜 E3.0 peanuts
+1FAD8 ; fully-qualified # 🫘 E14.0 beans
1F330 ; fully-qualified # 🌰 E0.6 chestnut
# subgroup: food-prepared
@@ -3491,6 +3590,7 @@
1F37B ; fully-qualified # 🍻 E0.6 clinking beer mugs
1F942 ; fully-qualified # 🥂 E3.0 clinking glasses
1F943 ; fully-qualified # 🥃 E3.0 tumbler glass
+1FAD7 ; fully-qualified # 🫗 E14.0 pouring liquid
1F964 ; fully-qualified # 🥤 E5.0 cup with straw
1F9CB ; fully-qualified # 🧋 E13.0 bubble tea
1F9C3 ; fully-qualified # 🧃 E12.0 beverage box
@@ -3504,10 +3604,11 @@
1F374 ; fully-qualified # 🍴 E0.6 fork and knife
1F944 ; fully-qualified # 🥄 E3.0 spoon
1F52A ; fully-qualified # 🔪 E0.6 kitchen knife
+1FAD9 ; fully-qualified # 🫙 E14.0 jar
1F3FA ; fully-qualified # 🏺 E1.0 amphora
-# Food & Drink subtotal: 131
-# Food & Drink subtotal: 131 w/o modifiers
+# Food & Drink subtotal: 134
+# Food & Drink subtotal: 134 w/o modifiers
# group: Travel & Places
@@ -3597,6 +3698,7 @@
2668 FE0F ; fully-qualified # ♨️ E0.6 hot springs
2668 ; unqualified # ♨ E0.6 hot springs
1F3A0 ; fully-qualified # 🎠 E0.6 carousel horse
+1F6DD ; fully-qualified # 🛝 E14.0 playground slide
1F3A1 ; fully-qualified # 🎡 E0.6 ferris wheel
1F3A2 ; fully-qualified # 🎢 E0.6 roller coaster
1F488 ; fully-qualified # 💈 E0.6 barber pole
@@ -3652,6 +3754,7 @@
1F6E2 FE0F ; fully-qualified # 🛢️ E0.7 oil drum
1F6E2 ; unqualified # 🛢 E0.7 oil drum
26FD ; fully-qualified # ⛽ E0.6 fuel pump
+1F6DE ; fully-qualified # 🛞 E14.0 wheel
1F6A8 ; fully-qualified # 🚨 E0.6 police car light
1F6A5 ; fully-qualified # 🚥 E0.6 horizontal traffic light
1F6A6 ; fully-qualified # 🚦 E1.0 vertical traffic light
@@ -3660,6 +3763,7 @@
# subgroup: transport-water
2693 ; fully-qualified # ⚓ E0.6 anchor
+1F6DF ; fully-qualified # 🛟 E14.0 ring buoy
26F5 ; fully-qualified # ⛵ E0.6 sailboat
1F6F6 ; fully-qualified # 🛶 E3.0 canoe
1F6A4 ; fully-qualified # 🚤 E0.6 speedboat
@@ -3797,8 +3901,8 @@
1F4A7 ; fully-qualified # 💧 E0.6 droplet
1F30A ; fully-qualified # 🌊 E0.6 water wave
-# Travel & Places subtotal: 264
-# Travel & Places subtotal: 264 w/o modifiers
+# Travel & Places subtotal: 267
+# Travel & Places subtotal: 267 w/o modifiers
# group: Activities
@@ -3874,6 +3978,7 @@
1F52E ; fully-qualified # 🔮 E0.6 crystal ball
1FA84 ; fully-qualified # 🪄 E13.0 magic wand
1F9FF ; fully-qualified # 🧿 E11.0 nazar amulet
+1FAAC ; fully-qualified # 🪬 E14.0 hamsa
1F3AE ; fully-qualified # 🎮 E0.6 video game
1F579 FE0F ; fully-qualified # 🕹️ E0.7 joystick
1F579 ; unqualified # 🕹 E0.7 joystick
@@ -3882,6 +3987,7 @@
1F9E9 ; fully-qualified # 🧩 E11.0 puzzle piece
1F9F8 ; fully-qualified # 🧸 E11.0 teddy bear
1FA85 ; fully-qualified # 🪅 E13.0 piñata
+1FAA9 ; fully-qualified # 🪩 E14.0 mirror ball
1FA86 ; fully-qualified # 🪆 E13.0 nesting dolls
2660 FE0F ; fully-qualified # ♠️ E0.6 spade suit
2660 ; unqualified # ♠ E0.6 spade suit
@@ -3907,8 +4013,8 @@
1F9F6 ; fully-qualified # 🧶 E11.0 yarn
1FAA2 ; fully-qualified # 🪢 E13.0 knot
-# Activities subtotal: 95
-# Activities subtotal: 95 w/o modifiers
+# Activities subtotal: 97
+# Activities subtotal: 97 w/o modifiers
# group: Objects
@@ -4009,6 +4115,7 @@
# subgroup: computer
1F50B ; fully-qualified # 🔋 E0.6 battery
+1FAAB ; fully-qualified # 🪫 E14.0 low battery
1F50C ; fully-qualified # 🔌 E0.6 electric plug
1F4BB ; fully-qualified # 💻 E0.6 laptop
1F5A5 FE0F ; fully-qualified # 🖥️ E0.7 desktop computer
@@ -4207,7 +4314,9 @@
1FA78 ; fully-qualified # 🩸 E12.0 drop of blood
1F48A ; fully-qualified # 💊 E0.6 pill
1FA79 ; fully-qualified # 🩹 E12.0 adhesive bandage
+1FA7C ; fully-qualified # 🩼 E14.0 crutch
1FA7A ; fully-qualified # 🩺 E12.0 stethoscope
+1FA7B ; fully-qualified # 🩻 E14.0 x-ray
# subgroup: household
1F6AA ; fully-qualified # 🚪 E0.6 door
@@ -4232,6 +4341,7 @@
1F9FB ; fully-qualified # 🧻 E11.0 roll of paper
1FAA3 ; fully-qualified # 🪣 E13.0 bucket
1F9FC ; fully-qualified # 🧼 E11.0 soap
+1FAE7 ; fully-qualified # 🫧 E14.0 bubbles
1FAA5 ; fully-qualified # 🪥 E13.0 toothbrush
1F9FD ; fully-qualified # 🧽 E11.0 sponge
1F9EF ; fully-qualified # 🧯 E11.0 fire extinguisher
@@ -4246,9 +4356,10 @@
26B1 ; unqualified # ⚱ E1.0 funeral urn
1F5FF ; fully-qualified # 🗿 E0.6 moai
1FAA7 ; fully-qualified # 🪧 E13.0 placard
+1FAAA ; fully-qualified # 🪪 E14.0 identification card
-# Objects subtotal: 299
-# Objects subtotal: 299 w/o modifiers
+# Objects subtotal: 304
+# Objects subtotal: 304 w/o modifiers
# group: Symbols
@@ -4409,6 +4520,7 @@
2795 ; fully-qualified # ➕ E0.6 plus
2796 ; fully-qualified # ➖ E0.6 minus
2797 ; fully-qualified # ➗ E0.6 divide
+1F7F0 ; fully-qualified # 🟰 E14.0 heavy equals sign
267E FE0F ; fully-qualified # ♾️ E11.0 infinity
267E ; unqualified # ♾ E11.0 infinity
@@ -4581,8 +4693,8 @@
1F533 ; fully-qualified # 🔳 E0.6 white square button
1F532 ; fully-qualified # 🔲 E0.6 black square button
-# Symbols subtotal: 301
-# Symbols subtotal: 301 w/o modifiers
+# Symbols subtotal: 302
+# Symbols subtotal: 302 w/o modifiers
# group: Flags
@@ -4871,7 +4983,7 @@
# Flags subtotal: 275 w/o modifiers
# Status Counts
-# fully-qualified : 3512
+# fully-qualified : 3624
# minimally-qualified : 817
# unqualified : 252
# component : 9
diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex
index cdbfeab02..53e2e9c89 100644
--- a/lib/pleroma/hashtag.ex
+++ b/lib/pleroma/hashtag.ex
@@ -61,7 +61,6 @@ def get_or_create_by_names(names) when is_list(names) do
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
end)
|> Repo.transaction() do
- Pleroma.Elasticsearch.maybe_bulk_post(hashtags, :hashtags)
{:ok, hashtags}
else
{:error, _name, value, _changes_so_far} -> {:error, value}
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 9e0ce0329..2ab09495d 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -341,6 +341,14 @@ def destroy_multiple(%{id: user_id} = _user, ids) do
|> Repo.delete_all()
end
+ def destroy_multiple_from_types(%{id: user_id}, types) do
+ from(n in Notification,
+ where: n.user_id == ^user_id,
+ where: n.type in ^types
+ )
+ |> Repo.delete_all()
+ end
+
def dismiss(%Pleroma.Activity{} = activity) do
Notification
|> where([n], n.activity_id == ^activity.id)
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index 99bce632c..3b266e59b 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -1,12 +1,17 @@
defmodule Pleroma.Search do
- @type search_map :: %{
- statuses: [map],
- accounts: [map],
- hashtags: [map]
- }
+ alias Pleroma.Workers.SearchIndexingWorker
- @doc """
- Searches for stuff
- """
- @callback search(map, map, keyword) :: search_map
+ def add_to_index(%Pleroma.Activity{id: activity_id}) do
+ SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id})
+ end
+
+ def remove_from_index(%Pleroma.Object{id: object_id}) do
+ SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id})
+ end
+
+ def search(query, options) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+
+ search_module.search(options[:for_user], query, options)
+ end
end
diff --git a/lib/pleroma/search/builtin.ex b/lib/pleroma/search/builtin.ex
deleted file mode 100644
index 3cbe2207a..000000000
--- a/lib/pleroma/search/builtin.ex
+++ /dev/null
@@ -1,138 +0,0 @@
-defmodule Pleroma.Search.Builtin do
- @behaviour Pleroma.Search
-
- alias Pleroma.Repo
- alias Pleroma.User
- alias Pleroma.Activity
- alias Pleroma.Web.MastodonAPI.AccountView
- alias Pleroma.Web.MastodonAPI.StatusView
- alias Pleroma.Web.Endpoint
-
- require Logger
-
- @impl Pleroma.Search
- def search(_conn, %{q: query} = params, options) do
- version = Keyword.get(options, :version)
- timeout = Keyword.get(Repo.config(), :timeout, 15_000)
- query = String.trim(query)
- default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}
-
- default_values
- |> Enum.map(fn {resource, default_value} ->
- if params[:type] in [nil, resource] do
- {resource, fn -> resource_search(version, resource, query, options) end}
- else
- {resource, fn -> default_value end}
- end
- end)
- |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end,
- timeout: timeout,
- on_timeout: :kill_task
- )
- |> Enum.reduce(default_values, fn
- {:ok, {resource, result}}, acc ->
- Map.put(acc, resource, result)
-
- _error, acc ->
- acc
- end)
- end
-
- defp resource_search(_, "accounts", query, options) do
- accounts = with_fallback(fn -> User.search(query, options) end)
-
- AccountView.render("index.json",
- users: accounts,
- for: options[:for_user],
- embed_relationships: options[:embed_relationships]
- )
- end
-
- defp resource_search(_, "statuses", query, options) do
- statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end)
-
- StatusView.render("index.json",
- activities: statuses,
- for: options[:for_user],
- as: :activity
- )
- end
-
- defp resource_search(:v2, "hashtags", query, options) do
- tags_path = Endpoint.url() <> "/tag/"
-
- query
- |> prepare_tags(options)
- |> Enum.map(fn tag ->
- %{name: tag, url: tags_path <> tag}
- end)
- end
-
- defp resource_search(:v1, "hashtags", query, options) do
- prepare_tags(query, options)
- end
-
- defp prepare_tags(query, options) do
- tags =
- query
- |> preprocess_uri_query()
- |> String.split(~r/[^#\w]+/u, trim: true)
- |> Enum.uniq_by(&String.downcase/1)
-
- explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end)
-
- tags =
- if Enum.any?(explicit_tags) do
- explicit_tags
- else
- tags
- end
-
- tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
-
- tags =
- if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
- add_joined_tag(tags)
- else
- tags
- end
-
- Pleroma.Pagination.paginate(tags, options)
- end
-
- # If `query` is a URI, returns last component of its path, otherwise returns `query`
- defp preprocess_uri_query(query) do
- if query =~ ~r/https?:\/\// do
- query
- |> String.trim_trailing("/")
- |> URI.parse()
- |> Map.get(:path)
- |> String.split("/")
- |> Enum.at(-1)
- else
- query
- end
- end
-
- defp add_joined_tag(tags) do
- tags
- |> Kernel.++([joined_tag(tags)])
- |> Enum.uniq_by(&String.downcase/1)
- end
-
- defp joined_tag(tags) do
- tags
- |> Enum.map(fn tag -> String.capitalize(tag) end)
- |> Enum.join()
- end
-
- defp with_fallback(f, fallback \\ []) do
- try do
- f.()
- rescue
- error ->
- Logger.error("#{__MODULE__} search error: #{inspect(error)}")
- fallback
- end
- end
-end
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/search/database_search.ex
similarity index 92%
rename from lib/pleroma/activity/search.ex
rename to lib/pleroma/search/database_search.ex
index 09671f621..3735a5fab 100644
--- a/lib/pleroma/activity/search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -2,7 +2,7 @@
# Copyright © 2017-2021 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.Activity.Search do
+defmodule Pleroma.Search.DatabaseSearch do
alias Pleroma.Activity
alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
@@ -13,6 +13,8 @@ defmodule Pleroma.Activity.Search do
import Ecto.Query
+ @behaviour Pleroma.Search.SearchBackend
+
def search(user, search_query, options \\ []) do
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
@@ -45,6 +47,12 @@ def search(user, search_query, options \\ []) do
end
end
+ @impl true
+ def add_to_index(_activity), do: nil
+
+ @impl true
+ def remove_from_index(_object), do: nil
+
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
@@ -57,7 +65,7 @@ def maybe_restrict_blocked(query, %User{} = user) do
def maybe_restrict_blocked(query, _), do: query
- defp restrict_public(q) do
+ def restrict_public(q) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
@@ -124,7 +132,7 @@ defp query_with(q, :rum, search_query, :websearch) do
)
end
- defp maybe_restrict_local(q, user) do
+ def maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
@@ -137,7 +145,7 @@ defp maybe_restrict_local(q, user) do
defp restrict_local(q), do: where(q, local: true)
- defp maybe_fetch(activities, user, search_query) do
+ def maybe_fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
diff --git a/lib/pleroma/search/elasticsearch.ex b/lib/pleroma/search/elasticsearch.ex
index 76d2c3277..7c7ca82c8 100644
--- a/lib/pleroma/search/elasticsearch.ex
+++ b/lib/pleroma/search/elasticsearch.ex
@@ -3,24 +3,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Elasticsearch do
- @behaviour Pleroma.Search
+ @behaviour Pleroma.Search.SearchBackend
alias Pleroma.Activity
alias Pleroma.Object.Fetcher
- alias Pleroma.Web.MastodonAPI.StatusView
- alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.ActivityPub.Visibility
alias Pleroma.Search.Elasticsearch.Parsers
- alias Pleroma.Web.Endpoint
- def es_query(:activity, query) do
+ def es_query(:activity, query, offset, limit) do
must = Parsers.Activity.parse(query)
if must == [] do
:skip
else
%{
- size: 50,
+ size: limit,
+ from: offset,
terminate_after: 50,
timeout: "5s",
sort: [
@@ -36,50 +34,6 @@ def es_query(:activity, query) do
end
end
- def es_query(:user, query) do
- must = Parsers.User.parse(query)
-
- if must == [] do
- :skip
- else
- %{
- size: 50,
- terminate_after: 50,
- timeout: "5s",
- sort: [
- "_score"
- ],
- query: %{
- bool: %{
- must: must
- }
- }
- }
- end
- end
-
- def es_query(:hashtag, query) do
- must = Parsers.Hashtag.parse(query)
-
- if must == [] do
- :skip
- else
- %{
- size: 50,
- terminate_after: 50,
- timeout: "5s",
- sort: [
- "_score"
- ],
- query: %{
- bool: %{
- must: Parsers.Hashtag.parse(query)
- }
- }
- }
- end
- end
-
defp maybe_fetch(:activity, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
@@ -90,8 +44,10 @@ defp maybe_fetch(:activity, search_query) do
end
end
- @impl Pleroma.Search
- def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) do
+ def search(user, query, options) do
+ limit = Enum.min([Keyword.get(options, :limit), 40])
+ offset = Keyword.get(options, :offset, 0)
+
parsed_query =
query
|> String.trim()
@@ -104,30 +60,13 @@ def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) d
activity_task =
Task.async(fn ->
- q = es_query(:activity, parsed_query)
+ q = es_query(:activity, parsed_query, offset, limit)
- Pleroma.Elasticsearch.search(:activities, q)
+ Pleroma.Search.Elasticsearch.Store.search(:activities, q)
|> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
end)
- user_task =
- Task.async(fn ->
- q = es_query(:user, parsed_query)
-
- Pleroma.Elasticsearch.search(:users, q)
- |> Enum.filter(fn x -> Pleroma.User.visible_for(x, user) == :visible end)
- end)
-
- hashtag_task =
- Task.async(fn ->
- q = es_query(:hashtag, parsed_query)
-
- Pleroma.Elasticsearch.search(:hashtags, q)
- end)
-
activity_results = Task.await(activity_task)
- user_results = Task.await(user_task)
- hashtag_results = Task.await(hashtag_task)
direct_activity = Task.await(activity_fetch_task)
activity_results =
@@ -137,25 +76,16 @@ def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) d
[direct_activity | activity_results]
end
- %{
- "accounts" =>
- AccountView.render("index.json",
- users: user_results,
- for: user
- ),
- "hashtags" =>
- Enum.map(hashtag_results, fn x ->
- %{
- url: Endpoint.url() <> "/tag/" <> x,
- name: x
- }
- end),
- "statuses" =>
- StatusView.render("index.json",
- activities: activity_results,
- for: user,
- as: :activity
- )
- }
+ activity_results
+ end
+
+ @impl true
+ def add_to_index(activity) do
+ Elasticsearch.put_document(Pleroma.Search.Elasticsearch.Cluster, activity, "activities")
+ end
+
+ @impl true
+ def remove_from_index(object) do
+ Elasticsearch.delete_document(Pleroma.Search.Elasticsearch.Cluster, object, "activities")
end
end
diff --git a/lib/pleroma/search/elasticsearch/cluster.ex b/lib/pleroma/search/elasticsearch/cluster.ex
new file mode 100644
index 000000000..4f76c4ebc
--- /dev/null
+++ b/lib/pleroma/search/elasticsearch/cluster.ex
@@ -0,0 +1,4 @@
+defmodule Pleroma.Search.Elasticsearch.Cluster do
+ @moduledoc false
+ use Elasticsearch.Cluster, otp_app: :pleroma
+end
diff --git a/lib/pleroma/search/elasticsearch/document_mappings/activity.ex b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex
new file mode 100644
index 000000000..3a84e991b
--- /dev/null
+++ b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex
@@ -0,0 +1,61 @@
+# Akkoma: A lightweight social networking server
+# Copyright © 2022-2022 Akkoma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defimpl Elasticsearch.Document, for: Pleroma.Activity do
+ alias Pleroma.Object
+ require Pleroma.Constants
+
+ def id(obj), do: obj.id
+ def routing(_), do: false
+
+ def object_to_search_data(object) do
+ # Only index public or unlisted Notes
+ if not is_nil(object) and object.data["type"] == "Note" and
+ not is_nil(object.data["content"]) and
+ (Pleroma.Constants.as_public() in object.data["to"] or
+ Pleroma.Constants.as_public() in object.data["cc"]) and
+ String.length(object.data["content"]) > 1 do
+ data = object.data
+
+ content_str =
+ case data["content"] do
+ [nil | rest] -> to_string(rest)
+ str -> str
+ end
+
+ content =
+ with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
+ trimmed <- String.trim(scrubbed) do
+ trimmed
+ end
+
+ if String.length(content) > 1 do
+ {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
+ %{
+ _timestamp: published,
+ content: content,
+ instance: URI.parse(object.data["actor"]).host,
+ hashtags: Object.hashtags(object),
+ user: Pleroma.User.get_cached_by_ap_id(object.data["actor"]).nickname
+ }
+ else
+ %{}
+ end
+ else
+ %{}
+ end
+ end
+
+ def encode(activity) do
+ object = Pleroma.Object.normalize(activity)
+ object_to_search_data(object)
+ end
+end
+
+defimpl Elasticsearch.Document, for: Pleroma.Object do
+ def id(obj), do: obj.id
+ def routing(_), do: false
+ def encode(_), do: nil
+end
diff --git a/lib/pleroma/search/elasticsearch/hashtag_parser.ex b/lib/pleroma/search/elasticsearch/hashtag_parser.ex
deleted file mode 100644
index 911dc651c..000000000
--- a/lib/pleroma/search/elasticsearch/hashtag_parser.ex
+++ /dev/null
@@ -1,34 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Search.Elasticsearch.Parsers.Hashtag do
- defp to_es(term) when is_binary(term) do
- %{
- term: %{
- hashtag: %{
- value: String.downcase(term)
- }
- }
- }
- end
-
- defp to_es({:quoted, term}), do: to_es(term)
-
- defp to_es({:filter, ["hashtag", query]}) do
- %{
- term: %{
- hashtag: %{
- value: String.downcase(query)
- }
- }
- }
- end
-
- defp to_es({:filter, _}), do: nil
-
- def parse(q) do
- Enum.map(q, &to_es/1)
- |> Enum.filter(fn x -> x != nil end)
- end
-end
diff --git a/lib/pleroma/search/elasticsearch/store.ex b/lib/pleroma/search/elasticsearch/store.ex
new file mode 100644
index 000000000..895b76d7f
--- /dev/null
+++ b/lib/pleroma/search/elasticsearch/store.ex
@@ -0,0 +1,52 @@
+# Akkoma: A lightweight social networking server
+# Copyright © 2022-2022 Akkoma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.Elasticsearch.Store do
+ @behaviour Elasticsearch.Store
+ alias Pleroma.Search.Elasticsearch.Cluster
+ require Logger
+
+ alias Pleroma.Repo
+
+ @impl true
+ def stream(schema) do
+ Repo.stream(schema)
+ end
+
+ @impl true
+ def transaction(fun) do
+ {:ok, result} = Repo.transaction(fun, timeout: :infinity)
+ result
+ end
+
+ def search(_, _, _, :skip), do: []
+
+ def search(:raw, index, q) do
+ with {:ok, raw_results} <- Elasticsearch.post(Cluster, "/#{index}/_search", q) do
+ results =
+ raw_results
+ |> Map.get("hits", %{})
+ |> Map.get("hits", [])
+
+ {:ok, results}
+ else
+ {:error, e} ->
+ Logger.error(e)
+ {:error, e}
+ end
+ end
+
+ def search(:activities, q) do
+ with {:ok, results} <- search(:raw, "activities", q) do
+ results
+ |> Enum.map(fn result -> result["_id"] end)
+ |> Pleroma.Activity.all_by_ids_with_object()
+ |> Enum.sort(&(&1.inserted_at >= &2.inserted_at))
+ else
+ e ->
+ Logger.error(e)
+ []
+ end
+ end
+end
diff --git a/lib/pleroma/search/elasticsearch/user_paser.ex b/lib/pleroma/search/elasticsearch/user_paser.ex
deleted file mode 100644
index 4176c6141..000000000
--- a/lib/pleroma/search/elasticsearch/user_paser.ex
+++ /dev/null
@@ -1,57 +0,0 @@
-# Akkoma: A lightweight social networking server
-# Copyright © 2022-2022 Akkoma Authors
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Search.Elasticsearch.Parsers.User do
- defp to_es(term) when is_binary(term) do
- %{
- bool: %{
- minimum_should_match: 1,
- should: [
- %{
- match: %{
- bio: %{
- query: term,
- operator: "AND"
- }
- }
- },
- %{
- term: %{
- nickname: %{
- value: term
- }
- }
- },
- %{
- match: %{
- display_name: %{
- query: term,
- operator: "AND"
- }
- }
- }
- ]
- }
- }
- end
-
- defp to_es({:quoted, term}), do: to_es(term)
-
- defp to_es({:filter, ["user", query]}) do
- %{
- term: %{
- nickname: %{
- value: query
- }
- }
- }
- end
-
- defp to_es({:filter, _}), do: nil
-
- def parse(q) do
- Enum.map(q, &to_es/1)
- |> Enum.filter(fn x -> x != nil end)
- end
-end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..3db65f261
--- /dev/null
+++ b/lib/pleroma/search/meilisearch.ex
@@ -0,0 +1,169 @@
+defmodule Pleroma.Search.Meilisearch do
+ require Logger
+ require Pleroma.Constants
+
+ alias Pleroma.Activity
+
+ import Pleroma.Search.DatabaseSearch
+ import Ecto.Query
+
+ @behaviour Pleroma.Search.SearchBackend
+
+ defp meili_headers do
+ private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key])
+
+ [{"Content-Type", "application/json"}] ++
+ if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
+ end
+
+ def meili_get(path) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ result =
+ Pleroma.HTTP.get(
+ Path.join(endpoint, path),
+ meili_headers()
+ )
+
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
+ end
+
+ def meili_post(path, params) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ result =
+ Pleroma.HTTP.post(
+ Path.join(endpoint, path),
+ Jason.encode!(params),
+ meili_headers()
+ )
+
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
+ end
+
+ def meili_put(path, params) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ result =
+ Pleroma.HTTP.request(
+ :put,
+ Path.join(endpoint, path),
+ Jason.encode!(params),
+ meili_headers(),
+ []
+ )
+
+ with {:ok, res} <- result do
+ {:ok, Jason.decode!(res.body)}
+ end
+ end
+
+ def meili_delete!(path) do
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, _} =
+ Pleroma.HTTP.request(
+ :delete,
+ Path.join(endpoint, path),
+ "",
+ meili_headers(),
+ []
+ )
+ end
+
+ def search(user, query, options \\ []) do
+ limit = Enum.min([Keyword.get(options, :limit), 40])
+ offset = Keyword.get(options, :offset, 0)
+ author = Keyword.get(options, :author)
+
+ res =
+ meili_post(
+ "/indexes/objects/search",
+ %{q: query, offset: offset, limit: limit}
+ )
+
+ with {:ok, result} <- res do
+ hits = result["hits"] |> Enum.map(& &1["ap"])
+
+ try do
+ hits
+ |> Activity.create_by_object_ap_id()
+ |> Activity.with_preloaded_object()
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> maybe_fetch(user, query)
+ |> order_by([object: obj], desc: obj.data["published"])
+ |> Pleroma.Repo.all()
+ rescue
+ _ -> maybe_fetch([], user, query)
+ end
+ end
+ end
+
+ def object_to_search_data(object) do
+ # Only index public or unlisted Notes
+ if not is_nil(object) and object.data["type"] == "Note" and
+ not is_nil(object.data["content"]) and
+ (Pleroma.Constants.as_public() in object.data["to"] or
+ Pleroma.Constants.as_public() in object.data["cc"]) and
+ String.length(object.data["content"]) > 1 do
+ data = object.data
+
+ content_str =
+ case data["content"] do
+ [nil | rest] -> to_string(rest)
+ str -> str
+ end
+
+ content =
+ with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str),
+ trimmed <- String.trim(scrubbed) do
+ trimmed
+ end
+
+ if String.length(content) > 1 do
+ {:ok, published, _} = DateTime.from_iso8601(data["published"])
+
+ %{
+ id: object.id,
+ content: content,
+ ap: data["id"],
+ published: published |> DateTime.to_unix()
+ }
+ end
+ end
+ end
+
+ @impl true
+ def add_to_index(activity) do
+ maybe_search_data = object_to_search_data(activity.object)
+
+ if activity.data["type"] == "Create" and maybe_search_data do
+ result =
+ meili_put(
+ "/indexes/objects/documents",
+ [maybe_search_data]
+ )
+
+ with {:ok, res} <- result,
+ true <- Map.has_key?(res, "uid") do
+ # Do nothing
+ else
+ _ ->
+ Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
+ end
+ end
+ end
+
+ @impl true
+ def remove_from_index(object) do
+ meili_delete!("/indexes/objects/documents/#{object.id}")
+ end
+end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
new file mode 100644
index 000000000..ed6bfd329
--- /dev/null
+++ b/lib/pleroma/search/search_backend.ex
@@ -0,0 +1,17 @@
+defmodule Pleroma.Search.SearchBackend do
+ @doc """
+ Add the object associated with the activity to the search index.
+
+ The whole activity is passed, to allow filtering on things such as scope.
+ """
+ @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil
+
+ @doc """
+ Remove the object from the index.
+
+ Just the object, as opposed to the whole activity, is passed, since the object
+ is what contains the actual content and there is no need for fitlering when removing
+ from index.
+ """
+ @callback remove_from_index(object :: Pleroma.Object.t()) :: nil
+end
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
index 35e245237..50f7fcf2a 100644
--- a/lib/pleroma/telemetry/logger.ex
+++ b/lib/pleroma/telemetry/logger.ex
@@ -12,8 +12,7 @@ defmodule Pleroma.Telemetry.Logger do
[:pleroma, :connection_pool, :reclaim, :stop],
[:pleroma, :connection_pool, :provision_failure],
[:pleroma, :connection_pool, :client, :dead],
- [:pleroma, :connection_pool, :client, :add],
- [:pleroma, :repo, :query]
+ [:pleroma, :connection_pool, :client, :add]
]
def attach do
:telemetry.attach_many(
@@ -93,64 +92,4 @@ def handle_event(
end
def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok
-
- def handle_event(
- [:pleroma, :repo, :query] = _name,
- %{query_time: query_time} = measurements,
- %{source: source} = metadata,
- config
- ) do
- logging_config = Pleroma.Config.get([:telemetry, :slow_queries_logging], [])
-
- if logging_config[:enabled] &&
- logging_config[:min_duration] &&
- query_time > logging_config[:min_duration] and
- (is_nil(logging_config[:exclude_sources]) or
- source not in logging_config[:exclude_sources]) do
- log_slow_query(measurements, metadata, config)
- else
- :ok
- end
- end
-
- defp log_slow_query(
- %{query_time: query_time} = _measurements,
- %{source: _source, query: query, params: query_params, repo: repo} = _metadata,
- _config
- ) do
- sql_explain =
- with {:ok, %{rows: explain_result_rows}} <-
- repo.query("EXPLAIN " <> query, query_params, log: false) do
- Enum.map_join(explain_result_rows, "\n", & &1)
- end
-
- {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace)
-
- pleroma_stacktrace =
- Enum.filter(stacktrace, fn
- {__MODULE__, _, _, _} ->
- false
-
- {mod, _, _, _} ->
- mod
- |> to_string()
- |> String.starts_with?("Elixir.Pleroma.")
- end)
-
- Logger.warn(fn ->
- """
- Slow query!
-
- Total time: #{round(query_time / 1_000)} ms
-
- #{query}
-
- #{inspect(query_params, limit: :infinity)}
-
- #{sql_explain}
-
- #{Exception.format_stacktrace(pleroma_stacktrace)}
- """
- end)
- end
end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index efe9ec5d6..dc6c661ea 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -151,6 +151,7 @@ defmodule Pleroma.User do
field(:pinned_objects, :map, default: %{})
field(:is_suggested, :boolean, default: false)
field(:last_status_at, :naive_datetime)
+ field(:language, :string)
embeds_one(
:notification_settings,
@@ -734,7 +735,8 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
:password_confirmation,
:emoji,
:accepts_chat_messages,
- :registration_reason
+ :registration_reason,
+ :language
])
|> validate_required([:name, :nickname, :password, :password_confirmation])
|> validate_confirmation(:password)
@@ -1089,11 +1091,24 @@ def update_and_set_cache(struct, params) do
|> update_and_set_cache()
end
- def update_and_set_cache(changeset) do
+ def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do
+ was_superuser_before_update = User.superuser?(user)
+
with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
- Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user)
set_cache(user)
end
+ |> maybe_remove_report_notifications(was_superuser_before_update)
+ end
+
+ defp maybe_remove_report_notifications({:ok, %Pleroma.User{} = user} = result, true) do
+ if not User.superuser?(user),
+ do: user |> Notification.destroy_multiple_from_types(["pleroma:report"])
+
+ result
+ end
+
+ defp maybe_remove_report_notifications(result, _) do
+ result
end
def get_user_friends_ap_ids(user) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 756096952..e6548a818 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -140,6 +140,9 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
+ # Add local posts to search index
+ if local, do: Pleroma.Search.add_to_index(activity)
+
{:ok, activity}
else
%Activity{} = activity ->
diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
index 851e95d22..627f52168 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
@@ -24,7 +24,7 @@ defp score_displayname("federationbot"), do: 1.0
defp score_displayname("fedibot"), do: 1.0
defp score_displayname(_), do: 0.0
- defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+ defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do
# nickname will be a binary string except when following a relay
nick_score =
if is_binary(nickname) do
@@ -45,19 +45,32 @@ defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
0.0
end
- nick_score + name_score
+ # actor_type "Service" is a Bot account
+ actor_type_score =
+ if actor_type == "Service" do
+ 1.0
+ else
+ 0.0
+ end
+
+ nick_score + name_score + actor_type_score
end
defp determine_if_followbot(_), do: 0.0
+ defp bot_allowed?(%{"object" => target}, bot_actor) do
+ %User{} = user = normalize_by_ap_id(target)
+
+ User.following?(user, bot_actor)
+ end
+
@impl true
def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
%User{} = actor = normalize_by_ap_id(actor_id)
score = determine_if_followbot(actor)
- # TODO: scan biography data for keywords and score it somehow.
- if score < 0.8 do
+ if score < 0.8 || bot_allowed?(message, actor) do
{:ok, message}
else
{:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"}
diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
index 0dd415732..61e95b49a 100644
--- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
@@ -12,6 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
+ defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do
+ shortcode == pattern
+ end
+
+ defp shortcode_matches?(shortcode, pattern) do
+ String.match?(shortcode, pattern)
+ end
+
defp steal_emoji({shortcode, url}, emoji_dir_path) do
url = Pleroma.Web.MediaProxy.url(url)
@@ -72,7 +80,7 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa
reject_emoji? =
[:mrf_steal_emoji, :rejected_shortcodes]
|> Config.get([])
- |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+ |> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end)
!reject_emoji?
end)
@@ -122,8 +130,12 @@ def config_description do
%{
key: :rejected_shortcodes,
type: {:list, :string},
- description: "Regex-list of shortcodes to reject",
- suggestions: [""]
+ description: """
+ A list of patterns or matches to reject shortcodes with.
+
+ Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+ """,
+ suggestions: ["foo", ~r/foo/]
},
%{
key: :size_limit,
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
index 815995895..a9f395c5e 100644
--- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
@@ -108,6 +108,7 @@ defp fix(data) do
|> fix_replies()
|> fix_source()
|> fix_misskey_content()
+ |> Transmogrifier.fix_attachments()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
end
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 214647dbf..d4e507287 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -28,7 +28,6 @@ def common_pipeline(object, meta) do
case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do
{:ok, {:ok, activity, meta}} ->
side_effects().handle_after_transaction(meta)
- side_effects().handle_after_transaction(activity)
{:ok, activity, meta}
{:ok, value} ->
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 9c2f89e72..e2371b693 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors
+# Copyright © 2017-2022 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.SideEffects do
@@ -193,6 +193,7 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do
# - Increase replies count
# - Set up ActivityExpiration
# - Set up notifications
+ # - Index incoming posts for search (if needed)
@impl true
def handle(%{data: %{"type" => "Create"}} = activity, meta) do
with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta),
@@ -222,6 +223,8 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
+ Pleroma.Search.add_to_index(Map.put(activity, :object, object))
+
meta =
meta
|> add_notifications(notifications)
@@ -269,6 +272,7 @@ def handle(%{data: %{"type" => "Undo", "object" => undone_object}} = object, met
def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do
reacted_object = Object.get_by_ap_id(object.data["object"])
Utils.add_emoji_reaction_to_object(object, reacted_object)
+
Notification.create_notifications(object)
{:ok, object, meta}
@@ -281,6 +285,7 @@ def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do
# - Reduce the user note count
# - Reduce the reply count
# - Stream out the activity
+ # - Removes posts from search index (if needed)
@impl true
def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do
deleted_object =
@@ -320,6 +325,12 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object,
if result == :ok do
Notification.create_notifications(object)
+
+ # Only remove from index when deleting actual objects, not users or anything else
+ with %Pleroma.Object{} <- deleted_object do
+ Pleroma.Search.remove_from_index(deleted_object)
+ end
+
{:ok, object, meta}
else
{:error, result}
@@ -537,24 +548,6 @@ defp add_notifications(meta, notifications) do
end
@impl true
- def handle_after_transaction(%Pleroma.Activity{data: %{"type" => "Create"}} = activity) do
- Pleroma.Elasticsearch.put_by_id(:activity, activity.id)
- end
-
- def handle_after_transaction(%Pleroma.Activity{
- data: %{"type" => "Delete", "deleted_activity_id" => id}
- }) do
- Pleroma.Elasticsearch.delete_by_id(:activity, id)
- end
-
- def handle_after_transaction(%Pleroma.Activity{}) do
- :ok
- end
-
- def handle_after_transaction(%Pleroma.Object{}) do
- :ok
- end
-
def handle_after_transaction(meta) do
meta
|> send_notifications()
diff --git a/lib/pleroma/web/activity_pub/side_effects/handling.ex b/lib/pleroma/web/activity_pub/side_effects/handling.ex
index a82305155..eb012f576 100644
--- a/lib/pleroma/web/activity_pub/side_effects/handling.ex
+++ b/lib/pleroma/web/activity_pub/side_effects/handling.ex
@@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors
+# Copyright © 2017-2022 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.SideEffects.Handling do
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index f5304d7d6..bdbae9b74 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -507,6 +507,11 @@ defp create_request do
type: :string,
nullable: true,
description: "Invite token required when the registrations aren't public"
+ },
+ language: %Schema{
+ type: :string,
+ nullable: true,
+ description: "User's preferred language for emails"
}
},
example: %{
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 92afd5cb6..856fa95b9 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -396,13 +396,7 @@ def listen(user, data) do
def post(user, %{status: _} = data) do
with {:ok, draft} <- ActivityDraft.create(user, data) do
- activity = ActivityPub.create(draft.changes, draft.preview?)
-
- unless draft.preview? do
- Pleroma.Elasticsearch.maybe_put_into_elasticsearch(activity)
- end
-
- activity
+ ActivityPub.create(draft.changes, draft.preview?)
end
end
diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
index c0fb35e01..52771205e 100644
--- a/lib/pleroma/web/feed/feed_view.ex
+++ b/lib/pleroma/web/feed/feed_view.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.Feed.FeedView do
alias Pleroma.Formatter
alias Pleroma.Object
alias Pleroma.User
+ alias Pleroma.Web.Gettext
alias Pleroma.Web.MediaProxy
require Pleroma.Constants
diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex
index c0ca4d0e9..7afcd38f0 100644
--- a/lib/pleroma/web/gettext.ex
+++ b/lib/pleroma/web/gettext.ex
@@ -25,4 +25,196 @@ defmodule Pleroma.Web.Gettext do
See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage.
"""
use Gettext, otp_app: :pleroma
+
+ def language_tag do
+ # Naive implementation: HTML lang attribute uses BCP 47, which
+ # uses - as a separator.
+ # https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang
+
+ Gettext.get_locale()
+ |> String.replace("_", "-", global: true)
+ end
+
+ def normalize_locale(locale) do
+ if is_binary(locale) do
+ String.replace(locale, "-", "_", global: true)
+ else
+ nil
+ end
+ end
+
+ def supports_locale?(locale) do
+ Pleroma.Web.Gettext
+ |> Gettext.known_locales()
+ |> Enum.member?(locale)
+ end
+
+ def variant?(locale), do: String.contains?(locale, "_")
+
+ def language_for_variant(locale) do
+ Enum.at(String.split(locale, "_"), 0)
+ end
+
+ def ensure_fallbacks(locales) do
+ locales
+ |> Enum.flat_map(fn locale ->
+ others =
+ other_supported_variants_of_locale(locale)
+ |> Enum.filter(fn l -> not Enum.member?(locales, l) end)
+
+ [locale] ++ others
+ end)
+ end
+
+ def other_supported_variants_of_locale(locale) do
+ cond do
+ supports_locale?(locale) ->
+ []
+
+ variant?(locale) ->
+ lang = language_for_variant(locale)
+ if supports_locale?(lang), do: [lang], else: []
+
+ true ->
+ Gettext.known_locales(Pleroma.Web.Gettext)
+ |> Enum.filter(fn l -> String.starts_with?(l, locale <> "_") end)
+ end
+ end
+
+ def get_locales do
+ Process.get({Pleroma.Web.Gettext, :locales}, [])
+ end
+
+ def is_locale_list(locales) do
+ Enum.all?(locales, &is_binary/1)
+ end
+
+ def put_locales(locales) do
+ if is_locale_list(locales) do
+ Process.put({Pleroma.Web.Gettext, :locales}, Enum.uniq(locales))
+ Gettext.put_locale(Enum.at(locales, 0, Gettext.get_locale()))
+ :ok
+ else
+ {:error, :not_locale_list}
+ end
+ end
+
+ def locale_or_default(locale) do
+ if supports_locale?(locale) do
+ locale
+ else
+ Gettext.get_locale()
+ end
+ end
+
+ def with_locales_func(locales, fun) do
+ prev_locales = Process.get({Pleroma.Web.Gettext, :locales})
+ put_locales(locales)
+
+ try do
+ fun.()
+ after
+ if prev_locales do
+ put_locales(prev_locales)
+ else
+ Process.delete({Pleroma.Web.Gettext, :locales})
+ Process.delete(Gettext)
+ end
+ end
+ end
+
+ defmacro with_locales(locales, do: fun) do
+ quote do
+ Pleroma.Web.Gettext.with_locales_func(unquote(locales), fn ->
+ unquote(fun)
+ end)
+ end
+ end
+
+ def to_locale_list(locale) when is_binary(locale) do
+ locale
+ |> String.split(",")
+ |> Enum.filter(&supports_locale?/1)
+ end
+
+ def to_locale_list(_), do: []
+
+ defmacro with_locale_or_default(locale, do: fun) do
+ quote do
+ Pleroma.Web.Gettext.with_locales_func(
+ Pleroma.Web.Gettext.to_locale_list(unquote(locale))
+ |> Enum.concat(Pleroma.Web.Gettext.get_locales()),
+ fn ->
+ unquote(fun)
+ end
+ )
+ end
+ end
+
+ defp next_locale(locale, list) do
+ index = Enum.find_index(list, fn item -> item == locale end)
+
+ if not is_nil(index) do
+ Enum.at(list, index + 1)
+ else
+ nil
+ end
+ end
+
+ # We do not yet have a proper English translation. The "English"
+ # version is currently but the fallback msgid. However, this
+ # will not work if the user puts English as the first language,
+ # and at the same time specifies other languages, as gettext will
+ # think the English translation is missing, and call
+ # handle_missing_translation functions. This may result in
+ # text in other languages being shown even if English is preferred
+ # by the user.
+ #
+ # To prevent this, we do not allow fallbacking when the current
+ # locale missing a translation is English.
+ defp should_fallback?(locale) do
+ locale != "en"
+ end
+
+ def handle_missing_translation(locale, domain, msgctxt, msgid, bindings) do
+ next = next_locale(locale, get_locales())
+
+ if is_nil(next) or not should_fallback?(locale) do
+ super(locale, domain, msgctxt, msgid, bindings)
+ else
+ {:ok,
+ Gettext.with_locale(next, fn ->
+ Gettext.dpgettext(Pleroma.Web.Gettext, domain, msgctxt, msgid, bindings)
+ end)}
+ end
+ end
+
+ def handle_missing_plural_translation(
+ locale,
+ domain,
+ msgctxt,
+ msgid,
+ msgid_plural,
+ n,
+ bindings
+ ) do
+ next = next_locale(locale, get_locales())
+
+ if is_nil(next) or not should_fallback?(locale) do
+ super(locale, domain, msgctxt, msgid, msgid_plural, n, bindings)
+ else
+ {:ok,
+ Gettext.with_locale(next, fn ->
+ Gettext.dpngettext(
+ Pleroma.Web.Gettext,
+ domain,
+ msgctxt,
+ msgid,
+ msgid_plural,
+ n,
+ bindings
+ )
+ end)}
+ end
+ end
end
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index a307807a9..83cebbb96 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -217,6 +217,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
|> Maps.put_if_present(:is_locked, params[:locked])
# Note: param name is indeed :discoverable (not an error)
|> Maps.put_if_present(:is_discoverable, params[:discoverable])
+ |> Maps.put_if_present(:language, Pleroma.Web.Gettext.normalize_locale(params[:language]))
# What happens here:
#
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 86ad388fd..e4acba226 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -1,13 +1,16 @@
# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors
+# Copyright © 2017-2022 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MastodonAPI.SearchController do
use Pleroma.Web, :controller
+ alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ControllerHelper
+ alias Pleroma.Web.Endpoint
alias Pleroma.Web.MastodonAPI.AccountView
+ alias Pleroma.Web.MastodonAPI.StatusView
alias Pleroma.Web.Plugs.OAuthScopesPlug
alias Pleroma.Web.Plugs.RateLimiter
@@ -41,13 +44,34 @@ def account_search(%{assigns: %{user: user}} = conn, %{q: query} = params) do
def search2(conn, params), do: do_search(:v2, conn, params)
def search(conn, params), do: do_search(:v1, conn, params)
- defp do_search(version, %{assigns: %{user: user}} = conn, params) do
- options =
- search_options(params, user)
- |> Keyword.put(:version, version)
+ defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do
+ query = String.trim(query)
+ options = search_options(params, user)
+ timeout = Keyword.get(Repo.config(), :timeout, 15_000)
+ default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}
- search_provider = Pleroma.Config.get([:search, :provider])
- json(conn, search_provider.search(conn, params, options))
+ result =
+ default_values
+ |> Enum.map(fn {resource, default_value} ->
+ if params[:type] in [nil, resource] do
+ {resource, fn -> resource_search(version, resource, query, options) end}
+ else
+ {resource, fn -> default_value end}
+ end
+ end)
+ |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end,
+ timeout: timeout,
+ on_timeout: :kill_task
+ )
+ |> Enum.reduce(default_values, fn
+ {:ok, {resource, result}}, acc ->
+ Map.put(acc, resource, result)
+
+ _error, acc ->
+ acc
+ end)
+
+ json(conn, result)
end
defp search_options(params, user) do
@@ -64,6 +88,104 @@ defp search_options(params, user) do
|> Enum.filter(&elem(&1, 1))
end
+ defp resource_search(_, "accounts", query, options) do
+ accounts = with_fallback(fn -> User.search(query, options) end)
+
+ AccountView.render("index.json",
+ users: accounts,
+ for: options[:for_user],
+ embed_relationships: options[:embed_relationships]
+ )
+ end
+
+ defp resource_search(_, "statuses", query, options) do
+ statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end)
+
+ StatusView.render("index.json",
+ activities: statuses,
+ for: options[:for_user],
+ as: :activity
+ )
+ end
+
+ defp resource_search(:v2, "hashtags", query, options) do
+ tags_path = Endpoint.url() <> "/tag/"
+
+ query
+ |> prepare_tags(options)
+ |> Enum.map(fn tag ->
+ %{name: tag, url: tags_path <> tag}
+ end)
+ end
+
+ defp resource_search(:v1, "hashtags", query, options) do
+ prepare_tags(query, options)
+ end
+
+ defp prepare_tags(query, options) do
+ tags =
+ query
+ |> preprocess_uri_query()
+ |> String.split(~r/[^#\w]+/u, trim: true)
+ |> Enum.uniq_by(&String.downcase/1)
+
+ explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end)
+
+ tags =
+ if Enum.any?(explicit_tags) do
+ explicit_tags
+ else
+ tags
+ end
+
+ tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
+
+ tags =
+ if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
+ add_joined_tag(tags)
+ else
+ tags
+ end
+
+ Pleroma.Pagination.paginate(tags, options)
+ end
+
+ defp add_joined_tag(tags) do
+ tags
+ |> Kernel.++([joined_tag(tags)])
+ |> Enum.uniq_by(&String.downcase/1)
+ end
+
+ # If `query` is a URI, returns last component of its path, otherwise returns `query`
+ defp preprocess_uri_query(query) do
+ if query =~ ~r/https?:\/\// do
+ query
+ |> String.trim_trailing("/")
+ |> URI.parse()
+ |> Map.get(:path)
+ |> String.split("/")
+ |> Enum.at(-1)
+ else
+ query
+ end
+ end
+
+ defp joined_tag(tags) do
+ tags
+ |> Enum.map(fn tag -> String.capitalize(tag) end)
+ |> Enum.join()
+ end
+
+ defp with_fallback(f, fallback \\ []) do
+ try do
+ f.()
+ rescue
+ error ->
+ Logger.error("#{__MODULE__} search error: #{inspect(error)}")
+ fallback
+ end
+ end
+
defp get_author(%{account_id: account_id}) when is_binary(account_id),
do: User.get_cached_by_id(account_id)
diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index 2eff4d9d0..60f4c44d7 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -384,11 +384,13 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{id: id}) do
def context(%{assigns: %{user: user}} = conn, %{id: id}) do
with %Activity{} = activity <- Activity.get_by_id(id) do
activities =
- ActivityPub.fetch_activities_for_context(activity.data["context"], %{
+ activity.data["context"]
+ |> ActivityPub.fetch_activities_for_context(%{
blocking_user: user,
user: user,
exclude_id: activity.id
})
+ |> Enum.filter(fn activity -> Visibility.visible_for_user?(activity, user) end)
render(conn, "context.json", activity: activity, activities: activities, user: user)
end
diff --git a/lib/pleroma/web/o_auth/mfa_view.ex b/lib/pleroma/web/o_auth/mfa_view.ex
index 3d473f29c..952c90efe 100644
--- a/lib/pleroma/web/o_auth/mfa_view.ex
+++ b/lib/pleroma/web/o_auth/mfa_view.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.MFAView do
use Pleroma.Web, :view
import Phoenix.HTML.Form
alias Pleroma.MFA
+ alias Pleroma.Web.Gettext
def render("mfa_response.json", %{token: token, user: user}) do
%{
diff --git a/lib/pleroma/web/o_auth/o_auth_view.ex b/lib/pleroma/web/o_auth/o_auth_view.ex
index 1419c96a2..57a315705 100644
--- a/lib/pleroma/web/o_auth/o_auth_view.ex
+++ b/lib/pleroma/web/o_auth/o_auth_view.ex
@@ -5,6 +5,8 @@
defmodule Pleroma.Web.OAuth.OAuthView do
use Pleroma.Web, :view
import Phoenix.HTML.Form
+ import Phoenix.HTML
+ alias Pleroma.Web.Gettext
alias Pleroma.Web.OAuth.Token.Utils
diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex
index d77191cff..e78917199 100644
--- a/lib/pleroma/web/plugs/set_locale_plug.ex
+++ b/lib/pleroma/web/plugs/set_locale_plug.ex
@@ -6,18 +6,56 @@
defmodule Pleroma.Web.Plugs.SetLocalePlug do
import Plug.Conn, only: [get_req_header: 2, assign: 3]
+ def frontend_language_cookie_name, do: "userLanguage"
+
def init(_), do: nil
def call(conn, _) do
- locale = get_locale_from_header(conn) || Gettext.get_locale()
- Gettext.put_locale(locale)
- assign(conn, :locale, locale)
+ locales = get_locales_from_header(conn)
+ first_locale = Enum.at(locales, 0, Gettext.get_locale())
+
+ Pleroma.Web.Gettext.put_locales(locales)
+
+ conn
+ |> assign(:locale, first_locale)
+ |> assign(:locales, locales)
end
- defp get_locale_from_header(conn) do
+ defp get_locales_from_header(conn) do
conn
- |> extract_accept_language()
- |> Enum.find(&supported_locale?/1)
+ |> extract_preferred_language()
+ |> normalize_language_codes()
+ |> all_supported()
+ |> Enum.uniq()
+ end
+
+ defp all_supported(locales) do
+ locales
+ |> Pleroma.Web.Gettext.ensure_fallbacks()
+ |> Enum.filter(&supported_locale?/1)
+ end
+
+ defp normalize_language_codes(codes) do
+ codes
+ |> Enum.map(fn code -> Pleroma.Web.Gettext.normalize_locale(code) end)
+ end
+
+ defp extract_preferred_language(conn) do
+ extract_frontend_language(conn) ++ extract_accept_language(conn)
+ end
+
+ defp extract_frontend_language(conn) do
+ %{req_cookies: cookies} =
+ conn
+ |> Plug.Conn.fetch_cookies()
+
+ case cookies[frontend_language_cookie_name()] do
+ nil ->
+ []
+
+ fe_lang ->
+ String.split(fe_lang, ",")
+ end
end
defp extract_accept_language(conn) do
@@ -29,7 +67,6 @@ defp extract_accept_language(conn) do
|> Enum.sort(&(&1.quality > &2.quality))
|> Enum.map(& &1.tag)
|> Enum.reject(&is_nil/1)
- |> ensure_language_fallbacks()
_ ->
[]
@@ -37,9 +74,7 @@ defp extract_accept_language(conn) do
end
defp supported_locale?(locale) do
- Pleroma.Web.Gettext
- |> Gettext.known_locales()
- |> Enum.member?(locale)
+ Pleroma.Web.Gettext.supports_locale?(locale)
end
defp parse_language_option(string) do
@@ -53,11 +88,4 @@ defp parse_language_option(string) do
%{tag: captures["tag"], quality: quality}
end
-
- defp ensure_language_fallbacks(tags) do
- Enum.flat_map(tags, fn tag ->
- [language | _] = String.split(tag, "-")
- if Enum.member?(tags, language), do: [tag], else: [tag, language]
- end)
- end
end
diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex
index 60eceff22..1efc76e1a 100644
--- a/lib/pleroma/web/templates/email/digest.html.eex
+++ b/lib/pleroma/web/templates/email/digest.html.eex
@@ -160,7 +160,7 @@
Hey <%= @user.nickname %>, here is what you've missed!
+ style="font-size: 30px; color: <%= @styling.header_color %>;"><%= Gettext.dpgettext("static_pages", "digest email header line", "Hey %{nickname}, here is what you've missed!", nickname: @user.nickname) %>
@@ -382,7 +382,7 @@
<%= length(@followers) %> New Followers <%= Gettext.dpngettext("static_pages", "new followers count header", "%{count} New Follower", "%{count} New Followers", length(@followers), count: length(@followers)) %>
@@ -535,16 +535,16 @@
style="color:<%= @styling.text_color %>;font-family:Arial, 'Helvetica Neue', Helvetica, sans-serif;line-height:120%;padding-top:10px;padding-right:10px;padding-bottom:10px;padding-left:10px;">
- You have received this email because you have signed up to receive digest emails from <%= @instance %> Pleroma instance.
+ <%= raw Gettext.dpgettext("static_pages", "digest email sending reason", "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance.", instance: safe_to_string(html_escape(@instance))) %>
- The email address you are subscribed as is <%= @user.email %> .
+ <%= raw Gettext.dpgettext("static_pages", "digest email receiver address", "The email address you are subscribed as is %{email} . ", color: safe_to_string(html_escape(@styling.link_color)), email: safe_to_string(html_escape(@user.email))) %>
- To unsubscribe, please go <%= link "here", style: "color: #{@styling.link_color};text-decoration: none;", to: @unsubscribe_link %>.
+ <%= raw Gettext.dpgettext("static_pages", "digest email unsubscribe action", "To unsubscribe, please go %{here}.", here: safe_to_string link(Gettext.dpgettext("static_pages", "digest email unsubscribe action link text", "here"), style: "color: #{@styling.link_color};text-decoration: none;", to: @unsubscribe_link)) %>
diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex
index de0731085..6d497e84c 100644
--- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex
@@ -1,6 +1,6 @@
-<%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %>
#<%= @tag %>
- These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse.
+ <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %>
<%= feed_logo() %>
<%= most_recent_update(@activities) %>
diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex
index 9c3613feb..edcc3e436 100644
--- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex
+++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex
@@ -4,7 +4,7 @@
#<%= @tag %>
- These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse.
+ <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %>
<%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %>
<%= feed_logo() %>
2b90d9
diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex
index 1ede59fd8..e33bada85 100644
--- a/lib/pleroma/web/templates/layout/app.html.eex
+++ b/lib/pleroma/web/templates/layout/app.html.eex
@@ -1,5 +1,5 @@
-
+
diff --git a/lib/pleroma/web/templates/layout/email.html.eex b/lib/pleroma/web/templates/layout/email.html.eex
index f6dcd7f0f..087aa4fc0 100644
--- a/lib/pleroma/web/templates/layout/email.html.eex
+++ b/lib/pleroma/web/templates/layout/email.html.eex
@@ -1,5 +1,5 @@
-
+
<%= @email.subject %>
@@ -7,4 +7,4 @@
<%= render @view_module, @view_template, assigns %>
-
\ No newline at end of file
+
diff --git a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex
index 7b476f02d..df090ffcd 100644
--- a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex
+++ b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex
@@ -1 +1 @@
-UNSUBSCRIBE FAILURE
+<%= Gettext.dpgettext("static_pages", "mailer unsubscribe failed message", "UNSUBSCRIBE FAILURE") %>
diff --git a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex
index 6dfa2c185..cbce495d4 100644
--- a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex
+++ b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex
@@ -1 +1 @@
-UNSUBSCRIBE SUCCESSFUL
+<%= Gettext.dpgettext("static_pages", "mailer unsubscribe successful message", "UNSUBSCRIBE SUCCESSFUL") %>
diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
index b9daa8d8b..e45d13bdf 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
@@ -5,11 +5,11 @@
<%= get_flash(@conn, :error) %>
<% end %>
-Two-factor recovery
+<%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %>
<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
- <%= label f, :code, "Recovery code" %>
+ <%= label f, :code, Gettext.dpgettext("static_pages", "mfa recover recovery code prompt", "Recovery code") %>
<%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %>
<%= hidden_input f, :mfa_token, value: @mfa_token %>
<%= hidden_input f, :state, value: @state %>
@@ -17,8 +17,8 @@
<%= hidden_input f, :challenge_type, value: "recovery" %>
-<%= submit "Verify" %>
+<%= submit Gettext.dpgettext("static_pages", "mfa recover verify recovery code button", "Verify") %>
<% end %>
">
- Enter a two-factor code
+ <%= Gettext.dpgettext("static_pages", "mfa recover use 2fa code link", "Enter a two-factor code") %>
diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
index 29ea7c5fb..50e6c04b6 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
@@ -5,20 +5,20 @@
<%= get_flash(@conn, :error) %>
<% end %>
-Two-factor authentication
+<%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %>
<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
- <%= label f, :code, "Authentication code" %>
- <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %>
+ <%= label f, :code, Gettext.dpgettext("static_pages", "mfa auth code prompt", "Authentication code") %>
+ <%= text_input f, :code, [autocomplete: "one-time-code", autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %>
<%= hidden_input f, :mfa_token, value: @mfa_token %>
<%= hidden_input f, :state, value: @state %>
<%= hidden_input f, :redirect_uri, value: @redirect_uri %>
<%= hidden_input f, :challenge_type, value: "totp" %>
-<%= submit "Verify" %>
+<%= submit Gettext.dpgettext("static_pages", "mfa auth verify code button", "Verify") %>
<% end %>
">
- Enter a two-factor recovery code
+ <%= Gettext.dpgettext("static_pages", "mfa auth page use recovery code link", "Enter a two-factor recovery code") %>
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex
index c9ec1ecbf..73115e92a 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex
@@ -1,5 +1,5 @@