From 2342fface1d1c13be27e3b3bdbb8d11d83403f02 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 29 Jun 2022 09:33:57 +0000 Subject: [PATCH 01/94] Add visibility check in context path (#26) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/26 --- .woodpecker/.release.yml | 8 +++-- .woodpecker/.test.yml | 2 ++ config/config.exs | 1 + .../controllers/status_controller.ex | 4 ++- .../controllers/status_controller_test.exs | 33 +++++++++++++++++++ 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml index 335f3c8e8..28043aa65 100644 --- a/.woodpecker/.release.yml +++ b/.woodpecker/.release.yml @@ -16,7 +16,9 @@ pipeline: glibc: when: event: - - tag + - push + branch: + - develop secrets: - SCW_ACCESS_KEY - SCW_SECRET_KEY @@ -44,7 +46,9 @@ pipeline: musl: when: event: - - tag + - push + branch: + - develop secrets: - SCW_ACCESS_KEY - SCW_SECRET_KEY diff --git a/.woodpecker/.test.yml b/.woodpecker/.test.yml index cef743643..6724d363d 100644 --- a/.woodpecker/.test.yml +++ b/.woodpecker/.test.yml @@ -11,6 +11,7 @@ pipeline: when: event: - push + - pull_request environment: MIX_ENV: test commands: @@ -25,6 +26,7 @@ pipeline: when: event: - push + - pull_request environment: MIX_ENV: test POSTGRES_DB: pleroma_test diff --git a/config/config.exs b/config/config.exs index ea0b23360..00f9af797 100644 --- a/config/config.exs +++ b/config/config.exs @@ -97,6 +97,7 @@ "http", "dat", "dweb", + "gopher", "hyper", "ipfs", "ipns", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 2eff4d9d0..60f4c44d7 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -384,11 +384,13 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{id: id}) do def context(%{assigns: %{user: user}} = conn, %{id: id}) do with %Activity{} = activity <- Activity.get_by_id(id) do activities = - ActivityPub.fetch_activities_for_context(activity.data["context"], %{ + activity.data["context"] + |> ActivityPub.fetch_activities_for_context(%{ blocking_user: user, user: user, exclude_id: activity.id }) + |> Enum.filter(fn activity -> Visibility.visible_for_user?(activity, user) end) render(conn, "context.json", activity: activity, activities: activities, user: user) end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index ed66d370a..3e0660031 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1810,6 +1810,39 @@ test "context" do } = response end + test "context when restrict_unauthenticated is on" do + user = insert(:user) + remote_user = insert(:user, local: false) + + {:ok, %{id: id1}} = CommonAPI.post(user, %{status: "1"}) + {:ok, %{id: id2}} = CommonAPI.post(user, %{status: "2", in_reply_to_status_id: id1}) + + {:ok, %{id: id3}} = + CommonAPI.post(remote_user, %{status: "3", in_reply_to_status_id: id2, local: false}) + + response = + build_conn() + |> get("/api/v1/statuses/#{id2}/context") + |> json_response_and_validate_schema(:ok) + + assert %{ + "ancestors" => [%{"id" => ^id1}], + "descendants" => [%{"id" => ^id3}] + } = response + + clear_config([:restrict_unauthenticated, :activities, :local], true) + + response = + build_conn() + |> get("/api/v1/statuses/#{id2}/context") + |> json_response_and_validate_schema(:ok) + + assert %{ + "ancestors" => [], + "descendants" => [] + } = response + end + test "favorites paginate correctly" do %{user: user, conn: conn} = oauth_access(["read:favourites"]) other_user = insert(:user) From 8de65de3b9ffe17b0ff2a507f5a747f4b8119010 Mon Sep 17 00:00:00 2001 From: Sam Therapy Date: Mon, 21 Feb 2022 12:28:33 -0600 Subject: [PATCH 02/94] Add unicode 14 support and add a test with a unicode 14 emoji --- lib/pleroma/emoji-test.txt | 156 +++++++++++++++++++++++++++++++----- test/pleroma/emoji_test.exs | 1 + 2 files changed, 135 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/emoji-test.txt b/lib/pleroma/emoji-test.txt index d3c6d12bd..dd5493366 100644 --- a/lib/pleroma/emoji-test.txt +++ b/lib/pleroma/emoji-test.txt @@ -1,11 +1,11 @@ # emoji-test.txt -# Date: 2020-09-12, 22:19:50 GMT -# ยฉ 2020 Unicodeยฎ, Inc. +# Date: 2021-08-26, 17:22:23 GMT +# ยฉ 2021 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Emoji Keyboard/Display Test Data for UTS #51 -# Version: 13.1 +# Version: 14.0 # # For documentation and usage, see http://www.unicode.org/reports/tr51 # @@ -43,6 +43,7 @@ 1F602 ; fully-qualified # ๐Ÿ˜‚ E0.6 face with tears of joy 1F642 ; fully-qualified # ๐Ÿ™‚ E1.0 slightly smiling face 1F643 ; fully-qualified # ๐Ÿ™ƒ E1.0 upside-down face +1FAE0 ; fully-qualified # ๐Ÿซ  E14.0 melting face 1F609 ; fully-qualified # ๐Ÿ˜‰ E0.6 winking face 1F60A ; fully-qualified # ๐Ÿ˜Š E0.6 smiling face with smiling eyes 1F607 ; fully-qualified # ๐Ÿ˜‡ E1.0 smiling face with halo @@ -68,10 +69,13 @@ 1F911 ; fully-qualified # ๐Ÿค‘ E1.0 money-mouth face # subgroup: face-hand -1F917 ; fully-qualified # ๐Ÿค— E1.0 hugging face +1F917 ; fully-qualified # ๐Ÿค— E1.0 smiling face with open hands 1F92D ; fully-qualified # ๐Ÿคญ E5.0 face with hand over mouth +1FAE2 ; fully-qualified # ๐Ÿซข E14.0 face with open eyes and hand over mouth +1FAE3 ; fully-qualified # ๐Ÿซฃ E14.0 face with peeking eye 1F92B ; fully-qualified # ๐Ÿคซ E5.0 shushing face 1F914 ; fully-qualified # ๐Ÿค” E1.0 thinking face +1FAE1 ; fully-qualified # ๐Ÿซก E14.0 saluting face # subgroup: face-neutral-skeptical 1F910 ; fully-qualified # ๐Ÿค E1.0 zipper-mouth face @@ -79,6 +83,7 @@ 1F610 ; fully-qualified # ๐Ÿ˜ E0.7 neutral face 1F611 ; fully-qualified # ๐Ÿ˜‘ E1.0 expressionless face 1F636 ; fully-qualified # ๐Ÿ˜ถ E1.0 face without mouth +1FAE5 ; fully-qualified # ๐Ÿซฅ E14.0 dotted line face 1F636 200D 1F32B FE0F ; fully-qualified # ๐Ÿ˜ถโ€๐ŸŒซ๏ธ E13.1 face in clouds 1F636 200D 1F32B ; minimally-qualified # ๐Ÿ˜ถโ€๐ŸŒซ E13.1 face in clouds 1F60F ; fully-qualified # ๐Ÿ˜ E0.6 smirking face @@ -105,7 +110,7 @@ 1F975 ; fully-qualified # ๐Ÿฅต E11.0 hot face 1F976 ; fully-qualified # ๐Ÿฅถ E11.0 cold face 1F974 ; fully-qualified # ๐Ÿฅด E11.0 woozy face -1F635 ; fully-qualified # ๐Ÿ˜ต E0.6 knocked-out face +1F635 ; fully-qualified # ๐Ÿ˜ต E0.6 face with crossed-out eyes 1F635 200D 1F4AB ; fully-qualified # ๐Ÿ˜ตโ€๐Ÿ’ซ E13.1 face with spiral eyes 1F92F ; fully-qualified # ๐Ÿคฏ E5.0 exploding head @@ -121,6 +126,7 @@ # subgroup: face-concerned 1F615 ; fully-qualified # ๐Ÿ˜• E1.0 confused face +1FAE4 ; fully-qualified # ๐Ÿซค E14.0 face with diagonal mouth 1F61F ; fully-qualified # ๐Ÿ˜Ÿ E1.0 worried face 1F641 ; fully-qualified # ๐Ÿ™ E1.0 slightly frowning face 2639 FE0F ; fully-qualified # โ˜น๏ธ E0.7 frowning face @@ -130,6 +136,7 @@ 1F632 ; fully-qualified # ๐Ÿ˜ฒ E0.6 astonished face 1F633 ; fully-qualified # ๐Ÿ˜ณ E0.6 flushed face 1F97A ; fully-qualified # ๐Ÿฅบ E11.0 pleading face +1F979 ; fully-qualified # ๐Ÿฅน E14.0 face holding back tears 1F626 ; fully-qualified # ๐Ÿ˜ฆ E1.0 frowning face with open mouth 1F627 ; fully-qualified # ๐Ÿ˜ง E1.0 anguished face 1F628 ; fully-qualified # ๐Ÿ˜จ E0.6 fearful face @@ -232,8 +239,8 @@ 1F4AD ; fully-qualified # ๐Ÿ’ญ E1.0 thought balloon 1F4A4 ; fully-qualified # ๐Ÿ’ค E0.6 zzz -# Smileys & Emotion subtotal: 170 -# Smileys & Emotion subtotal: 170 w/o modifiers +# Smileys & Emotion subtotal: 177 +# Smileys & Emotion subtotal: 177 w/o modifiers # group: People & Body @@ -269,6 +276,30 @@ 1F596 1F3FD ; fully-qualified # ๐Ÿ––๐Ÿฝ E1.0 vulcan salute: medium skin tone 1F596 1F3FE ; fully-qualified # ๐Ÿ––๐Ÿพ E1.0 vulcan salute: medium-dark skin tone 1F596 1F3FF ; fully-qualified # ๐Ÿ––๐Ÿฟ E1.0 vulcan salute: dark skin tone +1FAF1 ; fully-qualified # ๐Ÿซฑ E14.0 rightwards hand +1FAF1 1F3FB ; fully-qualified # ๐Ÿซฑ๐Ÿป E14.0 rightwards hand: light skin tone +1FAF1 1F3FC ; fully-qualified # ๐Ÿซฑ๐Ÿผ E14.0 rightwards hand: medium-light skin tone +1FAF1 1F3FD ; fully-qualified # ๐Ÿซฑ๐Ÿฝ E14.0 rightwards hand: medium skin tone +1FAF1 1F3FE ; fully-qualified # ๐Ÿซฑ๐Ÿพ E14.0 rightwards hand: medium-dark skin tone +1FAF1 1F3FF ; fully-qualified # ๐Ÿซฑ๐Ÿฟ E14.0 rightwards hand: dark skin tone +1FAF2 ; fully-qualified # ๐Ÿซฒ E14.0 leftwards hand +1FAF2 1F3FB ; fully-qualified # ๐Ÿซฒ๐Ÿป E14.0 leftwards hand: light skin tone +1FAF2 1F3FC ; fully-qualified # ๐Ÿซฒ๐Ÿผ E14.0 leftwards hand: medium-light skin tone +1FAF2 1F3FD ; fully-qualified # ๐Ÿซฒ๐Ÿฝ E14.0 leftwards hand: medium skin tone +1FAF2 1F3FE ; fully-qualified # ๐Ÿซฒ๐Ÿพ E14.0 leftwards hand: medium-dark skin tone +1FAF2 1F3FF ; fully-qualified # ๐Ÿซฒ๐Ÿฟ E14.0 leftwards hand: dark skin tone +1FAF3 ; fully-qualified # ๐Ÿซณ E14.0 palm down hand +1FAF3 1F3FB ; fully-qualified # ๐Ÿซณ๐Ÿป E14.0 palm down hand: light skin tone +1FAF3 1F3FC ; fully-qualified # ๐Ÿซณ๐Ÿผ E14.0 palm down hand: medium-light skin tone +1FAF3 1F3FD ; fully-qualified # ๐Ÿซณ๐Ÿฝ E14.0 palm down hand: medium skin tone +1FAF3 1F3FE ; fully-qualified # ๐Ÿซณ๐Ÿพ E14.0 palm down hand: medium-dark skin tone +1FAF3 1F3FF ; fully-qualified # ๐Ÿซณ๐Ÿฟ E14.0 palm down hand: dark skin tone +1FAF4 ; fully-qualified # ๐Ÿซด E14.0 palm up hand +1FAF4 1F3FB ; fully-qualified # ๐Ÿซด๐Ÿป E14.0 palm up hand: light skin tone +1FAF4 1F3FC ; fully-qualified # ๐Ÿซด๐Ÿผ E14.0 palm up hand: medium-light skin tone +1FAF4 1F3FD ; fully-qualified # ๐Ÿซด๐Ÿฝ E14.0 palm up hand: medium skin tone +1FAF4 1F3FE ; fully-qualified # ๐Ÿซด๐Ÿพ E14.0 palm up hand: medium-dark skin tone +1FAF4 1F3FF ; fully-qualified # ๐Ÿซด๐Ÿฟ E14.0 palm up hand: dark skin tone # subgroup: hand-fingers-partial 1F44C ; fully-qualified # ๐Ÿ‘Œ E0.6 OK hand @@ -302,6 +333,12 @@ 1F91E 1F3FD ; fully-qualified # ๐Ÿคž๐Ÿฝ E3.0 crossed fingers: medium skin tone 1F91E 1F3FE ; fully-qualified # ๐Ÿคž๐Ÿพ E3.0 crossed fingers: medium-dark skin tone 1F91E 1F3FF ; fully-qualified # ๐Ÿคž๐Ÿฟ E3.0 crossed fingers: dark skin tone +1FAF0 ; fully-qualified # ๐Ÿซฐ E14.0 hand with index finger and thumb crossed +1FAF0 1F3FB ; fully-qualified # ๐Ÿซฐ๐Ÿป E14.0 hand with index finger and thumb crossed: light skin tone +1FAF0 1F3FC ; fully-qualified # ๐Ÿซฐ๐Ÿผ E14.0 hand with index finger and thumb crossed: medium-light skin tone +1FAF0 1F3FD ; fully-qualified # ๐Ÿซฐ๐Ÿฝ E14.0 hand with index finger and thumb crossed: medium skin tone +1FAF0 1F3FE ; fully-qualified # ๐Ÿซฐ๐Ÿพ E14.0 hand with index finger and thumb crossed: medium-dark skin tone +1FAF0 1F3FF ; fully-qualified # ๐Ÿซฐ๐Ÿฟ E14.0 hand with index finger and thumb crossed: dark skin tone 1F91F ; fully-qualified # ๐ŸคŸ E5.0 love-you gesture 1F91F 1F3FB ; fully-qualified # ๐ŸคŸ๐Ÿป E5.0 love-you gesture: light skin tone 1F91F 1F3FC ; fully-qualified # ๐ŸคŸ๐Ÿผ E5.0 love-you gesture: medium-light skin tone @@ -359,6 +396,12 @@ 261D 1F3FD ; fully-qualified # โ˜๐Ÿฝ E1.0 index pointing up: medium skin tone 261D 1F3FE ; fully-qualified # โ˜๐Ÿพ E1.0 index pointing up: medium-dark skin tone 261D 1F3FF ; fully-qualified # โ˜๐Ÿฟ E1.0 index pointing up: dark skin tone +1FAF5 ; fully-qualified # ๐Ÿซต E14.0 index pointing at the viewer +1FAF5 1F3FB ; fully-qualified # ๐Ÿซต๐Ÿป E14.0 index pointing at the viewer: light skin tone +1FAF5 1F3FC ; fully-qualified # ๐Ÿซต๐Ÿผ E14.0 index pointing at the viewer: medium-light skin tone +1FAF5 1F3FD ; fully-qualified # ๐Ÿซต๐Ÿฝ E14.0 index pointing at the viewer: medium skin tone +1FAF5 1F3FE ; fully-qualified # ๐Ÿซต๐Ÿพ E14.0 index pointing at the viewer: medium-dark skin tone +1FAF5 1F3FF ; fully-qualified # ๐Ÿซต๐Ÿฟ E14.0 index pointing at the viewer: dark skin tone # subgroup: hand-fingers-closed 1F44D ; fully-qualified # ๐Ÿ‘ E0.6 thumbs up @@ -411,6 +454,12 @@ 1F64C 1F3FD ; fully-qualified # ๐Ÿ™Œ๐Ÿฝ E1.0 raising hands: medium skin tone 1F64C 1F3FE ; fully-qualified # ๐Ÿ™Œ๐Ÿพ E1.0 raising hands: medium-dark skin tone 1F64C 1F3FF ; fully-qualified # ๐Ÿ™Œ๐Ÿฟ E1.0 raising hands: dark skin tone +1FAF6 ; fully-qualified # ๐Ÿซถ E14.0 heart hands +1FAF6 1F3FB ; fully-qualified # ๐Ÿซถ๐Ÿป E14.0 heart hands: light skin tone +1FAF6 1F3FC ; fully-qualified # ๐Ÿซถ๐Ÿผ E14.0 heart hands: medium-light skin tone +1FAF6 1F3FD ; fully-qualified # ๐Ÿซถ๐Ÿฝ E14.0 heart hands: medium skin tone +1FAF6 1F3FE ; fully-qualified # ๐Ÿซถ๐Ÿพ E14.0 heart hands: medium-dark skin tone +1FAF6 1F3FF ; fully-qualified # ๐Ÿซถ๐Ÿฟ E14.0 heart hands: dark skin tone 1F450 ; fully-qualified # ๐Ÿ‘ E0.6 open hands 1F450 1F3FB ; fully-qualified # ๐Ÿ‘๐Ÿป E1.0 open hands: light skin tone 1F450 1F3FC ; fully-qualified # ๐Ÿ‘๐Ÿผ E1.0 open hands: medium-light skin tone @@ -424,6 +473,31 @@ 1F932 1F3FE ; fully-qualified # ๐Ÿคฒ๐Ÿพ E5.0 palms up together: medium-dark skin tone 1F932 1F3FF ; fully-qualified # ๐Ÿคฒ๐Ÿฟ E5.0 palms up together: dark skin tone 1F91D ; fully-qualified # ๐Ÿค E3.0 handshake +1F91D 1F3FB ; fully-qualified # ๐Ÿค๐Ÿป E3.0 handshake: light skin tone +1F91D 1F3FC ; fully-qualified # ๐Ÿค๐Ÿผ E3.0 handshake: medium-light skin tone +1F91D 1F3FD ; fully-qualified # ๐Ÿค๐Ÿฝ E3.0 handshake: medium skin tone +1F91D 1F3FE ; fully-qualified # ๐Ÿค๐Ÿพ E3.0 handshake: medium-dark skin tone +1F91D 1F3FF ; fully-qualified # ๐Ÿค๐Ÿฟ E3.0 handshake: dark skin tone +1FAF1 1F3FB 200D 1FAF2 1F3FC ; fully-qualified # ๐Ÿซฑ๐Ÿปโ€๐Ÿซฒ๐Ÿผ E14.0 handshake: light skin tone, medium-light skin tone +1FAF1 1F3FB 200D 1FAF2 1F3FD ; fully-qualified # ๐Ÿซฑ๐Ÿปโ€๐Ÿซฒ๐Ÿฝ E14.0 handshake: light skin tone, medium skin tone +1FAF1 1F3FB 200D 1FAF2 1F3FE ; fully-qualified # ๐Ÿซฑ๐Ÿปโ€๐Ÿซฒ๐Ÿพ E14.0 handshake: light skin tone, medium-dark skin tone +1FAF1 1F3FB 200D 1FAF2 1F3FF ; fully-qualified # ๐Ÿซฑ๐Ÿปโ€๐Ÿซฒ๐Ÿฟ E14.0 handshake: light skin tone, dark skin tone +1FAF1 1F3FC 200D 1FAF2 1F3FB ; fully-qualified # ๐Ÿซฑ๐Ÿผโ€๐Ÿซฒ๐Ÿป E14.0 handshake: medium-light skin tone, light skin tone +1FAF1 1F3FC 200D 1FAF2 1F3FD ; fully-qualified # ๐Ÿซฑ๐Ÿผโ€๐Ÿซฒ๐Ÿฝ E14.0 handshake: medium-light skin tone, medium skin tone +1FAF1 1F3FC 200D 1FAF2 1F3FE ; fully-qualified # ๐Ÿซฑ๐Ÿผโ€๐Ÿซฒ๐Ÿพ E14.0 handshake: medium-light skin tone, medium-dark skin tone +1FAF1 1F3FC 200D 1FAF2 1F3FF ; fully-qualified # ๐Ÿซฑ๐Ÿผโ€๐Ÿซฒ๐Ÿฟ E14.0 handshake: medium-light skin tone, dark skin tone +1FAF1 1F3FD 200D 1FAF2 1F3FB ; fully-qualified # ๐Ÿซฑ๐Ÿฝโ€๐Ÿซฒ๐Ÿป E14.0 handshake: medium skin tone, light skin tone +1FAF1 1F3FD 200D 1FAF2 1F3FC ; fully-qualified # ๐Ÿซฑ๐Ÿฝโ€๐Ÿซฒ๐Ÿผ E14.0 handshake: medium skin tone, medium-light skin tone +1FAF1 1F3FD 200D 1FAF2 1F3FE ; fully-qualified # ๐Ÿซฑ๐Ÿฝโ€๐Ÿซฒ๐Ÿพ E14.0 handshake: medium skin tone, medium-dark skin tone +1FAF1 1F3FD 200D 1FAF2 1F3FF ; fully-qualified # ๐Ÿซฑ๐Ÿฝโ€๐Ÿซฒ๐Ÿฟ E14.0 handshake: medium skin tone, dark skin tone +1FAF1 1F3FE 200D 1FAF2 1F3FB ; fully-qualified # ๐Ÿซฑ๐Ÿพโ€๐Ÿซฒ๐Ÿป E14.0 handshake: medium-dark skin tone, light skin tone +1FAF1 1F3FE 200D 1FAF2 1F3FC ; fully-qualified # ๐Ÿซฑ๐Ÿพโ€๐Ÿซฒ๐Ÿผ E14.0 handshake: medium-dark skin tone, medium-light skin tone +1FAF1 1F3FE 200D 1FAF2 1F3FD ; fully-qualified # ๐Ÿซฑ๐Ÿพโ€๐Ÿซฒ๐Ÿฝ E14.0 handshake: medium-dark skin tone, medium skin tone +1FAF1 1F3FE 200D 1FAF2 1F3FF ; fully-qualified # ๐Ÿซฑ๐Ÿพโ€๐Ÿซฒ๐Ÿฟ E14.0 handshake: medium-dark skin tone, dark skin tone +1FAF1 1F3FF 200D 1FAF2 1F3FB ; fully-qualified # ๐Ÿซฑ๐Ÿฟโ€๐Ÿซฒ๐Ÿป E14.0 handshake: dark skin tone, light skin tone +1FAF1 1F3FF 200D 1FAF2 1F3FC ; fully-qualified # ๐Ÿซฑ๐Ÿฟโ€๐Ÿซฒ๐Ÿผ E14.0 handshake: dark skin tone, medium-light skin tone +1FAF1 1F3FF 200D 1FAF2 1F3FD ; fully-qualified # ๐Ÿซฑ๐Ÿฟโ€๐Ÿซฒ๐Ÿฝ E14.0 handshake: dark skin tone, medium skin tone +1FAF1 1F3FF 200D 1FAF2 1F3FE ; fully-qualified # ๐Ÿซฑ๐Ÿฟโ€๐Ÿซฒ๐Ÿพ E14.0 handshake: dark skin tone, medium-dark skin tone 1F64F ; fully-qualified # ๐Ÿ™ E0.6 folded hands 1F64F 1F3FB ; fully-qualified # ๐Ÿ™๐Ÿป E1.0 folded hands: light skin tone 1F64F 1F3FC ; fully-qualified # ๐Ÿ™๐Ÿผ E1.0 folded hands: medium-light skin tone @@ -501,6 +575,7 @@ 1F441 ; unqualified # ๐Ÿ‘ E0.7 eye 1F445 ; fully-qualified # ๐Ÿ‘… E0.6 tongue 1F444 ; fully-qualified # ๐Ÿ‘„ E0.6 mouth +1FAE6 ; fully-qualified # ๐Ÿซฆ E14.0 biting lip # subgroup: person 1F476 ; fully-qualified # ๐Ÿ‘ถ E0.6 baby @@ -1472,6 +1547,12 @@ 1F477 1F3FE 200D 2640 ; minimally-qualified # ๐Ÿ‘ท๐Ÿพโ€โ™€ E4.0 woman construction worker: medium-dark skin tone 1F477 1F3FF 200D 2640 FE0F ; fully-qualified # ๐Ÿ‘ท๐Ÿฟโ€โ™€๏ธ E4.0 woman construction worker: dark skin tone 1F477 1F3FF 200D 2640 ; minimally-qualified # ๐Ÿ‘ท๐Ÿฟโ€โ™€ E4.0 woman construction worker: dark skin tone +1FAC5 ; fully-qualified # ๐Ÿซ… E14.0 person with crown +1FAC5 1F3FB ; fully-qualified # ๐Ÿซ…๐Ÿป E14.0 person with crown: light skin tone +1FAC5 1F3FC ; fully-qualified # ๐Ÿซ…๐Ÿผ E14.0 person with crown: medium-light skin tone +1FAC5 1F3FD ; fully-qualified # ๐Ÿซ…๐Ÿฝ E14.0 person with crown: medium skin tone +1FAC5 1F3FE ; fully-qualified # ๐Ÿซ…๐Ÿพ E14.0 person with crown: medium-dark skin tone +1FAC5 1F3FF ; fully-qualified # ๐Ÿซ…๐Ÿฟ E14.0 person with crown: dark skin tone 1F934 ; fully-qualified # ๐Ÿคด E3.0 prince 1F934 1F3FB ; fully-qualified # ๐Ÿคด๐Ÿป E3.0 prince: light skin tone 1F934 1F3FC ; fully-qualified # ๐Ÿคด๐Ÿผ E3.0 prince: medium-light skin tone @@ -1592,6 +1673,18 @@ 1F930 1F3FD ; fully-qualified # ๐Ÿคฐ๐Ÿฝ E3.0 pregnant woman: medium skin tone 1F930 1F3FE ; fully-qualified # ๐Ÿคฐ๐Ÿพ E3.0 pregnant woman: medium-dark skin tone 1F930 1F3FF ; fully-qualified # ๐Ÿคฐ๐Ÿฟ E3.0 pregnant woman: dark skin tone +1FAC3 ; fully-qualified # ๐Ÿซƒ E14.0 pregnant man +1FAC3 1F3FB ; fully-qualified # ๐Ÿซƒ๐Ÿป E14.0 pregnant man: light skin tone +1FAC3 1F3FC ; fully-qualified # ๐Ÿซƒ๐Ÿผ E14.0 pregnant man: medium-light skin tone +1FAC3 1F3FD ; fully-qualified # ๐Ÿซƒ๐Ÿฝ E14.0 pregnant man: medium skin tone +1FAC3 1F3FE ; fully-qualified # ๐Ÿซƒ๐Ÿพ E14.0 pregnant man: medium-dark skin tone +1FAC3 1F3FF ; fully-qualified # ๐Ÿซƒ๐Ÿฟ E14.0 pregnant man: dark skin tone +1FAC4 ; fully-qualified # ๐Ÿซ„ E14.0 pregnant person +1FAC4 1F3FB ; fully-qualified # ๐Ÿซ„๐Ÿป E14.0 pregnant person: light skin tone +1FAC4 1F3FC ; fully-qualified # ๐Ÿซ„๐Ÿผ E14.0 pregnant person: medium-light skin tone +1FAC4 1F3FD ; fully-qualified # ๐Ÿซ„๐Ÿฝ E14.0 pregnant person: medium skin tone +1FAC4 1F3FE ; fully-qualified # ๐Ÿซ„๐Ÿพ E14.0 pregnant person: medium-dark skin tone +1FAC4 1F3FF ; fully-qualified # ๐Ÿซ„๐Ÿฟ E14.0 pregnant person: dark skin tone 1F931 ; fully-qualified # ๐Ÿคฑ E5.0 breast-feeding 1F931 1F3FB ; fully-qualified # ๐Ÿคฑ๐Ÿป E5.0 breast-feeding: light skin tone 1F931 1F3FC ; fully-qualified # ๐Ÿคฑ๐Ÿผ E5.0 breast-feeding: medium-light skin tone @@ -1862,6 +1955,7 @@ 1F9DF 200D 2642 ; minimally-qualified # ๐ŸงŸโ€โ™‚ E5.0 man zombie 1F9DF 200D 2640 FE0F ; fully-qualified # ๐ŸงŸโ€โ™€๏ธ E5.0 woman zombie 1F9DF 200D 2640 ; minimally-qualified # ๐ŸงŸโ€โ™€ E5.0 woman zombie +1F9CC ; fully-qualified # ๐ŸงŒ E14.0 troll # subgroup: person-activity 1F486 ; fully-qualified # ๐Ÿ’† E0.6 person getting massage @@ -3168,8 +3262,8 @@ 1FAC2 ; fully-qualified # ๐Ÿซ‚ E13.0 people hugging 1F463 ; fully-qualified # ๐Ÿ‘ฃ E0.6 footprints -# People & Body subtotal: 2899 -# People & Body subtotal: 494 w/o modifiers +# People & Body subtotal: 2986 +# People & Body subtotal: 506 w/o modifiers # group: Component @@ -3304,6 +3398,7 @@ 1F988 ; fully-qualified # ๐Ÿฆˆ E3.0 shark 1F419 ; fully-qualified # ๐Ÿ™ E0.6 octopus 1F41A ; fully-qualified # ๐Ÿš E0.6 spiral shell +1FAB8 ; fully-qualified # ๐Ÿชธ E14.0 coral # subgroup: animal-bug 1F40C ; fully-qualified # ๐ŸŒ E0.6 snail @@ -3329,6 +3424,7 @@ 1F490 ; fully-qualified # ๐Ÿ’ E0.6 bouquet 1F338 ; fully-qualified # ๐ŸŒธ E0.6 cherry blossom 1F4AE ; fully-qualified # ๐Ÿ’ฎ E0.6 white flower +1FAB7 ; fully-qualified # ๐Ÿชท E14.0 lotus 1F3F5 FE0F ; fully-qualified # ๐Ÿต๏ธ E0.7 rosette 1F3F5 ; unqualified # ๐Ÿต E0.7 rosette 1F339 ; fully-qualified # ๐ŸŒน E0.6 rose @@ -3353,9 +3449,11 @@ 1F341 ; fully-qualified # ๐Ÿ E0.6 maple leaf 1F342 ; fully-qualified # ๐Ÿ‚ E0.6 fallen leaf 1F343 ; fully-qualified # ๐Ÿƒ E0.6 leaf fluttering in wind +1FAB9 ; fully-qualified # ๐Ÿชน E14.0 empty nest +1FABA ; fully-qualified # ๐Ÿชบ E14.0 nest with eggs -# Animals & Nature subtotal: 147 -# Animals & Nature subtotal: 147 w/o modifiers +# Animals & Nature subtotal: 151 +# Animals & Nature subtotal: 151 w/o modifiers # group: Food & Drink @@ -3396,6 +3494,7 @@ 1F9C5 ; fully-qualified # ๐Ÿง… E12.0 onion 1F344 ; fully-qualified # ๐Ÿ„ E0.6 mushroom 1F95C ; fully-qualified # ๐Ÿฅœ E3.0 peanuts +1FAD8 ; fully-qualified # ๐Ÿซ˜ E14.0 beans 1F330 ; fully-qualified # ๐ŸŒฐ E0.6 chestnut # subgroup: food-prepared @@ -3491,6 +3590,7 @@ 1F37B ; fully-qualified # ๐Ÿป E0.6 clinking beer mugs 1F942 ; fully-qualified # ๐Ÿฅ‚ E3.0 clinking glasses 1F943 ; fully-qualified # ๐Ÿฅƒ E3.0 tumbler glass +1FAD7 ; fully-qualified # ๐Ÿซ— E14.0 pouring liquid 1F964 ; fully-qualified # ๐Ÿฅค E5.0 cup with straw 1F9CB ; fully-qualified # ๐Ÿง‹ E13.0 bubble tea 1F9C3 ; fully-qualified # ๐Ÿงƒ E12.0 beverage box @@ -3504,10 +3604,11 @@ 1F374 ; fully-qualified # ๐Ÿด E0.6 fork and knife 1F944 ; fully-qualified # ๐Ÿฅ„ E3.0 spoon 1F52A ; fully-qualified # ๐Ÿ”ช E0.6 kitchen knife +1FAD9 ; fully-qualified # ๐Ÿซ™ E14.0 jar 1F3FA ; fully-qualified # ๐Ÿบ E1.0 amphora -# Food & Drink subtotal: 131 -# Food & Drink subtotal: 131 w/o modifiers +# Food & Drink subtotal: 134 +# Food & Drink subtotal: 134 w/o modifiers # group: Travel & Places @@ -3597,6 +3698,7 @@ 2668 FE0F ; fully-qualified # โ™จ๏ธ E0.6 hot springs 2668 ; unqualified # โ™จ E0.6 hot springs 1F3A0 ; fully-qualified # ๐ŸŽ  E0.6 carousel horse +1F6DD ; fully-qualified # ๐Ÿ› E14.0 playground slide 1F3A1 ; fully-qualified # ๐ŸŽก E0.6 ferris wheel 1F3A2 ; fully-qualified # ๐ŸŽข E0.6 roller coaster 1F488 ; fully-qualified # ๐Ÿ’ˆ E0.6 barber pole @@ -3652,6 +3754,7 @@ 1F6E2 FE0F ; fully-qualified # ๐Ÿ›ข๏ธ E0.7 oil drum 1F6E2 ; unqualified # ๐Ÿ›ข E0.7 oil drum 26FD ; fully-qualified # โ›ฝ E0.6 fuel pump +1F6DE ; fully-qualified # ๐Ÿ›ž E14.0 wheel 1F6A8 ; fully-qualified # ๐Ÿšจ E0.6 police car light 1F6A5 ; fully-qualified # ๐Ÿšฅ E0.6 horizontal traffic light 1F6A6 ; fully-qualified # ๐Ÿšฆ E1.0 vertical traffic light @@ -3660,6 +3763,7 @@ # subgroup: transport-water 2693 ; fully-qualified # โš“ E0.6 anchor +1F6DF ; fully-qualified # ๐Ÿ›Ÿ E14.0 ring buoy 26F5 ; fully-qualified # โ›ต E0.6 sailboat 1F6F6 ; fully-qualified # ๐Ÿ›ถ E3.0 canoe 1F6A4 ; fully-qualified # ๐Ÿšค E0.6 speedboat @@ -3797,8 +3901,8 @@ 1F4A7 ; fully-qualified # ๐Ÿ’ง E0.6 droplet 1F30A ; fully-qualified # ๐ŸŒŠ E0.6 water wave -# Travel & Places subtotal: 264 -# Travel & Places subtotal: 264 w/o modifiers +# Travel & Places subtotal: 267 +# Travel & Places subtotal: 267 w/o modifiers # group: Activities @@ -3874,6 +3978,7 @@ 1F52E ; fully-qualified # ๐Ÿ”ฎ E0.6 crystal ball 1FA84 ; fully-qualified # ๐Ÿช„ E13.0 magic wand 1F9FF ; fully-qualified # ๐Ÿงฟ E11.0 nazar amulet +1FAAC ; fully-qualified # ๐Ÿชฌ E14.0 hamsa 1F3AE ; fully-qualified # ๐ŸŽฎ E0.6 video game 1F579 FE0F ; fully-qualified # ๐Ÿ•น๏ธ E0.7 joystick 1F579 ; unqualified # ๐Ÿ•น E0.7 joystick @@ -3882,6 +3987,7 @@ 1F9E9 ; fully-qualified # ๐Ÿงฉ E11.0 puzzle piece 1F9F8 ; fully-qualified # ๐Ÿงธ E11.0 teddy bear 1FA85 ; fully-qualified # ๐Ÿช… E13.0 piรฑata +1FAA9 ; fully-qualified # ๐Ÿชฉ E14.0 mirror ball 1FA86 ; fully-qualified # ๐Ÿช† E13.0 nesting dolls 2660 FE0F ; fully-qualified # โ™ ๏ธ E0.6 spade suit 2660 ; unqualified # โ™  E0.6 spade suit @@ -3907,8 +4013,8 @@ 1F9F6 ; fully-qualified # ๐Ÿงถ E11.0 yarn 1FAA2 ; fully-qualified # ๐Ÿชข E13.0 knot -# Activities subtotal: 95 -# Activities subtotal: 95 w/o modifiers +# Activities subtotal: 97 +# Activities subtotal: 97 w/o modifiers # group: Objects @@ -4009,6 +4115,7 @@ # subgroup: computer 1F50B ; fully-qualified # ๐Ÿ”‹ E0.6 battery +1FAAB ; fully-qualified # ๐Ÿชซ E14.0 low battery 1F50C ; fully-qualified # ๐Ÿ”Œ E0.6 electric plug 1F4BB ; fully-qualified # ๐Ÿ’ป E0.6 laptop 1F5A5 FE0F ; fully-qualified # ๐Ÿ–ฅ๏ธ E0.7 desktop computer @@ -4207,7 +4314,9 @@ 1FA78 ; fully-qualified # ๐Ÿฉธ E12.0 drop of blood 1F48A ; fully-qualified # ๐Ÿ’Š E0.6 pill 1FA79 ; fully-qualified # ๐Ÿฉน E12.0 adhesive bandage +1FA7C ; fully-qualified # ๐Ÿฉผ E14.0 crutch 1FA7A ; fully-qualified # ๐Ÿฉบ E12.0 stethoscope +1FA7B ; fully-qualified # ๐Ÿฉป E14.0 x-ray # subgroup: household 1F6AA ; fully-qualified # ๐Ÿšช E0.6 door @@ -4232,6 +4341,7 @@ 1F9FB ; fully-qualified # ๐Ÿงป E11.0 roll of paper 1FAA3 ; fully-qualified # ๐Ÿชฃ E13.0 bucket 1F9FC ; fully-qualified # ๐Ÿงผ E11.0 soap +1FAE7 ; fully-qualified # ๐Ÿซง E14.0 bubbles 1FAA5 ; fully-qualified # ๐Ÿชฅ E13.0 toothbrush 1F9FD ; fully-qualified # ๐Ÿงฝ E11.0 sponge 1F9EF ; fully-qualified # ๐Ÿงฏ E11.0 fire extinguisher @@ -4246,9 +4356,10 @@ 26B1 ; unqualified # โšฑ E1.0 funeral urn 1F5FF ; fully-qualified # ๐Ÿ—ฟ E0.6 moai 1FAA7 ; fully-qualified # ๐Ÿชง E13.0 placard +1FAAA ; fully-qualified # ๐Ÿชช E14.0 identification card -# Objects subtotal: 299 -# Objects subtotal: 299 w/o modifiers +# Objects subtotal: 304 +# Objects subtotal: 304 w/o modifiers # group: Symbols @@ -4409,6 +4520,7 @@ 2795 ; fully-qualified # โž• E0.6 plus 2796 ; fully-qualified # โž– E0.6 minus 2797 ; fully-qualified # โž— E0.6 divide +1F7F0 ; fully-qualified # ๐ŸŸฐ E14.0 heavy equals sign 267E FE0F ; fully-qualified # โ™พ๏ธ E11.0 infinity 267E ; unqualified # โ™พ E11.0 infinity @@ -4581,8 +4693,8 @@ 1F533 ; fully-qualified # ๐Ÿ”ณ E0.6 white square button 1F532 ; fully-qualified # ๐Ÿ”ฒ E0.6 black square button -# Symbols subtotal: 301 -# Symbols subtotal: 301 w/o modifiers +# Symbols subtotal: 302 +# Symbols subtotal: 302 w/o modifiers # group: Flags @@ -4871,7 +4983,7 @@ # Flags subtotal: 275 w/o modifiers # Status Counts -# fully-qualified : 3512 +# fully-qualified : 3624 # minimally-qualified : 817 # unqualified : 252 # component : 9 diff --git a/test/pleroma/emoji_test.exs b/test/pleroma/emoji_test.exs index fe7fd111c..978473b14 100644 --- a/test/pleroma/emoji_test.exs +++ b/test/pleroma/emoji_test.exs @@ -20,6 +20,7 @@ test "tells if a string is an unicode emoji" do assert Emoji.is_unicode_emoji?("๐Ÿคฐ") assert Emoji.is_unicode_emoji?("โค๏ธ") assert Emoji.is_unicode_emoji?("๐Ÿณ๏ธโ€โšง๏ธ") + assert Emoji.is_unicode_emoji?("๐Ÿซต") # Additionally, we accept regional indicators. assert Emoji.is_unicode_emoji?("๐Ÿ‡ต") From 8b843be03ef5c2c15e8f43850810ee4c4cfc8587 Mon Sep 17 00:00:00 2001 From: Ilja Date: Tue, 22 Feb 2022 18:56:26 +0100 Subject: [PATCH 03/94] Fix test get_user_apps/1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason I had a test who suddenly failed, mix test test/pleroma/web/o_auth/app_test.exs:54. A user has a list of applications and this test adds them and then sees if the list it gets back is the same as the apps it added. When I ran mix test a day before I didn't have this problem and when I pushed code today in a different MR, the pipeline succeeded (see https://git.pleroma.social/ilja/pleroma/-/jobs/205827), yet locally it failed. So it seems the test can sometimes succeed and sometimes fail, which makes it untrustworthy. The failure I see is because the returned list is in reverse order. I assume that's not per sรฉ wrong. You just want to know if the apps you added are actually there. I fixed the test by first ordering the lists before comparing. AFAICT (and as far as that's relevant) the test got introduced in commit cb2a072e6252b7c3f6473f7cfd1af5c0ec732d7b --- test/pleroma/web/o_auth/app_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/o_auth/app_test.exs b/test/pleroma/web/o_auth/app_test.exs index a5223b0a5..3c5ca07ae 100644 --- a/test/pleroma/web/o_auth/app_test.exs +++ b/test/pleroma/web/o_auth/app_test.exs @@ -51,6 +51,6 @@ test "get_user_apps/1" do insert(:oauth_app, user_id: user.id) ] - assert App.get_user_apps(user) == apps + assert Enum.sort(App.get_user_apps(user)) == Enum.sort(apps) end end From 91d71500319255b0de5b18e8977d5e17fb1dda61 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 25 Feb 2022 10:31:42 +0100 Subject: [PATCH 04/94] mix: Check .git presence --- mix.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 8df00154e..76e84c2b8 100644 --- a/mix.exs +++ b/mix.exs @@ -248,9 +248,10 @@ defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"])) + dotgit_present? = File.exists?(".git") git_pre_release = - if git_available? do + if git_available? and dotgit_present? do {tag, tag_err} = System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) @@ -277,6 +278,7 @@ defp version(version) do # Branch name as pre-release version component, denoted with a dot branch_name = with true <- git_available?, + true <- dotgit_present?, {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, From 1048bc1bb9ec11766cff3ab3d75219f55b84f980 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sun, 6 Mar 2022 17:36:30 +0100 Subject: [PATCH 05/94] Delete report notifs when demoting from superuser When someone isn't a superuser any more, they shouldn't see the reporsts any more either. Here we delete the report notifications from a user when that user gets updated from being a superuser to a non-superuser. --- lib/pleroma/notification.ex | 8 ++++++++ lib/pleroma/user.ex | 19 ++++++++++++++++++- test/pleroma/notification_test.exs | 19 +++++++++++++++++++ test/pleroma/user_test.exs | 21 +++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 9e0ce0329..2ab09495d 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -341,6 +341,14 @@ def destroy_multiple(%{id: user_id} = _user, ids) do |> Repo.delete_all() end + def destroy_multiple_from_types(%{id: user_id}, types) do + from(n in Notification, + where: n.user_id == ^user_id, + where: n.type in ^types + ) + |> Repo.delete_all() + end + def dismiss(%Pleroma.Activity{} = activity) do Notification |> where([n], n.activity_id == ^activity.id) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index efe9ec5d6..809524f56 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1089,11 +1089,28 @@ def update_and_set_cache(struct, params) do |> update_and_set_cache() end - def update_and_set_cache(changeset) do + def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do + was_superuser_before_update = User.superuser?(user) + with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user) set_cache(user) end + |> maybe_remove_report_notifications(was_superuser_before_update) + end + + defp maybe_remove_report_notifications( + {:ok, %Pleroma.User{} = user} = result, + was_superuser_before_update + ) do + if was_superuser_before_update and not User.superuser?(user), + do: user |> Notification.destroy_multiple_from_types(["pleroma:report"]) + + result + end + + defp maybe_remove_report_notifications(result, _) do + result end def get_user_friends_ap_ids(user) do diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 716af496d..b47edd0a3 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -520,6 +520,25 @@ test "it clears all notifications belonging to the user" do end end + describe "destroy_multiple_from_types/2" do + test "clears all notifications of a certain type for a given user" do + report_activity = insert(:report_activity) + user1 = insert(:user, is_moderator: true, is_admin: true) + user2 = insert(:user, is_moderator: true, is_admin: true) + {:ok, _} = Notification.create_notifications(report_activity) + + {:ok, _} = + CommonAPI.post(user2, %{ + status: "hey @#{user1.nickname} !" + }) + + Notification.destroy_multiple_from_types(user1, ["pleroma:report"]) + + assert [%Pleroma.Notification{type: "mention"}] = Notification.for_user(user1) + assert [%Pleroma.Notification{type: "pleroma:report"}] = Notification.for_user(user2) + end + end + describe "set_read_up_to()" do test "it sets all notifications as read up to a specified notification ID" do user = insert(:user) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 7c30f39ad..756281a46 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder + alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers @@ -2153,6 +2154,26 @@ test "performs update cache if user updated" do assert {:ok, user} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") assert %User{bio: "test-bio"} = User.get_cached_by_ap_id(user.ap_id) end + + test "removes report notifs when user isn't superuser any more" do + report_activity = insert(:report_activity) + user = insert(:user, is_moderator: true, is_admin: true) + {:ok, _} = Notification.create_notifications(report_activity) + + assert [%Pleroma.Notification{type: "pleroma:report"}] = Notification.for_user(user) + + {:ok, user} = user |> User.admin_api_update(%{is_moderator: false}) + # is still superuser because still admin + assert [%Pleroma.Notification{type: "pleroma:report"}] = Notification.for_user(user) + + {:ok, user} = user |> User.admin_api_update(%{is_moderator: true, is_admin: false}) + # is still superuser because still moderator + assert [%Pleroma.Notification{type: "pleroma:report"}] = Notification.for_user(user) + + {:ok, user} = user |> User.admin_api_update(%{is_moderator: false}) + # is not a superuser any more + assert [] = Notification.for_user(user) + end end describe "following/followers synchronization" do From a20d2847e2959d4fce86757057a680f0fcfa87ec Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 7 Mar 2022 14:00:42 +0100 Subject: [PATCH 06/94] After code review Use patern matching to see if someone was superuser before --- lib/pleroma/user.ex | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 809524f56..27ed9bbc5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1099,11 +1099,8 @@ def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do |> maybe_remove_report_notifications(was_superuser_before_update) end - defp maybe_remove_report_notifications( - {:ok, %Pleroma.User{} = user} = result, - was_superuser_before_update - ) do - if was_superuser_before_update and not User.superuser?(user), + defp maybe_remove_report_notifications({:ok, %Pleroma.User{} = user} = result, true) do + if not User.superuser?(user), do: user |> Notification.destroy_multiple_from_types(["pleroma:report"]) result From 775f997c40a8a7670753fadb55c2d787cd7cef96 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 21 Feb 2022 17:54:18 -0500 Subject: [PATCH 07/94] Prefer userLanguage cookie over Accept-Language header in detecting locale https://git.pleroma.social/pleroma/pleroma-meta/-/issues/60 --- lib/pleroma/web/plugs/set_locale_plug.ex | 29 ++++++++- .../web/plugs/set_locale_plug_test.exs | 59 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index d77191cff..446baf24b 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.Plugs.SetLocalePlug do import Plug.Conn, only: [get_req_header: 2, assign: 3] + def frontend_language_cookie_name(), do: "userLanguage" + def init(_), do: nil def call(conn, _) do @@ -16,10 +18,35 @@ def call(conn, _) do defp get_locale_from_header(conn) do conn - |> extract_accept_language() + |> extract_preferred_language() + |> normalize_language_codes() |> Enum.find(&supported_locale?/1) end + defp normalize_language_codes(codes) do + codes + |> Enum.map(fn code -> String.replace(code, "-", "_") end) + end + + defp extract_preferred_language(conn) do + extract_frontend_language(conn) ++ extract_accept_language(conn) + end + + defp extract_frontend_language(conn) do + %{req_cookies: cookies} = + conn + |> Plug.Conn.fetch_cookies() + + case cookies[frontend_language_cookie_name()] do + nil -> + [] + + fe_lang -> + [fe_lang] + |> ensure_language_fallbacks() + end + end + defp extract_accept_language(conn) do case get_req_header(conn, "accept-language") do [value | _] -> diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index 5261e67ae..043d7eb18 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -33,6 +33,65 @@ test "use supported locale from `accept-language`" do assert %{locale: "ru"} == conn.assigns end + test "use supported locale with specifiers from `accept-language`" do + conn = + :get + |> conn("/cofe") + |> Conn.put_req_header( + "accept-language", + "zh-Hans;q=0.9, en;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "zh_Hans" == Gettext.get_locale() + assert %{locale: "zh_Hans"} == conn.assigns + end + + test "use supported locale from cookie" do + conn = + :get + |> conn("/cofe") + |> put_req_cookie(SetLocalePlug.frontend_language_cookie_name(), "zh-Hans") + |> Conn.put_req_header( + "accept-language", + "ru, fr-CH, fr;q=0.9, en;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "zh_Hans" == Gettext.get_locale() + assert %{locale: "zh_Hans"} == conn.assigns + end + + test "fallback to supported locale from `accept-language` if locale in cookie not supported" do + conn = + :get + |> conn("/cofe") + |> put_req_cookie(SetLocalePlug.frontend_language_cookie_name(), "x-nonexist") + |> Conn.put_req_header( + "accept-language", + "ru, fr-CH, fr;q=0.9, en;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "ru" == Gettext.get_locale() + assert %{locale: "ru"} == conn.assigns + end + + test "fallback to default if nothing is supported" do + conn = + :get + |> conn("/cofe") + |> put_req_cookie(SetLocalePlug.frontend_language_cookie_name(), "x-nonexist") + |> Conn.put_req_header( + "accept-language", + "x-nonexist" + ) + |> SetLocalePlug.call([]) + + assert "en" == Gettext.get_locale() + assert %{locale: "en"} == conn.assigns + end + test "use default locale if locale from `accept-language` is not supported" do conn = :get From ed1d9e91bc3448a6f514bdd23126a086dea18bde Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 21 Feb 2022 18:42:25 -0500 Subject: [PATCH 08/94] Make remote follow pages translatable --- .../templates/twitter_api/remote_follow/follow.html.eex | 6 +++--- .../twitter_api/remote_follow/follow_login.html.eex | 8 ++++---- .../twitter_api/remote_follow/follow_mfa.html.eex | 6 +++--- .../templates/twitter_api/remote_follow/followed.html.eex | 5 ++--- .../web/templates/twitter_api/util/subscribe.html.eex | 8 ++++---- lib/pleroma/web/twitter_api/views/remote_follow_view.ex | 1 + lib/pleroma/web/twitter_api/views/util_view.ex | 1 + 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex index a7be53091..e2d251fac 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex @@ -1,11 +1,11 @@ <%= if @error == :error do %> -

Error fetching user

+

<%= Gettext.dpgettext("static_pages", "remote follow error", "Error fetching user") %>

<% else %> -

Remote follow

+

<%= Gettext.dpgettext("static_pages", "remote follow header", "Remote follow") %>

<%= @followee.nickname %>

<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "user"], fn f -> %> <%= hidden_input f, :id, value: @followee.id %> - <%= submit "Authorize" %> + <%= submit Gettext.dpgettext("static_pages", "remote follow authorization button", "Authorize") %> <% end %> <% end %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex index a8026fa9d..26340a906 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex @@ -1,14 +1,14 @@ <%= if @error do %>

<%= @error %>

<% end %> -

Log in to follow

+

<%= Gettext.dpgettext("static_pages", "remote follow header, need login", "Log in to follow") %>

<%= @followee.nickname %>

<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "authorization"], fn f -> %> -<%= text_input f, :name, placeholder: "Username", required: true %> +<%= text_input f, :name, placeholder: Gettext.dpgettext("static_pages", "placeholder text for username entry", "Username"), required: true, autocomplete: "username" %>
-<%= password_input f, :password, placeholder: "Password", required: true %> +<%= password_input f, :password, placeholder: Gettext.dpgettext("static_pages", "placeholder text for password entry", "Password"), required: true, autocomplete: "password" %>
<%= hidden_input f, :id, value: @followee.id %> -<%= submit "Authorize" %> +<%= submit Gettext.dpgettext("static_pages", "remote follow authorization button for login", "Authorize") %> <% end %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex index a54ed83b5..638212c1e 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex @@ -1,13 +1,13 @@ <%= if @error do %>

<%= @error %>

<% end %> -

Two-factor authentication

+

<%= Gettext.dpgettext("static_pages", "remote follow mfa header", "Two-factor authentication") %>

<%= @followee.nickname %>

<%= form_for @conn, Routes.remote_follow_path(@conn, :do_follow), [as: "mfa"], fn f -> %> -<%= text_input f, :code, placeholder: "Authentication code", required: true %> +<%= text_input f, :code, placeholder: Gettext.dpgettext("static_pages", "placeholder text for auth code entry", "Authentication code"), required: true %>
<%= hidden_input f, :id, value: @followee.id %> <%= hidden_input f, :token, value: @mfa_token %> -<%= submit "Authorize" %> +<%= submit Gettext.dpgettext("static_pages", "remote follow authorization button for mfa", "Authorize") %> <% end %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex index da473d502..2fb4cc5d3 100644 --- a/lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex @@ -1,6 +1,5 @@ <%= if @error do %> -

Error following account

+

<%= Gettext.dpgettext("static_pages", "remote follow error", "Error following account") %>

<% else %> -

Account followed!

+

<%= Gettext.dpgettext("static_pages", "remote follow success", "Account followed!") %>

<% end %> - diff --git a/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex index a6b313d8a..848660f26 100644 --- a/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex +++ b/lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex @@ -1,10 +1,10 @@ <%= if @error do %> -

Error: <%= @error %>

+

<%= Gettext.dpgettext("static_pages", "remote follow error", "Error: %{error}", error: @error) %>

<% else %> -

Remotely follow <%= @nickname %>

+

<%= Gettext.dpgettext("static_pages", "remote follow header", "Remotely follow %{nickname}", nickname: @nickname) %>

<%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "user"], fn f -> %> <%= hidden_input f, :nickname, value: @nickname %> - <%= text_input f, :profile, placeholder: "Your account ID, e.g. lain@quitter.se" %> - <%= submit "Follow" %> + <%= text_input f, :profile, placeholder: Gettext.dpgettext("static_pages", "placeholder text for account id", "Your account ID, e.g. lain@quitter.se") %> + <%= submit Gettext.dpgettext("static_pages", "remote follow authorization button for following with a remote account", "Follow") %> <% end %> <% end %> diff --git a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex index ac3f15eec..618ba2ba5 100644 --- a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex +++ b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.Web.Gettext defdelegate avatar_url(user), to: Pleroma.User end diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index 87cb79dd7..a03020290 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do import Phoenix.HTML.Form alias Pleroma.Config alias Pleroma.Web.Endpoint + alias Pleroma.Web.Gettext def status_net_config(instance) do """ From 7107fdc1b92420680ff56b253ffa780fd6f9253d Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 21 Feb 2022 18:44:36 -0500 Subject: [PATCH 09/94] Extract translatable text --- priv/gettext/default.pot | 185 ++++++++++++++++++++ priv/gettext/errors.pot | 315 +++++++++++++++++----------------- priv/gettext/posix_errors.pot | 88 +++++----- priv/gettext/static_pages.pot | 107 ++++++++++++ 4 files changed, 490 insertions(+), 205 deletions(-) create mode 100644 priv/gettext/default.pot create mode 100644 priv/gettext/static_pages.pot diff --git a/priv/gettext/default.pot b/priv/gettext/default.pot new file mode 100644 index 000000000..fed111ccb --- /dev/null +++ b/priv/gettext/default.pot @@ -0,0 +1,185 @@ +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here as no +## effect: edit them in PO (.po) files instead. +msgid "" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:122 +msgid "%{name} - %{count} is not a multiple of %{multiple}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:131 +msgid "%{name} - %{value} is larger than exclusive maximum %{max}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:140 +msgid "%{name} - %{value} is larger than inclusive maximum %{max}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:149 +msgid "%{name} - %{value} is smaller than exclusive minimum %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:158 +msgid "%{name} - %{value} is smaller than inclusive minimum %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:102 +msgid "%{name} - Array items must be unique." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:114 +msgid "%{name} - Array length %{length} is larger than maxItems: %{}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:106 +msgid "%{name} - Array length %{length} is smaller than minItems: %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:166 +msgid "%{name} - Invalid %{type}. Got: %{value}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:174 +msgid "%{name} - Invalid format. Expected %{format}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:51 +msgid "%{name} - Invalid schema.type. Got: %{type}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:178 +msgid "%{name} - Invalid value for enum." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:95 +msgid "%{name} - String length is larger than maxLength: %{length}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:88 +msgid "%{name} - String length is smaller than minLength: %{length}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:63 +msgid "%{name} - null value where %{type} expected." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:60 +msgid "%{name} - null value." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:182 +msgid "Failed to cast to any schema in %{polymorphic_type}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:71 +msgid "Failed to cast value as %{invalid_schema}. Value must be castable using `allOf` schemas listed." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:84 +msgid "Failed to cast value to one of: %{failed_schemas}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:78 +msgid "Failed to cast value using any of: %{failed_schemas}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:212 +msgid "Invalid value for header: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:204 +msgid "Missing field: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:208 +msgid "Missing header: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:196 +msgid "No value provided for required discriminator `%{field}`." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:216 +msgid "Object property count %{property_count} is greater than maxProperties: %{max_properties}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:224 +msgid "Object property count %{property_count} is less than minProperties: %{min_properties}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/static_fe/static_fe/error.html.eex:2 +msgid "Oops" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:188 +msgid "Unexpected field: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:200 +msgid "Unknown schema: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:192 +msgid "Value used as discriminator for `%{field}` matches no schemas." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:43 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:37 +msgid "announces" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:44 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:38 +msgid "likes" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:42 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:36 +msgid "replies" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:27 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:22 +msgid "sensitive media" +msgstr "" diff --git a/priv/gettext/errors.pot b/priv/gettext/errors.pot index e337226a7..7644fc230 100644 --- a/priv/gettext/errors.pot +++ b/priv/gettext/errors.pot @@ -90,121 +90,99 @@ msgid "must be equal to %{number}" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:505 +#: lib/pleroma/web/common_api.ex:523 msgid "Account not found" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:339 +#: lib/pleroma/web/common_api.ex:316 msgid "Already voted" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:359 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:402 msgid "Bad request" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:426 -msgid "Can't delete object" -msgstr "" - -#, elixir-format -#: lib/pleroma/web/controller_helper.ex:105 -#: lib/pleroma/web/controller_helper.ex:111 +#: lib/pleroma/web/controller_helper.ex:97 +#: lib/pleroma/web/controller_helper.ex:103 msgid "Can't display this activity" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:285 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:324 msgid "Can't find user" msgstr "" #, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:61 +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:80 msgid "Can't get favorites" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:438 -msgid "Can't like object" -msgstr "" - -#, elixir-format -#: lib/pleroma/web/common_api/utils.ex:563 +#: lib/pleroma/web/common_api/utils.ex:482 msgid "Cannot post an empty status without attachments" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/utils.ex:511 +#: lib/pleroma/web/common_api/utils.ex:441 msgid "Comment must be up to %{max_size} characters" msgstr "" #, elixir-format -#: lib/pleroma/config/config_db.ex:191 +#: lib/pleroma/config_db.ex:200 msgid "Config with params %{params} not found" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:181 -#: lib/pleroma/web/common_api/common_api.ex:185 +#: lib/pleroma/web/common_api.ex:167 lib/pleroma/web/common_api.ex:171 msgid "Could not delete" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:231 +#: lib/pleroma/web/common_api.ex:217 msgid "Could not favorite" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:453 -msgid "Could not pin" -msgstr "" - -#, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:278 +#: lib/pleroma/web/common_api.ex:254 msgid "Could not unfavorite" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:463 -msgid "Could not unpin" -msgstr "" - -#, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:216 +#: lib/pleroma/web/common_api.ex:202 msgid "Could not unrepeat" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:512 -#: lib/pleroma/web/common_api/common_api.ex:521 +#: lib/pleroma/web/common_api.ex:530 lib/pleroma/web/common_api.ex:539 msgid "Could not update state" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:207 +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:205 msgid "Error." msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:106 +#: lib/pleroma/web/twitter_api/twitter_api.ex:99 msgid "Invalid CAPTCHA" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:116 -#: lib/pleroma/web/oauth/oauth_controller.ex:568 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:144 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:631 msgid "Invalid credentials" msgstr "" #, elixir-format -#: lib/pleroma/plugs/ensure_authenticated_plug.ex:38 +#: lib/pleroma/web/plugs/ensure_authenticated_plug.ex:42 msgid "Invalid credentials." msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:355 +#: lib/pleroma/web/common_api.ex:337 msgid "Invalid indices" msgstr "" @@ -214,189 +192,184 @@ msgid "Invalid parameters" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/utils.ex:414 +#: lib/pleroma/web/common_api/utils.ex:349 msgid "Invalid password." msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:220 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:254 msgid "Invalid request" msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:109 +#: lib/pleroma/web/twitter_api/twitter_api.ex:102 msgid "Kocaptcha service unavailable" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:112 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:140 msgid "Missing parameters" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/utils.ex:547 +#: lib/pleroma/web/common_api/utils.ex:477 msgid "No such conversation" msgstr "" #, elixir-format -#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:388 -#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:414 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:456 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:171 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:197 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:239 msgid "No such permission_group" msgstr "" #, elixir-format -#: lib/pleroma/plugs/uploaded_media.ex:84 -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:486 lib/pleroma/web/admin_api/controllers/fallback_controller.ex:11 -#: lib/pleroma/web/feed/user_controller.ex:71 lib/pleroma/web/ostatus/ostatus_controller.ex:143 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:504 +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:11 lib/pleroma/web/feed/tag_controller.ex:16 +#: lib/pleroma/web/feed/user_controller.ex:69 lib/pleroma/web/o_status/o_status_controller.ex:132 +#: lib/pleroma/web/plugs/uploaded_media.ex:84 msgid "Not found" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:331 +#: lib/pleroma/web/common_api.ex:308 msgid "Poll's author can't vote" msgstr "" #, elixir-format #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 -#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49 -#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:50 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:306 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:52 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:326 #: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71 msgid "Record not found" msgstr "" #, elixir-format #: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:35 -#: lib/pleroma/web/feed/user_controller.ex:77 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:36 -#: lib/pleroma/web/ostatus/ostatus_controller.ex:149 +#: lib/pleroma/web/feed/user_controller.ex:78 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:42 +#: lib/pleroma/web/o_status/o_status_controller.ex:138 msgid "Something went wrong" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/activity_draft.ex:107 +#: lib/pleroma/web/common_api/activity_draft.ex:143 msgid "The message visibility must be direct" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/utils.ex:573 +#: lib/pleroma/web/common_api/utils.ex:492 msgid "The status is over the character limit" msgstr "" #, elixir-format -#: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31 +#: lib/pleroma/web/plugs/ensure_public_or_authenticated_plug.ex:36 msgid "This resource requires authentication." msgstr "" #, elixir-format -#: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206 +#: lib/pleroma/web/plugs/rate_limiter.ex:208 msgid "Throttled" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:356 +#: lib/pleroma/web/common_api.ex:338 msgid "Too many choices" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:443 -msgid "Unhandled activity type" -msgstr "" - -#, elixir-format -#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:485 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:268 msgid "You can't revoke your own admin status." msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:221 -#: lib/pleroma/web/oauth/oauth_controller.ex:308 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:243 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:333 msgid "Your account is currently disabled" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:183 -#: lib/pleroma/web/oauth/oauth_controller.ex:331 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:205 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:356 msgid "Your login is missing a confirmed e-mail address" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:390 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:392 msgid "can't read inbox of %{nickname} as %{as_nickname}" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:473 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:491 msgid "can't update outbox of %{nickname} as %{as_nickname}" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:471 +#: lib/pleroma/web/common_api.ex:475 msgid "conversation is already muted" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:314 -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:492 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:510 msgid "error" msgstr "" #, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:32 +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:34 msgid "mascots can only be images" msgstr "" #, elixir-format -#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:62 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:63 msgid "not found" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:394 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:437 msgid "Bad OAuth request." msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:115 +#: lib/pleroma/web/twitter_api/twitter_api.ex:108 msgid "CAPTCHA already used" msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:112 +#: lib/pleroma/web/twitter_api/twitter_api.ex:105 msgid "CAPTCHA expired" msgstr "" #, elixir-format -#: lib/pleroma/plugs/uploaded_media.ex:57 +#: lib/pleroma/web/plugs/uploaded_media.ex:57 msgid "Failed" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:410 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:453 msgid "Failed to authenticate: %{message}." msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:441 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:484 msgid "Failed to set up user account." msgstr "" #, elixir-format -#: lib/pleroma/plugs/oauth_scopes_plug.ex:38 +#: lib/pleroma/web/plugs/o_auth_scopes_plug.ex:37 msgid "Insufficient permissions: %{permissions}." msgstr "" #, elixir-format -#: lib/pleroma/plugs/uploaded_media.ex:104 +#: lib/pleroma/web/plugs/uploaded_media.ex:111 msgid "Internal Error" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/fallback_controller.ex:22 -#: lib/pleroma/web/oauth/fallback_controller.ex:29 +#: lib/pleroma/web/o_auth/fallback_controller.ex:22 +#: lib/pleroma/web/o_auth/fallback_controller.ex:29 msgid "Invalid Username/Password" msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:118 +#: lib/pleroma/web/twitter_api/twitter_api.ex:111 msgid "Invalid answer data" msgstr "" @@ -406,28 +379,28 @@ msgid "Nodeinfo schema version not handled" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:172 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:194 msgid "This action is outside the authorized scopes" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/fallback_controller.ex:14 +#: lib/pleroma/web/o_auth/fallback_controller.ex:14 msgid "Unknown error, please check the details and try again." msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:119 -#: lib/pleroma/web/oauth/oauth_controller.ex:158 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:136 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:180 msgid "Unlisted redirect_uri." msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:390 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:433 msgid "Unsupported OAuth provider: %{provider}." msgstr "" #, elixir-format -#: lib/pleroma/uploaders/uploader.ex:72 +#: lib/pleroma/uploaders/uploader.ex:74 msgid "Uploader callback timeout" msgstr "" @@ -437,120 +410,101 @@ msgid "bad request" msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:103 +#: lib/pleroma/web/twitter_api/twitter_api.ex:96 msgid "CAPTCHA Error" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:290 +#: lib/pleroma/web/common_api.ex:266 msgid "Could not add reaction emoji" msgstr "" #, elixir-format -#: lib/pleroma/web/common_api/common_api.ex:301 +#: lib/pleroma/web/common_api.ex:277 msgid "Could not remove reaction emoji" msgstr "" #, elixir-format -#: lib/pleroma/web/twitter_api/twitter_api.ex:129 +#: lib/pleroma/web/twitter_api/twitter_api.ex:122 msgid "Invalid CAPTCHA (Missing parameter: %{name})" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92 +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:96 msgid "List not found" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:123 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:151 msgid "Missing parameter: %{name}" msgstr "" #, elixir-format -#: lib/pleroma/web/oauth/oauth_controller.ex:210 -#: lib/pleroma/web/oauth/oauth_controller.ex:321 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:232 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:346 msgid "Password reset is required" msgstr "" #, elixir-format #: lib/pleroma/tests/auth_test_controller.ex:9 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:6 -#: lib/pleroma/web/admin_api/controllers/config_controller.ex:6 lib/pleroma/web/admin_api/controllers/fallback_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/chat_controller.ex:6 lib/pleroma/web/admin_api/controllers/config_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:6 lib/pleroma/web/admin_api/controllers/frontend_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/instance_controller.ex:6 lib/pleroma/web/admin_api/controllers/instance_document_controller.ex:6 #: lib/pleroma/web/admin_api/controllers/invite_controller.ex:6 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex:6 -#: lib/pleroma/web/admin_api/controllers/oauth_app_controller.ex:6 lib/pleroma/web/admin_api/controllers/relay_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/o_auth_app_controller.ex:6 lib/pleroma/web/admin_api/controllers/relay_controller.ex:6 #: lib/pleroma/web/admin_api/controllers/report_controller.ex:6 lib/pleroma/web/admin_api/controllers/status_controller.ex:6 -#: lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/embed_controller.ex:6 -#: lib/pleroma/web/fallback_redirect_controller.ex:6 lib/pleroma/web/feed/tag_controller.ex:6 -#: lib/pleroma/web/feed/user_controller.ex:6 lib/pleroma/web/mailer/subscription_controller.ex:2 -#: lib/pleroma/web/masto_fe_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/app_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/user_controller.ex:6 lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/embed_controller.ex:6 +#: lib/pleroma/web/fallback/redirect_controller.ex:6 lib/pleroma/web/feed/tag_controller.ex:6 +#: lib/pleroma/web/feed/user_controller.ex:6 lib/pleroma/web/mailer/subscription_controller.ex:6 +#: lib/pleroma/web/manifest_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/app_controller.ex:11 lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 #: lib/pleroma/web/mastodon_api/controllers/conversation_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/custom_emoji_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 -#: lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/report_controller.ex:8 -#: lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 -#: lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 -#: lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 -#: lib/pleroma/web/media_proxy/media_proxy_controller.ex:6 lib/pleroma/web/mongooseim/mongoose_im_controller.ex:6 -#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6 lib/pleroma/web/oauth/fallback_controller.ex:6 -#: lib/pleroma/web/oauth/mfa_controller.ex:10 lib/pleroma/web/oauth/oauth_controller.ex:6 -#: lib/pleroma/web/ostatus/ostatus_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 -#: lib/pleroma/web/pleroma_api/controllers/chat_controller.ex:5 lib/pleroma/web/pleroma_api/controllers/conversation_controller.ex:6 -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:2 lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex:6 -#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/notification_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/directory_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/report_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 lib/pleroma/web/media_proxy/media_proxy_controller.ex:6 +#: lib/pleroma/web/mongoose_im/mongoose_im_controller.ex:6 lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6 +#: lib/pleroma/web/o_auth/fallback_controller.ex:6 lib/pleroma/web/o_auth/mfa_controller.ex:10 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:6 lib/pleroma/web/o_status/o_status_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/app_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/backup_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex:5 +#: lib/pleroma/web/pleroma_api/controllers/conversation_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/instances_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/notification_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/report_controller.ex:6 #: lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex:6 -#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/static_fe/static_fe_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex:6 +#: lib/pleroma/web/static_fe/static_fe_controller.ex:6 lib/pleroma/web/twitter_api/controller.ex:6 #: lib/pleroma/web/twitter_api/controllers/password_controller.ex:10 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex:6 -#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/twitter_api/twitter_api_controller.ex:6 -#: lib/pleroma/web/uploader_controller.ex:6 lib/pleroma/web/web_finger/web_finger_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/uploader_controller.ex:6 +#: lib/pleroma/web/web_finger/web_finger_controller.ex:6 msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped." msgstr "" #, elixir-format -#: lib/pleroma/plugs/ensure_authenticated_plug.ex:28 +#: lib/pleroma/web/plugs/ensure_authenticated_plug.ex:32 msgid "Two-factor authentication enabled, you must use a access token." msgstr "" -#, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:210 -msgid "Unexpected error occurred while adding file to pack." -msgstr "" - -#, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:138 -msgid "Unexpected error occurred while creating pack." -msgstr "" - -#, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:278 -msgid "Unexpected error occurred while removing file from pack." -msgstr "" - -#, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:250 -msgid "Unexpected error occurred while updating file in pack." -msgstr "" - -#, elixir-format -#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:179 -msgid "Unexpected error occurred while updating pack metadata." -msgstr "" - #, elixir-format #: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 msgid "Web push subscription is disabled on this Pleroma instance" msgstr "" #, elixir-format -#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:451 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:234 msgid "You can't revoke your own admin/moderator status." msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:126 +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:129 msgid "authorization required for timeline view" msgstr "" @@ -560,11 +514,50 @@ msgid "Access denied" msgstr "" #, elixir-format -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:282 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:321 msgid "This API requires an authenticated user" msgstr "" #, elixir-format -#: lib/pleroma/plugs/user_is_admin_plug.ex:21 +#: lib/pleroma/web/plugs/ensure_staff_privileged_plug.ex:26 +#: lib/pleroma/web/plugs/user_is_admin_plug.ex:21 msgid "User is not an admin." msgstr "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:75 +msgid "Last export was less than a day ago" +msgid_plural "Last export was less than %{days} days ago" +msgstr[0] "" +msgstr[1] "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:93 +msgid "Backups require enabled email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:423 +msgid "Character limit (%{limit} characters) exceeded, contains %{length} characters" +msgstr "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:98 +msgid "Email is required" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:507 +msgid "Too many attachments" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_staff_privileged_plug.ex:33 +#: lib/pleroma/web/plugs/user_is_staff_plug.ex:20 +msgid "User is not a staff member." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:366 +msgid "Your account is awaiting approval." +msgstr "" diff --git a/priv/gettext/posix_errors.pot b/priv/gettext/posix_errors.pot index c9f593944..3533639e0 100644 --- a/priv/gettext/posix_errors.pot +++ b/priv/gettext/posix_errors.pot @@ -15,135 +15,135 @@ msgstr "" msgid "eagain" msgstr "" - + msgid "ebadf" msgstr "" - + msgid "ebadmsg" msgstr "" - + msgid "ebusy" msgstr "" - + msgid "edeadlk" msgstr "" - + msgid "edeadlock" msgstr "" - + msgid "edquot" msgstr "" - + msgid "eexist" msgstr "" - + msgid "efault" msgstr "" - + msgid "efbig" msgstr "" - + msgid "eftype" msgstr "" - + msgid "eintr" msgstr "" - + msgid "einval" msgstr "" - + msgid "eio" msgstr "" - + msgid "eisdir" msgstr "" - + msgid "eloop" msgstr "" - + msgid "emfile" msgstr "" - + msgid "emlink" msgstr "" - + msgid "emultihop" msgstr "" - + msgid "enametoolong" msgstr "" - + msgid "enfile" msgstr "" - + msgid "enobufs" msgstr "" - + msgid "enodev" msgstr "" - + msgid "enolck" msgstr "" - + msgid "enolink" msgstr "" - + msgid "enoent" msgstr "" - + msgid "enomem" msgstr "" - + msgid "enospc" msgstr "" - + msgid "enosr" msgstr "" - + msgid "enostr" msgstr "" - + msgid "enosys" msgstr "" - + msgid "enotblk" msgstr "" - + msgid "enotdir" msgstr "" - + msgid "enotsup" msgstr "" - + msgid "enxio" msgstr "" - + msgid "eopnotsupp" msgstr "" - + msgid "eoverflow" msgstr "" - + msgid "epipe" msgstr "" - + msgid "erange" msgstr "" - + msgid "erofs" msgstr "" - + msgid "espipe" msgstr "" - + msgid "esrch" msgstr "" - + msgid "estale" msgstr "" - + msgid "etxtbsy" msgstr "" - + msgid "exdev" msgstr "" diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot new file mode 100644 index 000000000..72e5c00d9 --- /dev/null +++ b/priv/gettext/static_pages.pot @@ -0,0 +1,107 @@ +## This file is a PO Template file. +## +## "msgid"s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run "mix gettext.extract" to bring this file up to +## date. Leave "msgstr"s empty as changing them here as no +## effect: edit them in PO (.po) files instead. +msgid "" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 +msgctxt "remote follow authorization button" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:2 +msgctxt "remote follow error" +msgid "Error fetching user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:4 +msgctxt "remote follow header" +msgid "Remote follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:8 +msgctxt "placeholder text for auth code entry" +msgid "Authentication code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:10 +msgctxt "placeholder text for password entry" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:8 +msgctxt "placeholder text for username entry" +msgid "Username" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:13 +msgctxt "remote follow authorization button for login" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:12 +msgctxt "remote follow authorization button for mfa" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:2 +msgctxt "remote follow error" +msgid "Error following account" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:4 +msgctxt "remote follow header, need login" +msgid "Log in to follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:4 +msgctxt "remote follow mfa header" +msgid "Two-factor authentication" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:4 +msgctxt "remote follow success" +msgid "Account followed!" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:7 +msgctxt "placeholder text for account id" +msgid "Your account ID, e.g. lain@quitter.se" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:8 +msgctxt "remote follow authorization button for following with a remote account" +msgid "Follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:2 +msgctxt "remote follow error" +msgid "Error: %{error}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:4 +msgctxt "remote follow header" +msgid "Remotely follow %{nickname}" +msgstr "" From 8f08c902a5602cd6181e001d6d71a99f400fa9ef Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 21 Feb 2022 19:12:32 -0500 Subject: [PATCH 10/94] Make lint happy --- lib/pleroma/web/plugs/set_locale_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index 446baf24b..a9387ba7e 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Plugs.SetLocalePlug do import Plug.Conn, only: [get_req_header: 2, assign: 3] - def frontend_language_cookie_name(), do: "userLanguage" + def frontend_language_cookie_name, do: "userLanguage" def init(_), do: nil From 50913c4dc5c394b55fa5d62e6ab3cc72f3959c5d Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 01:04:04 -0500 Subject: [PATCH 11/94] Make password reset pages translatable --- .../password/invalid_token.html.eex | 2 +- .../twitter_api/password/reset.html.eex | 6 +-- .../password/reset_failed.html.eex | 8 +++- .../password/reset_success.html.eex | 4 +- .../web/twitter_api/views/password_view.ex | 1 + priv/gettext/static_pages.pot | 48 +++++++++++++++++++ 6 files changed, 61 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex b/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex index ee84750c7..5ac0aa4e0 100644 --- a/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex @@ -1 +1 @@ -

Invalid Token

+

<%= Gettext.dpgettext("static_pages", "password reset invalid token message", "Invalid Token") %>

diff --git a/lib/pleroma/web/templates/twitter_api/password/reset.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex index fbcacdc14..6a544af51 100644 --- a/lib/pleroma/web/templates/twitter_api/password/reset.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset.html.eex @@ -1,13 +1,13 @@

Password Reset for <%= @user.nickname %>

<%= form_for @conn, Routes.reset_password_path(@conn, :do_reset), [as: "data"], fn f -> %>
- <%= label f, :password, "Password" %> + <%= label f, :password, Gettext.dpgettext("static_pages", "password reset form password prompt", "Password") %> <%= password_input f, :password %>
- <%= label f, :password_confirmation, "Confirmation" %> + <%= label f, :password_confirmation, Gettext.dpgettext("static_pages", "password reset form confirm password prompt", "Confirmation") %> <%= password_input f, :password_confirmation %>
<%= hidden_input f, :token, value: @token.token %> - <%= submit "Reset" %> + <%= submit Gettext.dpgettext("static_pages", "password reset button", "Reset") %> <% end %> diff --git a/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex index 4ed4ac8bc..774e3462a 100644 --- a/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex @@ -1,2 +1,6 @@ -

Password reset failed

-

Homepage

+

<%= Gettext.dpgettext("static_pages", "password reset failed message", "Password reset failed") %>

+

+ + <%= Gettext.dpgettext("static_pages", "password reset failed homepage link", "Homepage") %> + +

diff --git a/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex b/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex index 086d4e08b..40f6bb3fc 100644 --- a/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex +++ b/lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex @@ -1,2 +1,2 @@ -

Password changed!

-

Homepage

+

<%= Gettext.dpgettext("static_pages", "password reset successful message", "Password changed!") %>

+

<%= Gettext.dpgettext("static_pages", "password reset successful homepage link", "Homepage") %>

diff --git a/lib/pleroma/web/twitter_api/views/password_view.ex b/lib/pleroma/web/twitter_api/views/password_view.ex index a9bb95a2c..40e7fca49 100644 --- a/lib/pleroma/web/twitter_api/views/password_view.ex +++ b/lib/pleroma/web/twitter_api/views/password_view.ex @@ -5,4 +5,5 @@ defmodule Pleroma.Web.TwitterAPI.PasswordView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.Web.Gettext end diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index 72e5c00d9..8b56ad7b4 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -105,3 +105,51 @@ msgstr "" msgctxt "remote follow header" msgid "Remotely follow %{nickname}" msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:12 +msgctxt "password reset button" +msgid "Reset" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:4 +msgctxt "password reset failed homepage link" +msgid "Homepage" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:1 +msgctxt "password reset failed message" +msgid "Password reset failed" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:8 +msgctxt "password reset form confirm password prompt" +msgid "Confirmation" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:4 +msgctxt "password reset form password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex:1 +msgctxt "password reset invalid token message" +msgid "Invalid Token" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:2 +msgctxt "password reset successful homepage link" +msgid "Homepage" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:1 +msgctxt "password reset successful message" +msgid "Password changed!" +msgstr "" From ec57e3480c58a003cb40f216c9aaddc1e90be771 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 01:13:39 -0500 Subject: [PATCH 12/94] Make tag feed translatable --- lib/pleroma/web/feed/feed_view.ex | 1 + lib/pleroma/web/templates/feed/feed/tag.atom.eex | 2 +- lib/pleroma/web/templates/feed/feed/tag.rss.eex | 2 +- priv/gettext/static_pages.pot | 7 +++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index c0fb35e01..d674bc26f 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.Feed.FeedView do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.MediaProxy + alias Pleroma.Web.Gettext require Pleroma.Constants diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index de0731085..2d860f12b 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -12,7 +12,7 @@ <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> #<%= @tag %> - These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. + <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> <%= feed_logo() %> <%= most_recent_update(@activities) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index 9c3613feb..edcc3e436 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -4,7 +4,7 @@ #<%= @tag %> - These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. + <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> <%= feed_logo() %> 2b90d9 diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index 8b56ad7b4..b230fbaa7 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -153,3 +153,10 @@ msgstr "" msgctxt "password reset successful message" msgid "Password changed!" msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/feed/feed/tag.atom.eex:15 +#: lib/pleroma/web/templates/feed/feed/tag.rss.eex:7 +msgctxt "tag feed description" +msgid "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse." +msgstr "" From 793922f1d8103d42cb37b3aef43f2a04491b8c06 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 01:28:23 -0500 Subject: [PATCH 13/94] Use proper lang attributes in htmls --- lib/pleroma/web/gettext.ex | 9 +++++++++ lib/pleroma/web/templates/feed/feed/tag.atom.eex | 2 +- lib/pleroma/web/templates/layout/app.html.eex | 2 +- lib/pleroma/web/templates/layout/email.html.eex | 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index c0ca4d0e9..c8a739c2b 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -25,4 +25,13 @@ defmodule Pleroma.Web.Gettext do See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage. """ use Gettext, otp_app: :pleroma + + def language_tag do + # Naive implementation: HTML lang attribute uses BCP 47, which + # uses - as a separator. + # https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang + + Gettext.get_locale() + |> String.replace("_", "-", global: true) + end end diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 2d860f12b..6d497e84c 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -1,6 +1,6 @@ - - + diff --git a/lib/pleroma/web/templates/layout/email.html.eex b/lib/pleroma/web/templates/layout/email.html.eex index f6dcd7f0f..087aa4fc0 100644 --- a/lib/pleroma/web/templates/layout/email.html.eex +++ b/lib/pleroma/web/templates/layout/email.html.eex @@ -1,5 +1,5 @@ - + <%= @email.subject %> @@ -7,4 +7,4 @@ <%= render @view_module, @view_template, assigns %> - \ No newline at end of file + From 8f8d2abb12395a2b3565868d6e52608b832fcbd5 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 02:11:57 -0500 Subject: [PATCH 14/94] Make oauth pages translatable --- lib/pleroma/web/o_auth/o_auth_view.ex | 2 + .../templates/o_auth/o_auth/_scopes.html.eex | 2 +- .../templates/o_auth/o_auth/consumer.html.eex | 4 +- .../o_auth/oob_authorization_created.html.eex | 4 +- .../o_auth/o_auth/oob_token_exists.html.eex | 4 +- .../templates/o_auth/o_auth/register.html.eex | 26 ++-- .../web/templates/o_auth/o_auth/show.html.eex | 22 +-- priv/gettext/static_pages.pot | 145 ++++++++++++++++++ 8 files changed, 179 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/web/o_auth/o_auth_view.ex b/lib/pleroma/web/o_auth/o_auth_view.ex index 1419c96a2..57a315705 100644 --- a/lib/pleroma/web/o_auth/o_auth_view.ex +++ b/lib/pleroma/web/o_auth/o_auth_view.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.OAuth.OAuthView do use Pleroma.Web, :view import Phoenix.HTML.Form + import Phoenix.HTML + alias Pleroma.Web.Gettext alias Pleroma.Web.OAuth.Token.Utils diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex index c9ec1ecbf..73115e92a 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex @@ -1,5 +1,5 @@
- <%= label @form, :scope, "The following permissions will be granted" %> + <%= label @form, :scope, Gettext.dpgettext("static_pages", "oauth scopes message", "The following permissions will be granted") %>
<%= for scope <- @available_scopes do %> <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index dc4521a62..8b894cd58 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,4 +1,4 @@ -

Sign in with external provider

+

<%= Gettext.dpgettext("static_pages", "oauth external provider page title", "Sign in with external provider") %>

<%= form_for @conn, Routes.o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %>
@@ -10,6 +10,6 @@ <%= hidden_input f, :state, value: @state %> <%= for strategy <- Pleroma.Config.oauth_consumer_strategies() do %> - <%= submit "Sign in with #{String.capitalize(strategy)}", name: "provider", value: strategy %> + <%= submit Gettext.dpgettext("static_pages", "oauth external provider sign in button", "Sign in with %{strategy}", strategy: String.capitalize(strategy)), name: "provider", value: strategy %> <% end %> <% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex index ffabe29a6..aaa38513a 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex @@ -1,2 +1,2 @@ -

Successfully authorized

-

Token code is
<%= @auth.token %>

+

<%= Gettext.dpgettext("static_pages", "oauth authorized page title", "Successfully authorized") %>

+

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: html_escape(@auth.token)) %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex index 82785c4b9..de4cd0f34 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex @@ -1,2 +1,2 @@ -

Authorization exists

-

Access token is
<%= @token.token %>

+

<%= Gettext.dpgettext("static_pages", "oauth authorization exists page title", "Authorization exists") %>

+

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: html_escape(@auth.token)) %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 99f900fb7..1f661efb2 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -5,34 +5,34 @@ <% end %> -

Registration Details

+

<%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %>

-

If you'd like to register a new account, please provide the details below.

+

<%= Gettext.dpgettext("static_pages", "oauth register page fill form prompt", "If you'd like to register a new account, please provide the details below.") %>

<%= form_for @conn, Routes.o_auth_path(@conn, :register), [as: "authorization"], fn f -> %>
- <%= label f, :nickname, "Nickname" %> - <%= text_input f, :nickname, value: @nickname %> + <%= label f, :nickname, Gettext.dpgettext("static_pages", "oauth register page nickname prompt", "Nickname") %> + <%= text_input f, :nickname, value: @nickname, autocomplete: "username" %>
- <%= label f, :email, "Email" %> - <%= text_input f, :email, value: @email %> + <%= label f, :email, Gettext.dpgettext("static_pages", "oauth register page email prompt", "Email") %> + <%= text_input f, :email, value: @email, autocomplete: "email" %>
-<%= submit "Proceed as new user", name: "op", value: "register" %> +<%= submit Gettext.dpgettext("static_pages", "oauth register page register button", "Proceed as new user"), name: "op", value: "register" %> -

Alternatively, sign in to connect to existing account.

+

<%= Gettext.dpgettext("static_pages", "oauth register page login prompt", "Alternatively, sign in to connect to existing account.") %>

- <%= label f, :name, "Name or email" %> - <%= text_input f, :name %> + <%= label f, :name, Gettext.dpgettext("static_pages", "oauth register page login username prompt", "Name or email") %> + <%= text_input f, :name, autocomplete: "username" %>
- <%= label f, :password, "Password" %> - <%= password_input f, :password %> + <%= label f, :password, Gettext.dpgettext("static_pages", "oauth register page login password prompt", "Password") %> + <%= password_input f, :password, autocomplete: "password" %>
-<%= submit "Proceed as existing user", name: "op", value: "connect" %> +<%= submit Gettext.dpgettext("static_pages", "oauth register page login button", "Proceed as existing user"), name: "op", value: "connect" %> <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 181a9519a..31ae3cd1b 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -20,36 +20,38 @@
<%= if @app do %> -

Application <%= @app.client_name %> is requesting access to your account.

+

<%= raw Gettext.dpgettext("static_pages", "oauth authorize message", "Application %{client_name} is requesting access to your account.", client_name: html_escape(@app.client_name)) %>

<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %> <% end %> <%= if @user do %>
- Cancel - <%= submit "Approve", class: "button--approve" %> + + <%= Gettext.dpgettext("static_pages", "oauth authorize cancel button", "Cancel") %> + + <%= submit Gettext.dpgettext("static_pages", "oauth authorize approve button", "Approve"), class: "button--approve" %>
<% else %> <%= if @params["registration"] in ["true", true] do %> -

This is the first time you visit! Please enter your Pleroma handle.

-

Choose carefully! You won't be able to change this later. You will be able to change your display name, though.

+

<%= Gettext.dpgettext("static_pages", "oauth register page title", "This is the first time you visit! Please enter your Pleroma handle.") %>

+

<%= Gettext.dpgettext("static_pages", "oauth register nickname unchangeable warning", "Choose carefully! You won't be able to change this later. You will be able to change your display name, though.") %>

- <%= label f, :nickname, "Pleroma Handle" %> - <%= text_input f, :nickname, placeholder: "lain" %> + <%= label f, :nickname, Gettext.dpgettext("static_pages", "oauth register nickname prompt", "Pleroma Handle") %> + <%= text_input f, :nickname, placeholder: "lain", autocomplete: "username" %>
<%= hidden_input f, :name, value: @params["name"] %> <%= hidden_input f, :password, value: @params["password"] %>
<% else %>
- <%= label f, :name, "Username" %> + <%= label f, :name, Gettext.dpgettext("static_pages", "oauth login username prompt", "Username") %> <%= text_input f, :name %>
- <%= label f, :password, "Password" %> + <%= label f, :password, Gettext.dpgettext("static_pages", "oauth login password prompt", "Password") %> <%= password_input f, :password %>
- <%= submit "Log In" %> + <%= submit Gettext.dpgettext("static_pages", "oauth login button", "Log In") %> <% end %> <% end %>
diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index b230fbaa7..f9d535838 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -160,3 +160,148 @@ msgstr "" msgctxt "tag feed description" msgid "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse." msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:1 +msgctxt "oauth authorization exists page title" +msgid "Authorization exists" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:32 +msgctxt "oauth authorize approve button" +msgid "Approve" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:30 +msgctxt "oauth authorize cancel button" +msgid "Cancel" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:23 +msgctxt "oauth authorize message" +msgid "Application %{client_name} is requesting access to your account." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:1 +msgctxt "oauth authorized page title" +msgid "Successfully authorized" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:1 +msgctxt "oauth external provider page title" +msgid "Sign in with external provider" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:13 +msgctxt "oauth external provider sign in button" +msgid "Sign in with %{strategy}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:54 +msgctxt "oauth login button" +msgid "Log In" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:51 +msgctxt "oauth login password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:47 +msgctxt "oauth login username prompt" +msgid "Username" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:39 +msgctxt "oauth register nickname prompt" +msgid "Pleroma Handle" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:37 +msgctxt "oauth register nickname unchangeable warning" +msgid "Choose carefully! You won't be able to change this later. You will be able to change your display name, though." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:18 +msgctxt "oauth register page email prompt" +msgid "Email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:10 +msgctxt "oauth register page fill form prompt" +msgid "If you'd like to register a new account, please provide the details below." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:35 +msgctxt "oauth register page login button" +msgid "Proceed as existing user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:31 +msgctxt "oauth register page login password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:24 +msgctxt "oauth register page login prompt" +msgid "Alternatively, sign in to connect to existing account." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:27 +msgctxt "oauth register page login username prompt" +msgid "Name or email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:14 +msgctxt "oauth register page nickname prompt" +msgid "Nickname" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:22 +msgctxt "oauth register page register button" +msgid "Proceed as new user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:8 +msgctxt "oauth register page title" +msgid "Registration Details" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:36 +msgctxt "oauth register page title" +msgid "This is the first time you visit! Please enter your Pleroma handle." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex:2 +msgctxt "oauth scopes message" +msgid "The following permissions will be granted" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:2 +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:2 +msgctxt "oauth token code message" +msgid "Token code is
%{token}" +msgstr "" From c6652fccae17a0c9580b71b86c62508468587b04 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 11:07:28 -0500 Subject: [PATCH 15/94] Make mfa pages translatable --- lib/pleroma/web/o_auth/mfa_view.ex | 1 + .../templates/o_auth/mfa/recovery.html.eex | 8 ++-- .../web/templates/o_auth/mfa/totp.html.eex | 10 ++-- priv/gettext/static_pages.pot | 48 +++++++++++++++++++ 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/o_auth/mfa_view.ex b/lib/pleroma/web/o_auth/mfa_view.ex index 3d473f29c..952c90efe 100644 --- a/lib/pleroma/web/o_auth/mfa_view.ex +++ b/lib/pleroma/web/o_auth/mfa_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.MFAView do use Pleroma.Web, :view import Phoenix.HTML.Form alias Pleroma.MFA + alias Pleroma.Web.Gettext def render("mfa_response.json", %{token: token, user: user}) do %{ diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index b9daa8d8b..e45d13bdf 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -5,11 +5,11 @@ <% end %> -

Two-factor recovery

+

<%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %>

<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
- <%= label f, :code, "Recovery code" %> + <%= label f, :code, Gettext.dpgettext("static_pages", "mfa recover recovery code prompt", "Recovery code") %> <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> @@ -17,8 +17,8 @@ <%= hidden_input f, :challenge_type, value: "recovery" %>
-<%= submit "Verify" %> +<%= submit Gettext.dpgettext("static_pages", "mfa recover verify recovery code button", "Verify") %> <% end %> "> - Enter a two-factor code + <%= Gettext.dpgettext("static_pages", "mfa recover use 2fa code link", "Enter a two-factor code") %> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 29ea7c5fb..50e6c04b6 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -5,20 +5,20 @@ <% end %> -

Two-factor authentication

+

<%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %>

<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
- <%= label f, :code, "Authentication code" %> - <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> + <%= label f, :code, Gettext.dpgettext("static_pages", "mfa auth code prompt", "Authentication code") %> + <%= text_input f, :code, [autocomplete: "one-time-code", autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> <%= hidden_input f, :mfa_token, value: @mfa_token %> <%= hidden_input f, :state, value: @state %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :challenge_type, value: "totp" %>
-<%= submit "Verify" %> +<%= submit Gettext.dpgettext("static_pages", "mfa auth verify code button", "Verify") %> <% end %> "> - Enter a two-factor recovery code + <%= Gettext.dpgettext("static_pages", "mfa auth page use recovery code link", "Enter a two-factor recovery code") %> diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index f9d535838..1af6796c8 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -305,3 +305,51 @@ msgstr "" msgctxt "oauth token code message" msgid "Token code is
%{token}" msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:12 +msgctxt "mfa auth code prompt" +msgid "Authentication code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:8 +msgctxt "mfa auth page title" +msgid "Two-factor authentication" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:23 +msgctxt "mfa auth page use recovery code link" +msgid "Enter a two-factor recovery code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:20 +msgctxt "mfa auth verify code button" +msgid "Verify" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:8 +msgctxt "mfa recover page title" +msgid "Two-factor recovery" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:12 +msgctxt "mfa recover recovery code prompt" +msgid "Recovery code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:23 +msgctxt "mfa recover use 2fa code link" +msgid "Enter a two-factor code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:20 +msgctxt "mfa recover verify recovery code button" +msgid "Verify" +msgstr "" From 03aac295312e2b96d25bdae5f56a3bfbe9c507e3 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 28 Feb 2022 11:23:15 -0500 Subject: [PATCH 16/94] Fix tests --- lib/pleroma/web/feed/feed_view.ex | 2 +- .../templates/o_auth/o_auth/oob_authorization_created.html.eex | 2 +- .../web/templates/o_auth/o_auth/oob_token_exists.html.eex | 2 +- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index d674bc26f..52771205e 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Web.Feed.FeedView do alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User - alias Pleroma.Web.MediaProxy alias Pleroma.Web.Gettext + alias Pleroma.Web.MediaProxy require Pleroma.Constants diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex index aaa38513a..76ed3fda5 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex @@ -1,2 +1,2 @@

<%= Gettext.dpgettext("static_pages", "oauth authorized page title", "Successfully authorized") %>

-

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: html_escape(@auth.token)) %>

+

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: safe_to_string(html_escape(@auth.token))) %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex index de4cd0f34..754bf2eb0 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex @@ -1,2 +1,2 @@

<%= Gettext.dpgettext("static_pages", "oauth authorization exists page title", "Authorization exists") %>

-

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: html_escape(@auth.token)) %>

+

<%= raw Gettext.dpgettext("static_pages", "oauth token code message", "Token code is
%{token}", token: safe_to_string(html_escape(@token.token))) %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 31ae3cd1b..a2f41618e 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -20,7 +20,7 @@
<%= if @app do %> -

<%= raw Gettext.dpgettext("static_pages", "oauth authorize message", "Application %{client_name} is requesting access to your account.", client_name: html_escape(@app.client_name)) %>

+

<%= raw Gettext.dpgettext("static_pages", "oauth authorize message", "Application %{client_name} is requesting access to your account.", client_name: safe_to_string(html_escape(@app.client_name))) %>

<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %> <% end %> From 39cfecb18899be6254d1c5eb9694f118e2475902 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 1 Mar 2022 18:48:08 -0500 Subject: [PATCH 17/94] Make static fe translatable --- .../web/templates/static_fe/static_fe/profile.html.eex | 2 +- priv/gettext/static_pages.pot | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 3191bf450..a14ca305e 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -5,7 +5,7 @@
- +
<%= raw Formatter.emojify(@user.name, @user.emoji) %> | <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %> diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index 1af6796c8..1bde0099d 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -353,3 +353,9 @@ msgstr "" msgctxt "mfa recover verify recovery code button" msgid "Verify" msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex:8 +msgctxt "static fe profile page remote follow button" +msgid "Remote follow" +msgstr "" From 63a0536d55a81b2924a00c0a373e58d570cc50a2 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 1 Mar 2022 19:17:11 -0500 Subject: [PATCH 18/94] Make mail and mailer translatable --- .../web/templates/email/digest.html.eex | 10 ++-- .../subscription/unsubscribe_failure.html.eex | 2 +- .../subscription/unsubscribe_success.html.eex | 2 +- lib/pleroma/web/views/email_view.ex | 1 + .../web/views/mailer/subscription_view.ex | 1 + priv/gettext/static_pages.pot | 50 +++++++++++++++++++ 6 files changed, 59 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex index 60eceff22..1efc76e1a 100644 --- a/lib/pleroma/web/templates/email/digest.html.eex +++ b/lib/pleroma/web/templates/email/digest.html.eex @@ -160,7 +160,7 @@

Hey <%= @user.nickname %>, here is what you've missed!

+ style="font-size: 30px; color: <%= @styling.header_color %>;"><%= Gettext.dpgettext("static_pages", "digest email header line", "Hey %{nickname}, here is what you've missed!", nickname: @user.nickname) %>

@@ -382,7 +382,7 @@

<%= length(@followers) %> New Followers<%= Gettext.dpngettext("static_pages", "new followers count header", "%{count} New Follower", "%{count} New Followers", length(@followers), count: length(@followers)) %>

@@ -535,16 +535,16 @@ style="color:<%= @styling.text_color %>;font-family:Arial, 'Helvetica Neue', Helvetica, sans-serif;line-height:120%;padding-top:10px;padding-right:10px;padding-bottom:10px;padding-left:10px;">

- You have received this email because you have signed up to receive digest emails from <%= @instance %> Pleroma instance.

+ <%= raw Gettext.dpgettext("static_pages", "digest email sending reason", "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance.", instance: safe_to_string(html_escape(@instance))) %>

ย 

- The email address you are subscribed as is <%= @user.email %>.

+ <%= raw Gettext.dpgettext("static_pages", "digest email receiver address", "The email address you are subscribed as is %{email}. ", color: safe_to_string(html_escape(@styling.link_color)), email: safe_to_string(html_escape(@user.email))) %>

- To unsubscribe, please go <%= link "here", style: "color: #{@styling.link_color};text-decoration: none;", to: @unsubscribe_link %>.

+ <%= raw Gettext.dpgettext("static_pages", "digest email unsubscribe action", "To unsubscribe, please go %{here}.", here: safe_to_string link(Gettext.dpgettext("static_pages", "digest email unsubscribe action link text", "here"), style: "color: #{@styling.link_color};text-decoration: none;", to: @unsubscribe_link)) %>

diff --git a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex index 7b476f02d..df090ffcd 100644 --- a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex +++ b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex @@ -1 +1 @@ -

UNSUBSCRIBE FAILURE

+

<%= Gettext.dpgettext("static_pages", "mailer unsubscribe failed message", "UNSUBSCRIBE FAILURE") %>

diff --git a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex index 6dfa2c185..cbce495d4 100644 --- a/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex +++ b/lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex @@ -1 +1 @@ -

UNSUBSCRIBE SUCCESSFUL

+

<%= Gettext.dpgettext("static_pages", "mailer unsubscribe successful message", "UNSUBSCRIBE SUCCESSFUL") %>

diff --git a/lib/pleroma/web/views/email_view.ex b/lib/pleroma/web/views/email_view.ex index f7659b994..2ef049d27 100644 --- a/lib/pleroma/web/views/email_view.ex +++ b/lib/pleroma/web/views/email_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.EmailView do use Pleroma.Web, :view import Phoenix.HTML import Phoenix.HTML.Link + alias Pleroma.Web.Gettext def avatar_url(user) do Pleroma.User.avatar_url(user) diff --git a/lib/pleroma/web/views/mailer/subscription_view.ex b/lib/pleroma/web/views/mailer/subscription_view.ex index 1dc80987b..01e96c61c 100644 --- a/lib/pleroma/web/views/mailer/subscription_view.ex +++ b/lib/pleroma/web/views/mailer/subscription_view.ex @@ -4,4 +4,5 @@ defmodule Pleroma.Web.Mailer.SubscriptionView do use Pleroma.Web, :view + alias Pleroma.Web.Gettext end diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index 1bde0099d..b76641e28 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -359,3 +359,53 @@ msgstr "" msgctxt "static fe profile page remote follow button" msgid "Remote follow" msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:163 +msgctxt "digest email header line" +msgid "Hey %{nickname}, here is what you've missed!" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:544 +msgctxt "digest email receiver address" +msgid "The email address you are subscribed as is %{email}. " +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:538 +msgctxt "digest email sending reason" +msgid "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:547 +msgctxt "digest email unsubscribe action" +msgid "To unsubscribe, please go %{here}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:547 +msgctxt "digest email unsubscribe action link text" +msgid "here" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex:1 +msgctxt "mailer unsubscribe failed message" +msgid "UNSUBSCRIBE FAILURE" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex:1 +msgctxt "mailer unsubscribe successful message" +msgid "UNSUBSCRIBE SUCCESSFUL" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:385 +msgctxt "new followers count header" +msgid "%{count} New Follower" +msgid_plural "%{count} New Followers" +msgstr[0] "" +msgstr[1] "" From 4ed0a780b72e822ede478e2233ee8a9e329ab8ce Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 1 Mar 2022 20:27:45 -0500 Subject: [PATCH 19/94] Fix digest test --- test/mix/tasks/pleroma/digest_test.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/mix/tasks/pleroma/digest_test.exs b/test/mix/tasks/pleroma/digest_test.exs index 4a9e461a9..b8050c7af 100644 --- a/test/mix/tasks/pleroma/digest_test.exs +++ b/test/mix/tasks/pleroma/digest_test.exs @@ -53,7 +53,13 @@ test "Sends digest to the given user" do assert_email_sent( to: {user2.name, user2.email}, - html_body: ~r/here is what you've missed!/i + html_body: + Regex.compile!( + "here is what you've missed!" + |> Phoenix.HTML.html_escape() + |> Phoenix.HTML.safe_to_string(), + "i" + ) ) end end From c301a95276fd1e66208a9f6ccdcf5e29f0be51e1 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 1 Mar 2022 20:29:26 -0500 Subject: [PATCH 20/94] Make all emails translatable --- lib/pleroma/emails/user_email.ex | 207 ++++++++++++++++++++++++------- priv/gettext/static_pages.pot | 102 +++++++++++++++ 2 files changed, 266 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index e38c681ba..cd06ab23c 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -5,9 +5,12 @@ defmodule Pleroma.Emails.UserEmail do @moduledoc "User emails" + require Pleroma.Web.Gettext + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.Endpoint + alias Pleroma.Web.Gettext alias Pleroma.Web.Router import Swoosh.Email @@ -30,25 +33,64 @@ def welcome(user, opts \\ %{}) do new() |> to(recipient(user)) |> from(Map.get(opts, :sender, sender())) - |> subject(Map.get(opts, :subject, "Welcome to #{instance_name()}!")) - |> html_body(Map.get(opts, :html, "Welcome to #{instance_name()}!")) - |> text_body(Map.get(opts, :text, "Welcome to #{instance_name()}!")) + |> subject( + Map.get( + opts, + :subject, + Gettext.dpgettext("static_pages", "welcome email subject", "Welcome to %{instance_name}!", + instance_name: instance_name() + ) + ) + ) + |> html_body( + Map.get( + opts, + :html, + Gettext.dpgettext( + "static_pages", + "welcome email html body", + "Welcome to %{instance_name}!", + instance_name: instance_name() + ) + ) + ) + |> text_body( + Map.get( + opts, + :text, + Gettext.dpgettext( + "static_pages", + "welcome email text body", + "Welcome to %{instance_name}!", + instance_name: instance_name() + ) + ) + ) end def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) - html_body = """ -

Reset your password at #{instance_name()}

-

Someone has requested password change for your account at #{instance_name()}.

-

If it was you, visit the following link to proceed: reset password.

-

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

- """ + html_body = + Gettext.dpgettext( + "static_pages", + "password reset email body", + """ +

Reset your password at %{instance_name}

+

Someone has requested password change for your account at %{instance_name}.

+

If it was you, visit the following link to proceed: reset password.

+

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

+ """, + instance_name: instance_name(), + password_reset_url: password_reset_url + ) new() |> to(recipient(user)) |> from(sender()) - |> subject("Password reset") + |> subject( + Gettext.dpgettext("static_pages", "password reset email subject", "Password reset") + ) |> html_body(html_body) end @@ -65,16 +107,31 @@ def user_invitation_email( user_invite_token.token ) - html_body = """ -

You are invited to #{instance_name()}

-

#{user.name} invites you to join #{instance_name()}, an instance of Pleroma federated social networking platform.

-

Click the following link to register: accept invitation.

- """ + html_body = + Gettext.dpgettext( + "static_pages", + "user invitation email body", + """ +

You are invited to %{instance_name}

+

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

+

Click the following link to register: accept invitation.

+ """, + instance_name: instance_name(), + inviter_name: user.name, + registration_url: registration_url + ) new() |> to(recipient(to_email, to_name)) |> from(sender()) - |> subject("Invitation to #{instance_name()}") + |> subject( + Gettext.dpgettext( + "static_pages", + "user invitation email subject", + "Invitation to %{instance_name}", + instance_name: instance_name() + ) + ) |> html_body(html_body) end @@ -87,43 +144,83 @@ def account_confirmation_email(user) do to_string(user.confirmation_token) ) - html_body = """ -

Thank you for registering on #{instance_name()}

-

Email confirmation is required to activate the account.

-

Please click the following link to activate your account.

- """ + html_body = + Gettext.dpgettext( + "static_pages", + "confirmation email body", + """ +

Thank you for registering on %{instance_name}

+

Email confirmation is required to activate the account.

+

Please click the following link to activate your account.

+ """, + instance_name: instance_name(), + confirmation_url: confirmation_url + ) new() |> to(recipient(user)) |> from(sender()) - |> subject("#{instance_name()} account confirmation") + |> subject( + Gettext.dpgettext( + "static_pages", + "confirmation email subject", + "%{instance_name} account confirmation", + instance_name: instance_name() + ) + ) |> html_body(html_body) end def approval_pending_email(user) do - html_body = """ -

Awaiting Approval

-

Your account at #{instance_name()} is being reviewed by staff. You will receive another email once your account is approved.

- """ + html_body = + Gettext.dpgettext( + "static_pages", + "approval pending email body", + """ +

Awaiting Approval

+

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

+ """, + instance_name: instance_name() + ) new() |> to(recipient(user)) |> from(sender()) - |> subject("Your account is awaiting approval") + |> subject( + Gettext.dpgettext( + "static_pages", + "approval pending email subject", + "Your account is awaiting approval" + ) + ) |> html_body(html_body) end def successful_registration_email(user) do - html_body = """ -

Hello @#{user.nickname},

-

Your account at #{instance_name()} has been registered successfully.

-

No further action is required to activate your account.

- """ + html_body = + Gettext.dpgettext( + "static_pages", + "successful registration email body", + """ +

Hello @%{nickname},

+

Your account at %{instance_name} has been registered successfully.

+

No further action is required to activate your account.

+ """, + nickname: user.nickname, + instance_name: instance_name() + ) new() |> to(recipient(user)) |> from(sender()) - |> subject("Account registered on #{instance_name()}") + |> subject( + Gettext.dpgettext( + "static_pages", + "successful registration email subject", + "Account registered on %{instance_name}", + instance_name: instance_name() + ) + ) |> html_body(html_body) end @@ -193,7 +290,14 @@ def digest_email(user) do new() |> to(recipient(user)) |> from(sender()) - |> subject("Your digest from #{instance_name()}") + |> subject( + Gettext.dpgettext( + "static_pages", + "digest email subject", + "Your digest from %{instance_name}", + instance_name: instance_name() + ) + ) |> put_layout(false) |> render_body("digest.html", html_data) |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline)) @@ -230,23 +334,40 @@ def backup_is_ready_email(backup, admin_user_id \\ nil) do html_body = if is_nil(admin_user_id) do - """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

#{download_url}

- """ + Gettext.dpgettext( + "static_pages", + "account archive email body - self-requested", + """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

%{download_url}

+ """, + download_url: download_url + ) else admin = Pleroma.Repo.get(User, admin_user_id) - """ -

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

-

#{download_url}

- """ + Gettext.dpgettext( + "static_pages", + "account archive email body - admin requested", + """ +

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

+

%{download_url}

+ """, + admin_nickname: admin.nickname, + download_url: download_url + ) end new() |> to(recipient(user)) |> from(sender()) - |> subject("Your account archive is ready") + |> subject( + Gettext.dpgettext( + "static_pages", + "account archive email subject", + "Your account archive is ready" + ) + ) |> html_body(html_body) end end diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index b76641e28..a14cedae9 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -409,3 +409,105 @@ msgid "%{count} New Follower" msgid_plural "%{count} New Followers" msgstr[0] "" msgstr[1] "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:349 +msgctxt "account archive email body - admin requested" +msgid "

Admin @%{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:337 +msgctxt "account archive email body - self-requested" +msgid "

You requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:365 +msgctxt "account archive email subject" +msgid "Your account archive is ready" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:176 +msgctxt "approval pending email body" +msgid "

Awaiting Approval

\n

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:190 +msgctxt "approval pending email subject" +msgid "Your account is awaiting approval" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:148 +msgctxt "confirmation email body" +msgid "

Thank you for registering on %{instance_name}

\n

Email confirmation is required to activate the account.

\n

Please click the following link to activate your account.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:164 +msgctxt "confirmation email subject" +msgid "%{instance_name} account confirmation" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:294 +msgctxt "digest email subject" +msgid "Your digest from %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:75 +msgctxt "password reset email body" +msgid "

Reset your password at %{instance_name}

\n

Someone has requested password change for your account at %{instance_name}.

\n

If it was you, visit the following link to proceed: reset password.

\n

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:92 +msgctxt "password reset email subject" +msgid "Password reset" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:201 +msgctxt "successful registration email body" +msgid "

Hello @%{nickname},

\n

Your account at %{instance_name} has been registered successfully.

\n

No further action is required to activate your account.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:217 +msgctxt "successful registration email subject" +msgid "Account registered on %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:111 +msgctxt "user invitation email body" +msgid "

You are invited to %{instance_name}

\n

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

\n

Click the following link to register: accept invitation.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:128 +msgctxt "user invitation email subject" +msgid "Invitation to %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:49 +msgctxt "welcome email html body" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:40 +msgctxt "welcome email subject" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:61 +msgctxt "welcome email text body" +msgid "Welcome to %{instance_name}!" +msgstr "" From 7726148472cd142b2053172abf61b80c10a98840 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 1 Mar 2022 21:24:17 -0500 Subject: [PATCH 21/94] Send emails i18n'd using backend-stored user language --- lib/pleroma/emails/user_email.ex | 516 ++++++++-------- lib/pleroma/user.ex | 1 + lib/pleroma/web/gettext.ex | 22 + lib/pleroma/web/plugs/set_locale_plug.ex | 4 +- priv/gettext/en_test/LC_MESSAGES/default.po | 186 ++++++ priv/gettext/en_test/LC_MESSAGES/errors.po | 557 ++++++++++++++++++ .../en_test/LC_MESSAGES/posix_errors.po | 153 +++++ .../en_test/LC_MESSAGES/static_pages.po | 529 +++++++++++++++++ priv/gettext/static_pages.pot | 44 +- .../20220302013920_add_language_to_users.exs | 9 + test/pleroma/emails/user_email_test.exs | 12 + 11 files changed, 1760 insertions(+), 273 deletions(-) create mode 100644 priv/gettext/en_test/LC_MESSAGES/default.po create mode 100644 priv/gettext/en_test/LC_MESSAGES/errors.po create mode 100644 priv/gettext/en_test/LC_MESSAGES/posix_errors.po create mode 100644 priv/gettext/en_test/LC_MESSAGES/static_pages.po create mode 100644 priv/repo/migrations/20220302013920_add_language_to_users.exs diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index cd06ab23c..24adfabd7 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -30,68 +30,75 @@ defp recipient(%User{} = user), do: recipient(user.email, user.name) @spec welcome(User.t(), map()) :: Swoosh.Email.t() def welcome(user, opts \\ %{}) do - new() - |> to(recipient(user)) - |> from(Map.get(opts, :sender, sender())) - |> subject( - Map.get( - opts, - :subject, - Gettext.dpgettext("static_pages", "welcome email subject", "Welcome to %{instance_name}!", - instance_name: instance_name() + Gettext.with_locale_or_default user.language do + new() + |> to(recipient(user)) + |> from(Map.get(opts, :sender, sender())) + |> subject( + Map.get( + opts, + :subject, + Gettext.dpgettext( + "static_pages", + "welcome email subject", + "Welcome to %{instance_name}!", + instance_name: instance_name() + ) ) ) - ) - |> html_body( - Map.get( - opts, - :html, - Gettext.dpgettext( - "static_pages", - "welcome email html body", - "Welcome to %{instance_name}!", - instance_name: instance_name() + |> html_body( + Map.get( + opts, + :html, + Gettext.dpgettext( + "static_pages", + "welcome email html body", + "Welcome to %{instance_name}!", + instance_name: instance_name() + ) ) ) - ) - |> text_body( - Map.get( - opts, - :text, - Gettext.dpgettext( - "static_pages", - "welcome email text body", - "Welcome to %{instance_name}!", - instance_name: instance_name() + |> text_body( + Map.get( + opts, + :text, + Gettext.dpgettext( + "static_pages", + "welcome email text body", + "Welcome to %{instance_name}!", + instance_name: instance_name() + ) ) ) - ) + end end def password_reset_email(user, token) when is_binary(token) do - password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) + Gettext.with_locale_or_default user.language do + password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) - html_body = - Gettext.dpgettext( - "static_pages", - "password reset email body", - """ -

Reset your password at %{instance_name}

-

Someone has requested password change for your account at %{instance_name}.

-

If it was you, visit the following link to proceed: reset password.

-

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

- """, - instance_name: instance_name(), - password_reset_url: password_reset_url + html_body = + Gettext.dpgettext( + "static_pages", + "password reset email body", + """ +

Reset your password at %{instance_name}

+

Someone has requested password change for your account at %{instance_name}.

+

If it was you, visit the following link to proceed: reset password.

+

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

+ """, + instance_name: instance_name(), + password_reset_url: password_reset_url + ) + + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( + Gettext.dpgettext("static_pages", "password reset email subject", "Password reset") ) - - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext("static_pages", "password reset email subject", "Password reset") - ) - |> html_body(html_body) + |> html_body(html_body) + end end def user_invitation_email( @@ -100,128 +107,136 @@ def user_invitation_email( to_email, to_name \\ nil ) do - registration_url = - Router.Helpers.redirect_url( - Endpoint, - :registration_page, - user_invite_token.token - ) + Gettext.with_locale_or_default user.language do + registration_url = + Router.Helpers.redirect_url( + Endpoint, + :registration_page, + user_invite_token.token + ) - html_body = - Gettext.dpgettext( - "static_pages", - "user invitation email body", - """ -

You are invited to %{instance_name}

-

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

-

Click the following link to register: accept invitation.

- """, - instance_name: instance_name(), - inviter_name: user.name, - registration_url: registration_url - ) + html_body = + Gettext.dpgettext( + "static_pages", + "user invitation email body", + """ +

You are invited to %{instance_name}

+

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

+

Click the following link to register: accept invitation.

+ """, + instance_name: instance_name(), + inviter_name: user.name, + registration_url: registration_url + ) - new() - |> to(recipient(to_email, to_name)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "user invitation email subject", - "Invitation to %{instance_name}", - instance_name: instance_name() + new() + |> to(recipient(to_email, to_name)) + |> from(sender()) + |> subject( + Gettext.dpgettext( + "static_pages", + "user invitation email subject", + "Invitation to %{instance_name}", + instance_name: instance_name() + ) ) - ) - |> html_body(html_body) + |> html_body(html_body) + end end def account_confirmation_email(user) do - confirmation_url = - Router.Helpers.confirm_email_url( - Endpoint, - :confirm_email, - user.id, - to_string(user.confirmation_token) - ) + Gettext.with_locale_or_default user.language do + confirmation_url = + Router.Helpers.confirm_email_url( + Endpoint, + :confirm_email, + user.id, + to_string(user.confirmation_token) + ) - html_body = - Gettext.dpgettext( - "static_pages", - "confirmation email body", - """ -

Thank you for registering on %{instance_name}

-

Email confirmation is required to activate the account.

-

Please click the following link to activate your account.

- """, - instance_name: instance_name(), - confirmation_url: confirmation_url - ) + html_body = + Gettext.dpgettext( + "static_pages", + "confirmation email body", + """ +

Thank you for registering on %{instance_name}

+

Email confirmation is required to activate the account.

+

Please click the following link to activate your account.

+ """, + instance_name: instance_name(), + confirmation_url: confirmation_url + ) - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "confirmation email subject", - "%{instance_name} account confirmation", - instance_name: instance_name() + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( + Gettext.dpgettext( + "static_pages", + "confirmation email subject", + "%{instance_name} account confirmation", + instance_name: instance_name() + ) ) - ) - |> html_body(html_body) + |> html_body(html_body) + end end def approval_pending_email(user) do - html_body = - Gettext.dpgettext( - "static_pages", - "approval pending email body", - """ -

Awaiting Approval

-

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

- """, - instance_name: instance_name() - ) + Gettext.with_locale_or_default user.language do + html_body = + Gettext.dpgettext( + "static_pages", + "approval pending email body", + """ +

Awaiting Approval

+

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

+ """, + instance_name: instance_name() + ) - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "approval pending email subject", - "Your account is awaiting approval" + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( + Gettext.dpgettext( + "static_pages", + "approval pending email subject", + "Your account is awaiting approval" + ) ) - ) - |> html_body(html_body) + |> html_body(html_body) + end end def successful_registration_email(user) do - html_body = - Gettext.dpgettext( - "static_pages", - "successful registration email body", - """ -

Hello @%{nickname},

-

Your account at %{instance_name} has been registered successfully.

-

No further action is required to activate your account.

- """, - nickname: user.nickname, - instance_name: instance_name() - ) + Gettext.with_locale_or_default user.language do + html_body = + Gettext.dpgettext( + "static_pages", + "successful registration email body", + """ +

Hello @%{nickname},

+

Your account at %{instance_name} has been registered successfully.

+

No further action is required to activate your account.

+ """, + nickname: user.nickname, + instance_name: instance_name() + ) - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "successful registration email subject", - "Account registered on %{instance_name}", - instance_name: instance_name() + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( + Gettext.dpgettext( + "static_pages", + "successful registration email subject", + "Account registered on %{instance_name}", + instance_name: instance_name() + ) ) - ) - |> html_body(html_body) + |> html_body(html_body) + end end @doc """ @@ -231,76 +246,78 @@ def successful_registration_email(user) do """ @spec digest_email(User.t()) :: Swoosh.Email.t() | nil def digest_email(user) do - notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) + Gettext.with_locale_or_default user.language do + notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - mentions = - notifications - |> Enum.filter(&(&1.activity.data["type"] == "Create")) - |> Enum.map(fn notification -> - object = Pleroma.Object.normalize(notification.activity, fetch: false) + mentions = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Create")) + |> Enum.map(fn notification -> + object = Pleroma.Object.normalize(notification.activity, fetch: false) - if not is_nil(object) do - object = update_in(object.data["content"], &format_links/1) + if not is_nil(object) do + object = update_in(object.data["content"], &format_links/1) - %{ - data: notification, - object: object, - from: User.get_by_ap_id(notification.activity.actor) - } - end - end) - |> Enum.filter(& &1) + %{ + data: notification, + object: object, + from: User.get_by_ap_id(notification.activity.actor) + } + end + end) + |> Enum.filter(& &1) - followers = - notifications - |> Enum.filter(&(&1.activity.data["type"] == "Follow")) - |> Enum.map(fn notification -> - from = User.get_by_ap_id(notification.activity.actor) + followers = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Follow")) + |> Enum.map(fn notification -> + from = User.get_by_ap_id(notification.activity.actor) - if not is_nil(from) do - %{ - data: notification, - object: Pleroma.Object.normalize(notification.activity, fetch: false), - from: User.get_by_ap_id(notification.activity.actor) - } - end - end) - |> Enum.filter(& &1) + if not is_nil(from) do + %{ + data: notification, + object: Pleroma.Object.normalize(notification.activity, fetch: false), + from: User.get_by_ap_id(notification.activity.actor) + } + end + end) + |> Enum.filter(& &1) - unless Enum.empty?(mentions) do - styling = Config.get([__MODULE__, :styling]) - logo = Config.get([__MODULE__, :logo]) + unless Enum.empty?(mentions) do + styling = Config.get([__MODULE__, :styling]) + logo = Config.get([__MODULE__, :logo]) - html_data = %{ - instance: instance_name(), - user: user, - mentions: mentions, - followers: followers, - unsubscribe_link: unsubscribe_url(user, "digest"), - styling: styling - } + html_data = %{ + instance: instance_name(), + user: user, + mentions: mentions, + followers: followers, + unsubscribe_link: unsubscribe_url(user, "digest"), + styling: styling + } - logo_path = - if is_nil(logo) do - Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg") - else - Path.join(Config.get([:instance, :static_dir]), logo) - end + logo_path = + if is_nil(logo) do + Path.join(:code.priv_dir(:pleroma), "static/static/logo.svg") + else + Path.join(Config.get([:instance, :static_dir]), logo) + end - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "digest email subject", - "Your digest from %{instance_name}", - instance_name: instance_name() + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( + Gettext.dpgettext( + "static_pages", + "digest email subject", + "Your digest from %{instance_name}", + instance_name: instance_name() + ) ) - ) - |> put_layout(false) - |> render_body("digest.html", html_data) - |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline)) + |> put_layout(false) + |> render_body("digest.html", html_data) + |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.svg", type: :inline)) + end end end @@ -330,44 +347,47 @@ def unsubscribe_url(user, notifications_type) do def backup_is_ready_email(backup, admin_user_id \\ nil) do %{user: user} = Pleroma.Repo.preload(backup, :user) - download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) - html_body = - if is_nil(admin_user_id) do + Gettext.with_locale_or_default user.language do + download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) + + html_body = + if is_nil(admin_user_id) do + Gettext.dpgettext( + "static_pages", + "account archive email body - self-requested", + """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

%{download_url}

+ """, + download_url: download_url + ) + else + admin = Pleroma.Repo.get(User, admin_user_id) + + Gettext.dpgettext( + "static_pages", + "account archive email body - admin requested", + """ +

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

+

%{download_url}

+ """, + admin_nickname: admin.nickname, + download_url: download_url + ) + end + + new() + |> to(recipient(user)) + |> from(sender()) + |> subject( Gettext.dpgettext( "static_pages", - "account archive email body - self-requested", - """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

%{download_url}

- """, - download_url: download_url + "account archive email subject", + "Your account archive is ready" ) - else - admin = Pleroma.Repo.get(User, admin_user_id) - - Gettext.dpgettext( - "static_pages", - "account archive email body - admin requested", - """ -

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

-

%{download_url}

- """, - admin_nickname: admin.nickname, - download_url: download_url - ) - end - - new() - |> to(recipient(user)) - |> from(sender()) - |> subject( - Gettext.dpgettext( - "static_pages", - "account archive email subject", - "Your account archive is ready" ) - ) - |> html_body(html_body) + |> html_body(html_body) + end end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 27ed9bbc5..70afba3ae 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -151,6 +151,7 @@ defmodule Pleroma.User do field(:pinned_objects, :map, default: %{}) field(:is_suggested, :boolean, default: false) field(:last_status_at, :naive_datetime) + field(:language, :string) embeds_one( :notification_settings, diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index c8a739c2b..e85290496 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -34,4 +34,26 @@ def language_tag do Gettext.get_locale() |> String.replace("_", "-", global: true) end + + def supports_locale?(locale) do + Pleroma.Web.Gettext + |> Gettext.known_locales() + |> Enum.member?(locale) + end + + def locale_or_default(locale) do + if supports_locale?(locale) do + locale + else + Gettext.get_locale() + end + end + + defmacro with_locale_or_default(locale, do: fun) do + quote do + Gettext.with_locale(Pleroma.Web.Gettext.locale_or_default(unquote(locale)), fn -> + unquote(fun) + end) + end + end end diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index a9387ba7e..3c3fffa81 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -64,9 +64,7 @@ defp extract_accept_language(conn) do end defp supported_locale?(locale) do - Pleroma.Web.Gettext - |> Gettext.known_locales() - |> Enum.member?(locale) + Pleroma.Web.Gettext.supports_locale?(locale) end defp parse_language_option(string) do diff --git a/priv/gettext/en_test/LC_MESSAGES/default.po b/priv/gettext/en_test/LC_MESSAGES/default.po new file mode 100644 index 000000000..63db74608 --- /dev/null +++ b/priv/gettext/en_test/LC_MESSAGES/default.po @@ -0,0 +1,186 @@ +## "msgid"s in this file come from POT (.pot) files. +## +## Do not add, change, or remove "msgid"s manually here as +## they're tied to the ones in the corresponding POT file +## (with the same domain). +## +## Use "mix gettext.extract --merge" or "mix gettext.merge" +## to merge POT files into PO files. +msgid "" +msgstr "" +"Language: en_test\n" +"Plural-Forms: nplurals=2\n" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:122 +msgid "%{name} - %{count} is not a multiple of %{multiple}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:131 +msgid "%{name} - %{value} is larger than exclusive maximum %{max}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:140 +msgid "%{name} - %{value} is larger than inclusive maximum %{max}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:149 +msgid "%{name} - %{value} is smaller than exclusive minimum %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:158 +msgid "%{name} - %{value} is smaller than inclusive minimum %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:102 +msgid "%{name} - Array items must be unique." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:114 +msgid "%{name} - Array length %{length} is larger than maxItems: %{}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:106 +msgid "%{name} - Array length %{length} is smaller than minItems: %{min}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:166 +msgid "%{name} - Invalid %{type}. Got: %{value}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:174 +msgid "%{name} - Invalid format. Expected %{format}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:51 +msgid "%{name} - Invalid schema.type. Got: %{type}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:178 +msgid "%{name} - Invalid value for enum." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:95 +msgid "%{name} - String length is larger than maxLength: %{length}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:88 +msgid "%{name} - String length is smaller than minLength: %{length}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:63 +msgid "%{name} - null value where %{type} expected." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:60 +msgid "%{name} - null value." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:182 +msgid "Failed to cast to any schema in %{polymorphic_type}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:71 +msgid "Failed to cast value as %{invalid_schema}. Value must be castable using `allOf` schemas listed." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:84 +msgid "Failed to cast value to one of: %{failed_schemas}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:78 +msgid "Failed to cast value using any of: %{failed_schemas}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:212 +msgid "Invalid value for header: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:204 +msgid "Missing field: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:208 +msgid "Missing header: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:196 +msgid "No value provided for required discriminator `%{field}`." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:216 +msgid "Object property count %{property_count} is greater than maxProperties: %{max_properties}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:224 +msgid "Object property count %{property_count} is less than minProperties: %{min_properties}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/static_fe/static_fe/error.html.eex:2 +msgid "Oops" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:188 +msgid "Unexpected field: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:200 +msgid "Unknown schema: %{name}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/api_spec/render_error.ex:192 +msgid "Value used as discriminator for `%{field}` matches no schemas." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:43 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:37 +msgid "announces" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:44 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:38 +msgid "likes" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:42 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:36 +msgid "replies" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/embed/show.html.eex:27 +#: lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex:22 +msgid "sensitive media" +msgstr "" diff --git a/priv/gettext/en_test/LC_MESSAGES/errors.po b/priv/gettext/en_test/LC_MESSAGES/errors.po new file mode 100644 index 000000000..a40de7f8b --- /dev/null +++ b/priv/gettext/en_test/LC_MESSAGES/errors.po @@ -0,0 +1,557 @@ +## "msgid"s in this file come from POT (.pot) files. +## +## Do not add, change, or remove "msgid"s manually here as +## they're tied to the ones in the corresponding POT file +## (with the same domain). +## +## Use "mix gettext.extract --merge" or "mix gettext.merge" +## to merge POT files into PO files. +msgid "" +msgstr "" +"Language: en_test\n" +"Plural-Forms: nplurals=2\n" + +msgid "can't be blank" +msgstr "" + +msgid "has already been taken" +msgstr "" + +msgid "is invalid" +msgstr "" + +msgid "has invalid format" +msgstr "" + +msgid "has an invalid entry" +msgstr "" + +msgid "is reserved" +msgstr "" + +msgid "does not match confirmation" +msgstr "" + +msgid "is still associated with this entry" +msgstr "" + +msgid "are still associated with this entry" +msgstr "" + +msgid "should be %{count} character(s)" +msgid_plural "should be %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have %{count} item(s)" +msgid_plural "should have %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at least %{count} character(s)" +msgid_plural "should be at least %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at least %{count} item(s)" +msgid_plural "should have at least %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at most %{count} character(s)" +msgid_plural "should be at most %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at most %{count} item(s)" +msgid_plural "should have at most %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "must be less than %{number}" +msgstr "" + +msgid "must be greater than %{number}" +msgstr "" + +msgid "must be less than or equal to %{number}" +msgstr "" + +msgid "must be greater than or equal to %{number}" +msgstr "" + +msgid "must be equal to %{number}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:523 +msgid "Account not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:316 +msgid "Already voted" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:402 +msgid "Bad request" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/controller_helper.ex:97 +#: lib/pleroma/web/controller_helper.ex:103 +msgid "Can't display this activity" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:324 +msgid "Can't find user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:80 +msgid "Can't get favorites" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:482 +msgid "Cannot post an empty status without attachments" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:441 +msgid "Comment must be up to %{max_size} characters" +msgstr "" + +#, elixir-format +#: lib/pleroma/config_db.ex:200 +msgid "Config with params %{params} not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:167 lib/pleroma/web/common_api.ex:171 +msgid "Could not delete" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:217 +msgid "Could not favorite" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:254 +msgid "Could not unfavorite" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:202 +msgid "Could not unrepeat" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:530 lib/pleroma/web/common_api.ex:539 +msgid "Could not update state" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:205 +msgid "Error." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:99 +msgid "Invalid CAPTCHA" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:144 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:631 +msgid "Invalid credentials" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_authenticated_plug.ex:42 +msgid "Invalid credentials." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:337 +msgid "Invalid indices" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:29 +msgid "Invalid parameters" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:349 +msgid "Invalid password." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:254 +msgid "Invalid request" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:102 +msgid "Kocaptcha service unavailable" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:140 +msgid "Missing parameters" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:477 +msgid "No such conversation" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:171 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:197 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:239 +msgid "No such permission_group" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:504 +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:11 lib/pleroma/web/feed/tag_controller.ex:16 +#: lib/pleroma/web/feed/user_controller.ex:69 lib/pleroma/web/o_status/o_status_controller.ex:132 +#: lib/pleroma/web/plugs/uploaded_media.ex:84 +msgid "Not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:308 +msgid "Poll's author can't vote" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:52 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:326 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71 +msgid "Record not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:35 +#: lib/pleroma/web/feed/user_controller.ex:78 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:42 +#: lib/pleroma/web/o_status/o_status_controller.ex:138 +msgid "Something went wrong" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/activity_draft.ex:143 +msgid "The message visibility must be direct" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:492 +msgid "The status is over the character limit" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_public_or_authenticated_plug.ex:36 +msgid "This resource requires authentication." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/rate_limiter.ex:208 +msgid "Throttled" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:338 +msgid "Too many choices" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:268 +msgid "You can't revoke your own admin status." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:243 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:333 +msgid "Your account is currently disabled" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:205 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:356 +msgid "Your login is missing a confirmed e-mail address" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:392 +msgid "can't read inbox of %{nickname} as %{as_nickname}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:491 +msgid "can't update outbox of %{nickname} as %{as_nickname}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:475 +msgid "conversation is already muted" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:510 +msgid "error" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:34 +msgid "mascots can only be images" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:63 +msgid "not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:437 +msgid "Bad OAuth request." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:108 +msgid "CAPTCHA already used" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:105 +msgid "CAPTCHA expired" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/uploaded_media.ex:57 +msgid "Failed" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:453 +msgid "Failed to authenticate: %{message}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:484 +msgid "Failed to set up user account." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/o_auth_scopes_plug.ex:37 +msgid "Insufficient permissions: %{permissions}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/uploaded_media.ex:111 +msgid "Internal Error" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/fallback_controller.ex:22 +#: lib/pleroma/web/o_auth/fallback_controller.ex:29 +msgid "Invalid Username/Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:111 +msgid "Invalid answer data" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:33 +msgid "Nodeinfo schema version not handled" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:194 +msgid "This action is outside the authorized scopes" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/fallback_controller.ex:14 +msgid "Unknown error, please check the details and try again." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:136 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:180 +msgid "Unlisted redirect_uri." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:433 +msgid "Unsupported OAuth provider: %{provider}." +msgstr "" + +#, elixir-format +#: lib/pleroma/uploaders/uploader.ex:74 +msgid "Uploader callback timeout" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/uploader_controller.ex:23 +msgid "bad request" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:96 +msgid "CAPTCHA Error" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:266 +msgid "Could not add reaction emoji" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api.ex:277 +msgid "Could not remove reaction emoji" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/twitter_api/twitter_api.ex:122 +msgid "Invalid CAPTCHA (Missing parameter: %{name})" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:96 +msgid "List not found" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:151 +msgid "Missing parameter: %{name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:232 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:346 +msgid "Password reset is required" +msgstr "" + +#, elixir-format +#: lib/pleroma/tests/auth_test_controller.ex:9 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/chat_controller.ex:6 lib/pleroma/web/admin_api/controllers/config_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:6 lib/pleroma/web/admin_api/controllers/frontend_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/instance_controller.ex:6 lib/pleroma/web/admin_api/controllers/instance_document_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/invite_controller.ex:6 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/o_auth_app_controller.ex:6 lib/pleroma/web/admin_api/controllers/relay_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/report_controller.ex:6 lib/pleroma/web/admin_api/controllers/status_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/user_controller.ex:6 lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/embed_controller.ex:6 +#: lib/pleroma/web/fallback/redirect_controller.ex:6 lib/pleroma/web/feed/tag_controller.ex:6 +#: lib/pleroma/web/feed/user_controller.ex:6 lib/pleroma/web/mailer/subscription_controller.ex:6 +#: lib/pleroma/web/manifest_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/app_controller.ex:11 lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/conversation_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/custom_emoji_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/directory_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/report_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 lib/pleroma/web/media_proxy/media_proxy_controller.ex:6 +#: lib/pleroma/web/mongoose_im/mongoose_im_controller.ex:6 lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6 +#: lib/pleroma/web/o_auth/fallback_controller.ex:6 lib/pleroma/web/o_auth/mfa_controller.ex:10 +#: lib/pleroma/web/o_auth/o_auth_controller.ex:6 lib/pleroma/web/o_status/o_status_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/app_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/backup_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex:5 +#: lib/pleroma/web/pleroma_api/controllers/conversation_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/instances_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/notification_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/report_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex:6 +#: lib/pleroma/web/static_fe/static_fe_controller.ex:6 lib/pleroma/web/twitter_api/controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/password_controller.ex:10 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/uploader_controller.ex:6 +#: lib/pleroma/web/web_finger/web_finger_controller.ex:6 +msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_authenticated_plug.ex:32 +msgid "Two-factor authentication enabled, you must use a access token." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 +msgid "Web push subscription is disabled on this Pleroma instance" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:234 +msgid "You can't revoke your own admin/moderator status." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:129 +msgid "authorization required for timeline view" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:24 +msgid "Access denied" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:321 +msgid "This API requires an authenticated user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_staff_privileged_plug.ex:26 +#: lib/pleroma/web/plugs/user_is_admin_plug.ex:21 +msgid "User is not an admin." +msgstr "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:75 +msgid "Last export was less than a day ago" +msgid_plural "Last export was less than %{days} days ago" +msgstr[0] "" +msgstr[1] "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:93 +msgid "Backups require enabled email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:423 +msgid "Character limit (%{limit} characters) exceeded, contains %{length} characters" +msgstr "" + +#, elixir-format +#: lib/pleroma/user/backup.ex:98 +msgid "Email is required" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/common_api/utils.ex:507 +msgid "Too many attachments" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/plugs/ensure_staff_privileged_plug.ex:33 +#: lib/pleroma/web/plugs/user_is_staff_plug.ex:20 +msgid "User is not a staff member." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/o_auth/o_auth_controller.ex:366 +msgid "Your account is awaiting approval." +msgstr "" diff --git a/priv/gettext/en_test/LC_MESSAGES/posix_errors.po b/priv/gettext/en_test/LC_MESSAGES/posix_errors.po new file mode 100644 index 000000000..663fc5924 --- /dev/null +++ b/priv/gettext/en_test/LC_MESSAGES/posix_errors.po @@ -0,0 +1,153 @@ +## "msgid"s in this file come from POT (.pot) files. +## +## Do not add, change, or remove "msgid"s manually here as +## they're tied to the ones in the corresponding POT file +## (with the same domain). +## +## Use "mix gettext.extract --merge" or "mix gettext.merge" +## to merge POT files into PO files. +msgid "" +msgstr "" +"Language: en_test\n" +"Plural-Forms: nplurals=2\n" + +msgid "eperm" +msgstr "" + +msgid "eacces" +msgstr "" + +msgid "eagain" +msgstr "" + +msgid "ebadf" +msgstr "" + +msgid "ebadmsg" +msgstr "" + +msgid "ebusy" +msgstr "" + +msgid "edeadlk" +msgstr "" + +msgid "edeadlock" +msgstr "" + +msgid "edquot" +msgstr "" + +msgid "eexist" +msgstr "" + +msgid "efault" +msgstr "" + +msgid "efbig" +msgstr "" + +msgid "eftype" +msgstr "" + +msgid "eintr" +msgstr "" + +msgid "einval" +msgstr "" + +msgid "eio" +msgstr "" + +msgid "eisdir" +msgstr "" + +msgid "eloop" +msgstr "" + +msgid "emfile" +msgstr "" + +msgid "emlink" +msgstr "" + +msgid "emultihop" +msgstr "" + +msgid "enametoolong" +msgstr "" + +msgid "enfile" +msgstr "" + +msgid "enobufs" +msgstr "" + +msgid "enodev" +msgstr "" + +msgid "enolck" +msgstr "" + +msgid "enolink" +msgstr "" + +msgid "enoent" +msgstr "" + +msgid "enomem" +msgstr "" + +msgid "enospc" +msgstr "" + +msgid "enosr" +msgstr "" + +msgid "enostr" +msgstr "" + +msgid "enosys" +msgstr "" + +msgid "enotblk" +msgstr "" + +msgid "enotdir" +msgstr "" + +msgid "enotsup" +msgstr "" + +msgid "enxio" +msgstr "" + +msgid "eopnotsupp" +msgstr "" + +msgid "eoverflow" +msgstr "" + +msgid "epipe" +msgstr "" + +msgid "erange" +msgstr "" + +msgid "erofs" +msgstr "" + +msgid "espipe" +msgstr "" + +msgid "esrch" +msgstr "" + +msgid "estale" +msgstr "" + +msgid "etxtbsy" +msgstr "" + +msgid "exdev" +msgstr "" diff --git a/priv/gettext/en_test/LC_MESSAGES/static_pages.po b/priv/gettext/en_test/LC_MESSAGES/static_pages.po new file mode 100644 index 000000000..a3378089c --- /dev/null +++ b/priv/gettext/en_test/LC_MESSAGES/static_pages.po @@ -0,0 +1,529 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR Free Software Foundation, Inc. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"PO-Revision-Date: 2022-03-01 21:15-0500\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#~ ## "msgid"s in this file come from POT (.pot) files. +#~ ## +#~ ## Do not add, change, or remove "msgid"s manually here as +#~ ## they're tied to the ones in the corresponding POT file +#~ ## (with the same domain). +#~ ## +#~ ## Use "mix gettext.extract --merge" or "mix gettext.merge" +#~ ## to merge POT files into PO files. +#~ msgid "" +#~ msgstr "" +#~ "Language: en_test\n" +#~ "Plural-Forms: nplurals=2\n" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:9 +msgctxt "remote follow authorization button" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:2 +msgctxt "remote follow error" +msgid "Error fetching user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex:4 +msgctxt "remote follow header" +msgid "Remote follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:8 +msgctxt "placeholder text for auth code entry" +msgid "Authentication code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:10 +msgctxt "placeholder text for password entry" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:8 +msgctxt "placeholder text for username entry" +msgid "Username" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:13 +msgctxt "remote follow authorization button for login" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:12 +msgctxt "remote follow authorization button for mfa" +msgid "Authorize" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:2 +msgctxt "remote follow error" +msgid "Error following account" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex:4 +msgctxt "remote follow header, need login" +msgid "Log in to follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/follow_mfa.html.eex:4 +msgctxt "remote follow mfa header" +msgid "Two-factor authentication" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex:4 +msgctxt "remote follow success" +msgid "Account followed!" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:7 +msgctxt "placeholder text for account id" +msgid "Your account ID, e.g. lain@quitter.se" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:8 +msgctxt "remote follow authorization button for following with a remote account" +msgid "Follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:2 +msgctxt "remote follow error" +msgid "Error: %{error}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex:4 +msgctxt "remote follow header" +msgid "Remotely follow %{nickname}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:12 +msgctxt "password reset button" +msgid "Reset" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:4 +msgctxt "password reset failed homepage link" +msgid "Homepage" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_failed.html.eex:1 +msgctxt "password reset failed message" +msgid "Password reset failed" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:8 +msgctxt "password reset form confirm password prompt" +msgid "Confirmation" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset.html.eex:4 +msgctxt "password reset form password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/invalid_token.html.eex:1 +msgctxt "password reset invalid token message" +msgid "Invalid Token" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:2 +msgctxt "password reset successful homepage link" +msgid "Homepage" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/twitter_api/password/reset_success.html.eex:1 +msgctxt "password reset successful message" +msgid "Password changed!" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/feed/feed/tag.atom.eex:15 +#: lib/pleroma/web/templates/feed/feed/tag.rss.eex:7 +msgctxt "tag feed description" +msgid "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:1 +msgctxt "oauth authorization exists page title" +msgid "Authorization exists" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:32 +msgctxt "oauth authorize approve button" +msgid "Approve" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:30 +msgctxt "oauth authorize cancel button" +msgid "Cancel" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:23 +msgctxt "oauth authorize message" +msgid "Application %{client_name} is requesting access to your account." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:1 +msgctxt "oauth authorized page title" +msgid "Successfully authorized" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:1 +msgctxt "oauth external provider page title" +msgid "Sign in with external provider" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex:13 +msgctxt "oauth external provider sign in button" +msgid "Sign in with %{strategy}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:54 +msgctxt "oauth login button" +msgid "Log In" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:51 +msgctxt "oauth login password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:47 +msgctxt "oauth login username prompt" +msgid "Username" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:39 +msgctxt "oauth register nickname prompt" +msgid "Pleroma Handle" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:37 +msgctxt "oauth register nickname unchangeable warning" +msgid "Choose carefully! You won't be able to change this later. You will be able to change your display name, though." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:18 +msgctxt "oauth register page email prompt" +msgid "Email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:10 +msgctxt "oauth register page fill form prompt" +msgid "If you'd like to register a new account, please provide the details below." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:35 +msgctxt "oauth register page login button" +msgid "Proceed as existing user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:31 +msgctxt "oauth register page login password prompt" +msgid "Password" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:24 +msgctxt "oauth register page login prompt" +msgid "Alternatively, sign in to connect to existing account." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:27 +msgctxt "oauth register page login username prompt" +msgid "Name or email" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:14 +msgctxt "oauth register page nickname prompt" +msgid "Nickname" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:22 +msgctxt "oauth register page register button" +msgid "Proceed as new user" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/register.html.eex:8 +msgctxt "oauth register page title" +msgid "Registration Details" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/show.html.eex:36 +msgctxt "oauth register page title" +msgid "This is the first time you visit! Please enter your Pleroma handle." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex:2 +msgctxt "oauth scopes message" +msgid "The following permissions will be granted" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex:2 +#: lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex:2 +msgctxt "oauth token code message" +msgid "Token code is
%{token}" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:12 +msgctxt "mfa auth code prompt" +msgid "Authentication code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:8 +msgctxt "mfa auth page title" +msgid "Two-factor authentication" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:23 +msgctxt "mfa auth page use recovery code link" +msgid "Enter a two-factor recovery code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/totp.html.eex:20 +msgctxt "mfa auth verify code button" +msgid "Verify" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:8 +msgctxt "mfa recover page title" +msgid "Two-factor recovery" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:12 +msgctxt "mfa recover recovery code prompt" +msgid "Recovery code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:23 +msgctxt "mfa recover use 2fa code link" +msgid "Enter a two-factor code" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex:20 +msgctxt "mfa recover verify recovery code button" +msgid "Verify" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex:8 +msgctxt "static fe profile page remote follow button" +msgid "Remote follow" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:163 +msgctxt "digest email header line" +msgid "Hey %{nickname}, here is what you've missed!" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:544 +msgctxt "digest email receiver address" +msgid "The email address you are subscribed as is %{email}. " +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:538 +msgctxt "digest email sending reason" +msgid "You have received this email because you have signed up to receive digest emails from %{instance} Pleroma instance." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:547 +msgctxt "digest email unsubscribe action" +msgid "To unsubscribe, please go %{here}." +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:547 +msgctxt "digest email unsubscribe action link text" +msgid "here" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex:1 +msgctxt "mailer unsubscribe failed message" +msgid "UNSUBSCRIBE FAILURE" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex:1 +msgctxt "mailer unsubscribe successful message" +msgid "UNSUBSCRIBE SUCCESSFUL" +msgstr "" + +#, elixir-format +#: lib/pleroma/web/templates/email/digest.html.eex:385 +msgctxt "new followers count header" +msgid "%{count} New Follower" +msgid_plural "%{count} New Followers" +msgstr[0] "" +msgstr[1] "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:356 +msgctxt "account archive email body - self-requested" +msgid "

You requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:384 +msgctxt "account archive email subject" +msgid "Your account archive is ready" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:188 +msgctxt "approval pending email body" +msgid "

Awaiting Approval

\n

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:202 +msgctxt "approval pending email subject" +msgid "Your account is awaiting approval" +msgstr "xxYour account is awaiting approvalxx" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:158 +msgctxt "confirmation email body" +msgid "

Thank you for registering on %{instance_name}

\n

Email confirmation is required to activate the account.

\n

Please click the following link to activate your account.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:174 +msgctxt "confirmation email subject" +msgid "%{instance_name} account confirmation" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:310 +msgctxt "digest email subject" +msgid "Your digest from %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:81 +msgctxt "password reset email body" +msgid "

Reset your password at %{instance_name}

\n

Someone has requested password change for your account at %{instance_name}.

\n

If it was you, visit the following link to proceed: reset password.

\n

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:98 +msgctxt "password reset email subject" +msgid "Password reset" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:215 +msgctxt "successful registration email body" +msgid "

Hello @%{nickname},

\n

Your account at %{instance_name} has been registered successfully.

\n

No further action is required to activate your account.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:231 +msgctxt "successful registration email subject" +msgid "Account registered on %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:119 +msgctxt "user invitation email body" +msgid "

You are invited to %{instance_name}

\n

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

\n

Click the following link to register: accept invitation.

\n" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:136 +msgctxt "user invitation email subject" +msgid "Invitation to %{instance_name}" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:53 +msgctxt "welcome email html body" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:41 +msgctxt "welcome email subject" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:65 +msgctxt "welcome email text body" +msgid "Welcome to %{instance_name}!" +msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:368 +msgctxt "account archive email body - admin requested" +msgid "

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" diff --git a/priv/gettext/static_pages.pot b/priv/gettext/static_pages.pot index a14cedae9..fbc3e61a3 100644 --- a/priv/gettext/static_pages.pot +++ b/priv/gettext/static_pages.pot @@ -411,103 +411,103 @@ msgstr[0] "" msgstr[1] "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:349 -msgctxt "account archive email body - admin requested" -msgid "

Admin @%{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" -msgstr "" - -#, elixir-format -#: lib/pleroma/emails/user_email.ex:337 +#: lib/pleroma/emails/user_email.ex:356 msgctxt "account archive email body - self-requested" msgid "

You requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:365 +#: lib/pleroma/emails/user_email.ex:384 msgctxt "account archive email subject" msgid "Your account archive is ready" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:176 +#: lib/pleroma/emails/user_email.ex:188 msgctxt "approval pending email body" msgid "

Awaiting Approval

\n

Your account at %{instance_name} is being reviewed by staff. You will receive another email once your account is approved.

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:190 +#: lib/pleroma/emails/user_email.ex:202 msgctxt "approval pending email subject" msgid "Your account is awaiting approval" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:148 +#: lib/pleroma/emails/user_email.ex:158 msgctxt "confirmation email body" msgid "

Thank you for registering on %{instance_name}

\n

Email confirmation is required to activate the account.

\n

Please click the following link to activate your account.

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:164 +#: lib/pleroma/emails/user_email.ex:174 msgctxt "confirmation email subject" msgid "%{instance_name} account confirmation" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:294 +#: lib/pleroma/emails/user_email.ex:310 msgctxt "digest email subject" msgid "Your digest from %{instance_name}" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:75 +#: lib/pleroma/emails/user_email.ex:81 msgctxt "password reset email body" msgid "

Reset your password at %{instance_name}

\n

Someone has requested password change for your account at %{instance_name}.

\n

If it was you, visit the following link to proceed: reset password.

\n

If it was someone else, nothing to worry about: your data is secure and your password has not been changed.

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:92 +#: lib/pleroma/emails/user_email.ex:98 msgctxt "password reset email subject" msgid "Password reset" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:201 +#: lib/pleroma/emails/user_email.ex:215 msgctxt "successful registration email body" msgid "

Hello @%{nickname},

\n

Your account at %{instance_name} has been registered successfully.

\n

No further action is required to activate your account.

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:217 +#: lib/pleroma/emails/user_email.ex:231 msgctxt "successful registration email subject" msgid "Account registered on %{instance_name}" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:111 +#: lib/pleroma/emails/user_email.ex:119 msgctxt "user invitation email body" msgid "

You are invited to %{instance_name}

\n

%{inviter_name} invites you to join %{instance_name}, an instance of Pleroma federated social networking platform.

\n

Click the following link to register: accept invitation.

\n" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:128 +#: lib/pleroma/emails/user_email.ex:136 msgctxt "user invitation email subject" msgid "Invitation to %{instance_name}" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:49 +#: lib/pleroma/emails/user_email.ex:53 msgctxt "welcome email html body" msgid "Welcome to %{instance_name}!" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:40 +#: lib/pleroma/emails/user_email.ex:41 msgctxt "welcome email subject" msgid "Welcome to %{instance_name}!" msgstr "" #, elixir-format -#: lib/pleroma/emails/user_email.ex:61 +#: lib/pleroma/emails/user_email.ex:65 msgctxt "welcome email text body" msgid "Welcome to %{instance_name}!" msgstr "" + +#, elixir-format +#: lib/pleroma/emails/user_email.ex:368 +msgctxt "account archive email body - admin requested" +msgid "

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

\n

%{download_url}

\n" +msgstr "" diff --git a/priv/repo/migrations/20220302013920_add_language_to_users.exs b/priv/repo/migrations/20220302013920_add_language_to_users.exs new file mode 100644 index 000000000..7a63c36aa --- /dev/null +++ b/priv/repo/migrations/20220302013920_add_language_to_users.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddLanguageToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add_if_not_exists(:language, :string) + end + end +end diff --git a/test/pleroma/emails/user_email_test.exs b/test/pleroma/emails/user_email_test.exs index 21fd06ea6..771a9a490 100644 --- a/test/pleroma/emails/user_email_test.exs +++ b/test/pleroma/emails/user_email_test.exs @@ -56,4 +56,16 @@ test "build approval pending email" do assert email.subject == "Your account is awaiting approval" assert email.html_body =~ "Awaiting Approval" end + + test "email i18n" do + user = insert(:user, language: "en_test") + email = UserEmail.approval_pending_email(user) + assert email.subject == "xxYour account is awaiting approvalxx" + end + + test "email i18n should fallback to default locale if user language is unsupported" do + user = insert(:user, language: "unsupported") + email = UserEmail.approval_pending_email(user) + assert email.subject == "Your account is awaiting approval" + end end From 1b77308644acd5fec50e712d3fa333e3008355da Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 00:58:02 -0500 Subject: [PATCH 22/94] Allow update_credentials to update User.language --- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index a307807a9..5303bdfd9 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -217,6 +217,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p |> Maps.put_if_present(:is_locked, params[:locked]) # Note: param name is indeed :discoverable (not an error) |> Maps.put_if_present(:is_discoverable, params[:discoverable]) + |> Maps.put_if_present(:language, params[:language]) # What happens here: # From 72bdb0640f28021ecbcc28711cc9268b17f5ea87 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 01:41:13 -0500 Subject: [PATCH 23/94] Allow user to register with custom language --- lib/pleroma/user.ex | 3 +- .../api_spec/operations/account_operation.ex | 5 ++ lib/pleroma/web/gettext.ex | 8 +++ .../controllers/account_controller.ex | 2 +- lib/pleroma/web/plugs/set_locale_plug.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api.ex | 6 ++ .../controllers/account_controller_test.exs | 70 +++++++++++++++++++ 7 files changed, 93 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 70afba3ae..9a50ee3ec 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -735,7 +735,8 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do :password_confirmation, :emoji, :accepts_chat_messages, - :registration_reason + :registration_reason, + :language ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index f5304d7d6..bdbae9b74 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -507,6 +507,11 @@ defp create_request do type: :string, nullable: true, description: "Invite token required when the registrations aren't public" + }, + language: %Schema{ + type: :string, + nullable: true, + description: "User's preferred language for emails" } }, example: %{ diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index e85290496..828b98b15 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -35,6 +35,14 @@ def language_tag do |> String.replace("_", "-", global: true) end + def normalize_locale(locale) do + if is_binary(locale) do + String.replace(locale, "-", "_") + else + nil + end + end + def supports_locale?(locale) do Pleroma.Web.Gettext |> Gettext.known_locales() diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 5303bdfd9..83cebbb96 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -217,7 +217,7 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p |> Maps.put_if_present(:is_locked, params[:locked]) # Note: param name is indeed :discoverable (not an error) |> Maps.put_if_present(:is_discoverable, params[:discoverable]) - |> Maps.put_if_present(:language, params[:language]) + |> Maps.put_if_present(:language, Pleroma.Web.Gettext.normalize_locale(params[:language])) # What happens here: # diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index 3c3fffa81..4c6e44fb5 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -25,7 +25,7 @@ defp get_locale_from_header(conn) do defp normalize_language_codes(codes) do codes - |> Enum.map(fn code -> String.replace(code, "-", "_") end) + |> Enum.map(fn code -> Pleroma.Web.Gettext.normalize_locale(code) end) end defp extract_preferred_language(conn) do diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 76ca82d20..7921653a8 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do alias Pleroma.UserInviteToken def register_user(params, opts \\ []) do + fallback_language = Gettext.get_locale() + params = params |> Map.take([:email, :token, :password]) @@ -20,6 +22,10 @@ def register_user(params, opts \\ []) do |> Map.put(:name, Map.get(params, :fullname, params[:username])) |> Map.put(:password_confirmation, params[:password]) |> Map.put(:registration_reason, params[:reason]) + |> Map.put( + :language, + Pleroma.Web.Gettext.normalize_locale(params[:language]) || fallback_language + ) if Pleroma.Config.get([:instance, :registrations_open]) do create_user(params, opts) diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index 374e2048a..de38a9798 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.Plugs.SetLocalePlug import Pleroma.Factory @@ -1586,6 +1587,75 @@ test "returns an error if captcha is invalid", %{conn: conn} do end end + describe "create account with language" do + setup %{conn: conn} do + app_token = insert(:oauth_token, user: nil) + + conn = + conn + |> put_req_header("authorization", "Bearer " <> app_token.token) + |> put_req_header("content-type", "multipart/form-data") + |> put_req_cookie(SetLocalePlug.frontend_language_cookie_name(), "zh-Hans") + |> SetLocalePlug.call([]) + + [conn: conn] + end + + test "creates an account with language parameter", %{conn: conn} do + params = %{ + username: "foo", + email: "foo@example.org", + password: "dupa.8", + agreement: true, + language: "ru" + } + + res = + conn + |> post("/api/v1/accounts", params) + + assert json_response_and_validate_schema(res, 200) + + assert %{language: "ru"} = Pleroma.User.get_by_nickname("foo") + end + + test "language parameter should be normalized", %{conn: conn} do + params = %{ + username: "foo", + email: "foo@example.org", + password: "dupa.8", + agreement: true, + language: "ru-RU" + } + + res = + conn + |> post("/api/v1/accounts", params) + + assert json_response_and_validate_schema(res, 200) + + assert %{language: "ru_RU"} = Pleroma.User.get_by_nickname("foo") + end + + test "createing an account without language parameter should fallback to cookie/header language", + %{conn: conn} do + params = %{ + username: "foo2", + email: "foo2@example.org", + password: "dupa.8", + agreement: true + } + + res = + conn + |> post("/api/v1/accounts", params) + + assert json_response_and_validate_schema(res, 200) + + assert %{language: "zh_Hans"} = Pleroma.User.get_by_nickname("foo2") + end + end + describe "GET /api/v1/accounts/:id/lists - account_lists" do test "returns lists to which the account belongs" do %{user: user, conn: conn} = oauth_access(["read:lists"]) From dadc4eabf7ede1568667a026414d80190dabb7cc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 09:47:51 -0500 Subject: [PATCH 24/94] Document API addition --- docs/development/API/differences_in_mastoapi_responses.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 518aca114..def718b95 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -241,6 +241,7 @@ Additional parameters can be added to the JSON body/Form data: - `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results). - `actor_type` - the type of this account. - `accepts_chat_messages` - if false, this account will reject all chat messages. +- `language` - user's preferred language for receiving emails (digest, confirmation, etc.) All images (avatar, banner and background) can be reset to the default by sending an empty string ("") instead of a file. @@ -292,6 +293,7 @@ Has these additional parameters (which are the same as in Pleroma-API): - `captcha_token`: optional, contains provider-specific captcha token - `captcha_answer_data`: optional, contains provider-specific captcha data - `token`: invite token required when the registrations aren't public. +- `language`: optional, user's preferred language for receiving emails (digest, confirmation, etc.), default to the language set in the `userLanguage` cookies or `Accept-Language` header. ## Instance From 6e8123a3304a207fd37b8904730cd7657787243e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 09:51:46 -0500 Subject: [PATCH 25/94] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9513a54fa..1cc902d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Readded mastoFE - Added support for custom emoji reactions - Added `emoji_url` in notifications to allow for custom emoji rendering +- Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field. ### Fixed - Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies From ef73f61b07266785850142ee4d215481a7a36d17 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 19:59:11 -0500 Subject: [PATCH 26/94] Fallback to a variant if the language in general is not supported For an example, here, zh is not supported, but zh_Hans and zh_Hant are. If the user asks for zh, we should choose a variant for them instead of fallbacking to default. Some browsers (e.g. Firefox) does not allow users to customize their language codes. For example, there is no zh-Hans, but only zh, zh-CN, zh-TW, zh-HK, etc. This provides a workaround for those users suffering from bad design decisions. --- lib/pleroma/web/gettext.ex | 14 ++++++++++++++ lib/pleroma/web/plugs/set_locale_plug.ex | 6 ++++++ test/pleroma/web/plugs/set_locale_plug_test.exs | 14 ++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index 828b98b15..cfd92f991 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -49,6 +49,20 @@ def supports_locale?(locale) do |> Enum.member?(locale) end + def variant?(locale), do: String.contains?(locale, "_") + + def supported_variants_of_locale(locale) do + cond do + variant?(locale) -> + [locale] + supports_locale?(locale) -> + [locale] + true -> + Gettext.known_locales(Pleroma.Web.Gettext) + |> Enum.filter(fn l -> String.starts_with?(l, locale <> "_") end) + end + end + def locale_or_default(locale) do if supports_locale?(locale) do locale diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index 4c6e44fb5..78ae566c7 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -20,6 +20,12 @@ defp get_locale_from_header(conn) do conn |> extract_preferred_language() |> normalize_language_codes() + |> first_supported() + end + + defp first_supported(locales) do + locales + |> Enum.flat_map(&Pleroma.Web.Gettext.supported_variants_of_locale/1) |> Enum.find(&supported_locale?/1) end diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index 043d7eb18..349326c24 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -47,6 +47,20 @@ test "use supported locale with specifiers from `accept-language`" do assert %{locale: "zh_Hans"} == conn.assigns end + test "fallback to some variant of the language if the unqualified language is not supported" do + conn = + :get + |> conn("/cofe") + |> Conn.put_req_header( + "accept-language", + "zh;q=0.9, en;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "zh_" <> _ = Gettext.get_locale() + assert %{locale: "zh_" <> _} = conn.assigns + end + test "use supported locale from cookie" do conn = :get From ff0bb3a3ac56e84bb28e5e7782629b7f5a5c4ed4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 20:04:30 -0500 Subject: [PATCH 27/94] Add test for fallbacking to a general language --- test/pleroma/web/plugs/set_locale_plug_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index 349326c24..f5d3ab995 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -33,6 +33,20 @@ test "use supported locale from `accept-language`" do assert %{locale: "ru"} == conn.assigns end + test "fallback to the general language if a variant is not supported" do + conn = + :get + |> conn("/cofe") + |> Conn.put_req_header( + "accept-language", + "ru-CA;q=0.9, en;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "ru" == Gettext.get_locale() + assert %{locale: "ru"} == conn.assigns + end + test "use supported locale with specifiers from `accept-language`" do conn = :get From 845e5769ceb424b6f6dae5811efed5ab55042334 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 2 Mar 2022 22:56:19 -0500 Subject: [PATCH 28/94] Make lint happy --- lib/pleroma/web/gettext.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index cfd92f991..694ad8ad6 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -55,8 +55,10 @@ def supported_variants_of_locale(locale) do cond do variant?(locale) -> [locale] + supports_locale?(locale) -> [locale] + true -> Gettext.known_locales(Pleroma.Web.Gettext) |> Enum.filter(fn l -> String.starts_with?(l, locale <> "_") end) From fa95bc87253e1c707511cafd9404a8a902338518 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 3 Mar 2022 02:03:44 -0500 Subject: [PATCH 29/94] Support multiple locales formally elixir gettext current does not fully support fallback to another language [0]. But it might in the future. We adapt it so that all languages in Accept-Language headers are received by Pleroma.Web.Gettext. User.languages is now a comma-separated list. [0]: https://github.com/elixir-gettext/gettext/issues/303 --- lib/pleroma/web/gettext.ex | 57 ++++++++++++++++++- lib/pleroma/web/plugs/set_locale_plug.ex | 19 ++++--- .../web/plugs/set_locale_plug_test.exs | 30 +++++++--- 3 files changed, 89 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index 694ad8ad6..e17451c09 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -65,6 +65,24 @@ def supported_variants_of_locale(locale) do end end + def get_locales() do + Process.get({Pleroma.Web.Gettext, :locales}, []) + end + + def is_locale_list(locales) do + Enum.all?(locales, &is_binary/1) + end + + def put_locales(locales) do + if is_locale_list(locales) do + Process.put({Pleroma.Web.Gettext, :locales}, Enum.uniq(locales)) + Gettext.put_locale(Enum.at(locales, 0, Gettext.get_locale())) + :ok + else + {:error, :not_locale_list} + end + end + def locale_or_default(locale) do if supports_locale?(locale) do locale @@ -73,11 +91,46 @@ def locale_or_default(locale) do end end - defmacro with_locale_or_default(locale, do: fun) do + def with_locales_func(locales, fun) do + prev_locales = Process.get({Pleroma.Web.Gettext, :locales}) + put_locales(locales) + + try do + fun.() + after + if prev_locales do + put_locales(prev_locales) + else + Process.delete({Pleroma.Web.Gettext, :locales}) + end + end + end + + defmacro with_locales(locales, do: fun) do quote do - Gettext.with_locale(Pleroma.Web.Gettext.locale_or_default(unquote(locale)), fn -> + Pleroma.Web.Gettext.with_locales_func(unquote(locales), fn -> unquote(fun) end) end end + + def to_locale_list(locale) when is_binary(locale) do + locale + |> String.split(",") + |> Enum.filter(&supports_locale?/1) + end + + def to_locale_list(_), do: [] + + defmacro with_locale_or_default(locale, do: fun) do + quote do + Pleroma.Web.Gettext.with_locales_func( + Pleroma.Web.Gettext.to_locale_list(unquote(locale)) + |> Enum.concat(Pleroma.Web.Gettext.get_locales()), + fn -> + unquote(fun) + end + ) + end + end end diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index 78ae566c7..936f65f5d 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -11,22 +11,27 @@ def frontend_language_cookie_name, do: "userLanguage" def init(_), do: nil def call(conn, _) do - locale = get_locale_from_header(conn) || Gettext.get_locale() - Gettext.put_locale(locale) - assign(conn, :locale, locale) + locales = get_locales_from_header(conn) + first_locale = Enum.at(locales, 0, Gettext.get_locale()) + + Pleroma.Web.Gettext.put_locales(locales) + + conn + |> assign(:locale, first_locale) + |> assign(:locales, locales) end - defp get_locale_from_header(conn) do + defp get_locales_from_header(conn) do conn |> extract_preferred_language() |> normalize_language_codes() - |> first_supported() + |> all_supported() end - defp first_supported(locales) do + defp all_supported(locales) do locales |> Enum.flat_map(&Pleroma.Web.Gettext.supported_variants_of_locale/1) - |> Enum.find(&supported_locale?/1) + |> Enum.filter(&supported_locale?/1) end defp normalize_language_codes(codes) do diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index f5d3ab995..b0e7afffd 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -16,7 +16,7 @@ test "default locale is `en`" do |> SetLocalePlug.call([]) assert "en" == Gettext.get_locale() - assert %{locale: "en"} == conn.assigns + assert %{locale: "en"} = conn.assigns end test "use supported locale from `accept-language`" do @@ -30,7 +30,7 @@ test "use supported locale from `accept-language`" do |> SetLocalePlug.call([]) assert "ru" == Gettext.get_locale() - assert %{locale: "ru"} == conn.assigns + assert %{locale: "ru"} = conn.assigns end test "fallback to the general language if a variant is not supported" do @@ -44,7 +44,7 @@ test "fallback to the general language if a variant is not supported" do |> SetLocalePlug.call([]) assert "ru" == Gettext.get_locale() - assert %{locale: "ru"} == conn.assigns + assert %{locale: "ru"} = conn.assigns end test "use supported locale with specifiers from `accept-language`" do @@ -58,7 +58,21 @@ test "use supported locale with specifiers from `accept-language`" do |> SetLocalePlug.call([]) assert "zh_Hans" == Gettext.get_locale() - assert %{locale: "zh_Hans"} == conn.assigns + assert %{locale: "zh_Hans"} = conn.assigns + end + + test "it assigns all supported locales" do + conn = + :get + |> conn("/cofe") + |> Conn.put_req_header( + "accept-language", + "ru, fr-CH, fr;q=0.9, en;q=0.8, x-unsupported;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "ru" == Gettext.get_locale() + assert %{locale: "ru", locales: ["ru", "fr", "en"]} = conn.assigns end test "fallback to some variant of the language if the unqualified language is not supported" do @@ -87,7 +101,7 @@ test "use supported locale from cookie" do |> SetLocalePlug.call([]) assert "zh_Hans" == Gettext.get_locale() - assert %{locale: "zh_Hans"} == conn.assigns + assert %{locale: "zh_Hans"} = conn.assigns end test "fallback to supported locale from `accept-language` if locale in cookie not supported" do @@ -102,7 +116,7 @@ test "fallback to supported locale from `accept-language` if locale in cookie no |> SetLocalePlug.call([]) assert "ru" == Gettext.get_locale() - assert %{locale: "ru"} == conn.assigns + assert %{locale: "ru"} = conn.assigns end test "fallback to default if nothing is supported" do @@ -117,7 +131,7 @@ test "fallback to default if nothing is supported" do |> SetLocalePlug.call([]) assert "en" == Gettext.get_locale() - assert %{locale: "en"} == conn.assigns + assert %{locale: "en"} = conn.assigns end test "use default locale if locale from `accept-language` is not supported" do @@ -128,6 +142,6 @@ test "use default locale if locale from `accept-language` is not supported" do |> SetLocalePlug.call([]) assert "en" == Gettext.get_locale() - assert %{locale: "en"} == conn.assigns + assert %{locale: "en"} = conn.assigns end end From 07bd35227af9a4f3eefd2cf0a7cefc6ea2d110b3 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 3 Mar 2022 02:31:36 -0500 Subject: [PATCH 30/94] Support multiple locales from userLanguage cookie --- lib/pleroma/web/gettext.ex | 27 ++++++++++++++----- lib/pleroma/web/plugs/set_locale_plug.ex | 14 +++------- .../web/plugs/set_locale_plug_test.exs | 15 +++++++++++ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index e17451c09..cd795008d 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -37,7 +37,7 @@ def language_tag do def normalize_locale(locale) do if is_binary(locale) do - String.replace(locale, "-", "_") + String.replace(locale, "-", "_", global: true) else nil end @@ -51,13 +51,28 @@ def supports_locale?(locale) do def variant?(locale), do: String.contains?(locale, "_") - def supported_variants_of_locale(locale) do - cond do - variant?(locale) -> - [locale] + def language_for_variant(locale) do + Enum.at(String.split(locale, "_"), 0) + end + def ensure_fallbacks(locales) do + locales + |> Enum.flat_map(fn locale -> + others = other_supported_variants_of_locale(locale) + |> Enum.filter(fn l -> not Enum.member?(locales, l) end) + + [locale] ++ others + end) + end + + def other_supported_variants_of_locale(locale) do + cond do supports_locale?(locale) -> - [locale] + [] + + variant?(locale) -> + lang = language_for_variant(locale) + if supports_locale?(lang), do: [lang], else: [] true -> Gettext.known_locales(Pleroma.Web.Gettext) diff --git a/lib/pleroma/web/plugs/set_locale_plug.ex b/lib/pleroma/web/plugs/set_locale_plug.ex index 936f65f5d..e78917199 100644 --- a/lib/pleroma/web/plugs/set_locale_plug.ex +++ b/lib/pleroma/web/plugs/set_locale_plug.ex @@ -26,11 +26,12 @@ defp get_locales_from_header(conn) do |> extract_preferred_language() |> normalize_language_codes() |> all_supported() + |> Enum.uniq() end defp all_supported(locales) do locales - |> Enum.flat_map(&Pleroma.Web.Gettext.supported_variants_of_locale/1) + |> Pleroma.Web.Gettext.ensure_fallbacks() |> Enum.filter(&supported_locale?/1) end @@ -53,8 +54,7 @@ defp extract_frontend_language(conn) do [] fe_lang -> - [fe_lang] - |> ensure_language_fallbacks() + String.split(fe_lang, ",") end end @@ -67,7 +67,6 @@ defp extract_accept_language(conn) do |> Enum.sort(&(&1.quality > &2.quality)) |> Enum.map(& &1.tag) |> Enum.reject(&is_nil/1) - |> ensure_language_fallbacks() _ -> [] @@ -89,11 +88,4 @@ defp parse_language_option(string) do %{tag: captures["tag"], quality: quality} end - - defp ensure_language_fallbacks(tags) do - Enum.flat_map(tags, fn tag -> - [language | _] = String.split(tag, "-") - if Enum.member?(tags, language), do: [tag], else: [tag, language] - end) - end end diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index b0e7afffd..ff04a859e 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -75,6 +75,21 @@ test "it assigns all supported locales" do assert %{locale: "ru", locales: ["ru", "fr", "en"]} = conn.assigns end + test "it assigns all supported locales in cookie" do + conn = + :get + |> conn("/cofe") + |> put_req_cookie(SetLocalePlug.frontend_language_cookie_name(), "zh-Hans,uk,zh-Hant") + |> Conn.put_req_header( + "accept-language", + "ru, fr-CH, fr;q=0.9, en;q=0.8, x-unsupported;q=0.8, *;q=0.5" + ) + |> SetLocalePlug.call([]) + + assert "zh_Hans" == Gettext.get_locale() + assert %{locale: "zh_Hans", locales: ["zh_Hans", "uk", "zh_Hant", "ru", "fr", "en"]} = conn.assigns + end + test "fallback to some variant of the language if the unqualified language is not supported" do conn = :get From fcfb5a4967621be57006fdc62e30957ca381d846 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 3 Mar 2022 09:40:18 -0500 Subject: [PATCH 31/94] Lint --- lib/pleroma/web/gettext.ex | 5 +++-- test/pleroma/web/plugs/set_locale_plug_test.exs | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index cd795008d..89feb0bb3 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -58,7 +58,8 @@ def language_for_variant(locale) do def ensure_fallbacks(locales) do locales |> Enum.flat_map(fn locale -> - others = other_supported_variants_of_locale(locale) + others = + other_supported_variants_of_locale(locale) |> Enum.filter(fn l -> not Enum.member?(locales, l) end) [locale] ++ others @@ -80,7 +81,7 @@ def other_supported_variants_of_locale(locale) do end end - def get_locales() do + def get_locales do Process.get({Pleroma.Web.Gettext, :locales}, []) end diff --git a/test/pleroma/web/plugs/set_locale_plug_test.exs b/test/pleroma/web/plugs/set_locale_plug_test.exs index ff04a859e..f9d34bbe4 100644 --- a/test/pleroma/web/plugs/set_locale_plug_test.exs +++ b/test/pleroma/web/plugs/set_locale_plug_test.exs @@ -87,7 +87,9 @@ test "it assigns all supported locales in cookie" do |> SetLocalePlug.call([]) assert "zh_Hans" == Gettext.get_locale() - assert %{locale: "zh_Hans", locales: ["zh_Hans", "uk", "zh_Hant", "ru", "fr", "en"]} = conn.assigns + + assert %{locale: "zh_Hans", locales: ["zh_Hans", "uk", "zh_Hant", "ru", "fr", "en"]} = + conn.assigns end test "fallback to some variant of the language if the unqualified language is not supported" do From 2df25e6666d0fb65a142b29becee9af0adce998b Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 6 Mar 2022 11:43:31 -0500 Subject: [PATCH 32/94] Support fallbacking to other languages --- lib/pleroma/web/gettext.ex | 53 ++++++ mix.exs | 5 +- mix.lock | 2 +- .../en_test/LC_MESSAGES/static_pages.po | 8 +- test/pleroma/web/gettext_test.exs | 162 ++++++++++++++++++ 5 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 test/pleroma/web/gettext_test.exs diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index 89feb0bb3..51e56939e 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -118,6 +118,7 @@ def with_locales_func(locales, fun) do put_locales(prev_locales) else Process.delete({Pleroma.Web.Gettext, :locales}) + Process.delete(Gettext) end end end @@ -149,4 +150,56 @@ defmacro with_locale_or_default(locale, do: fun) do ) end end + + defp next_locale(locale, list) do + index = Enum.find_index(list, fn item -> item == locale end) + + if not is_nil(index) do + Enum.at(list, index + 1) + else + nil + end + end + + def handle_missing_translation(locale, domain, msgctxt, msgid, bindings) do + next = next_locale(locale, get_locales()) + + if is_nil(next) do + super(locale, domain, msgctxt, msgid, bindings) + else + {:ok, + Gettext.with_locale(next, fn -> + Gettext.dpgettext(Pleroma.Web.Gettext, domain, msgctxt, msgid, bindings) + end)} + end + end + + def handle_missing_plural_translation( + locale, + domain, + msgctxt, + msgid, + msgid_plural, + n, + bindings + ) do + next = next_locale(locale, get_locales()) + + if is_nil(next) do + super(locale, domain, msgctxt, msgid, msgid_plural, n, bindings) + else + {:ok, + Gettext.with_locale(next, fn -> + Gettext.dpngettext( + Pleroma.Web.Gettext, + domain, + msgctxt, + msgid, + msgid_plural, + n, + bindings + ) + end)} + end + end end diff --git a/mix.exs b/mix.exs index 76e84c2b8..564db2d75 100644 --- a/mix.exs +++ b/mix.exs @@ -123,7 +123,10 @@ defp deps do {:ecto_sql, "~> 3.6.2"}, {:postgrex, ">= 0.15.5"}, {:oban, "~> 2.3.4"}, - {:gettext, "~> 0.18"}, + {:gettext, + git: "https://github.com/tusooa/gettext.git", + ref: "72fb2496b6c5280ed911bdc3756890e7f38a4808", + override: true}, {:bcrypt_elixir, "~> 2.2"}, {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.0"}, diff --git a/mix.lock b/mix.lock index 8c39d2199..422bbea5e 100644 --- a/mix.lock +++ b/mix.lock @@ -57,7 +57,7 @@ "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, - "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"}, + "gettext": {:git, "https://github.com/tusooa/gettext.git", "72fb2496b6c5280ed911bdc3756890e7f38a4808", [ref: "72fb2496b6c5280ed911bdc3756890e7f38a4808"]}, "gun": {:hex, :gun, "2.0.0-rc.2", "7c489a32dedccb77b6e82d1f3c5a7dadfbfa004ec14e322cdb5e579c438632d2", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "6b9d1eae146410d727140dbf8b404b9631302ecc2066d1d12f22097ad7d254fc"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "hpax": {:hex, :hpax, "0.1.1", "2396c313683ada39e98c20a75a82911592b47e5c24391363343bde74f82396ca", [:mix], [], "hexpm", "0ae7d5a0b04a8a60caf7a39fcf3ec476f35cc2cc16c05abea730d3ce6ac6c826"}, diff --git a/priv/gettext/en_test/LC_MESSAGES/static_pages.po b/priv/gettext/en_test/LC_MESSAGES/static_pages.po index a3378089c..1a3b7b355 100644 --- a/priv/gettext/en_test/LC_MESSAGES/static_pages.po +++ b/priv/gettext/en_test/LC_MESSAGES/static_pages.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" -"PO-Revision-Date: 2022-03-01 21:15-0500\n" +"PO-Revision-Date: 2022-03-06 11:27-0500\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "MIME-Version: 1.0\n" @@ -423,8 +423,8 @@ msgstr "" msgctxt "new followers count header" msgid "%{count} New Follower" msgid_plural "%{count} New Followers" -msgstr[0] "" -msgstr[1] "" +msgstr[0] "xx%{count} New Followerxx" +msgstr[1] "xx%{count} New Followersxx" #, elixir-format #: lib/pleroma/emails/user_email.ex:356 @@ -466,7 +466,7 @@ msgstr "" #: lib/pleroma/emails/user_email.ex:310 msgctxt "digest email subject" msgid "Your digest from %{instance_name}" -msgstr "" +msgstr "xxYour digest from %{instance_name}xx" #, elixir-format #: lib/pleroma/emails/user_email.ex:81 diff --git a/test/pleroma/web/gettext_test.exs b/test/pleroma/web/gettext_test.exs new file mode 100644 index 000000000..9ede4827e --- /dev/null +++ b/test/pleroma/web/gettext_test.exs @@ -0,0 +1,162 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.GettextTest do + use ExUnit.Case + + require Pleroma.Web.Gettext + + test "put_locales/1: set the first in the list to Gettext's locale" do + Pleroma.Web.Gettext.put_locales(["zh_Hans", "en_test"]) + + assert "zh_Hans" == Gettext.get_locale(Pleroma.Web.Gettext) + end + + test "with_locales/2: reset locale on exit" do + old_first_locale = Gettext.get_locale(Pleroma.Web.Gettext) + old_locales = Pleroma.Web.Gettext.get_locales() + + Pleroma.Web.Gettext.with_locales ["zh_Hans", "en_test"] do + assert "zh_Hans" == Gettext.get_locale(Pleroma.Web.Gettext) + assert ["zh_Hans", "en_test"] == Pleroma.Web.Gettext.get_locales() + end + + assert old_first_locale == Gettext.get_locale(Pleroma.Web.Gettext) + assert old_locales == Pleroma.Web.Gettext.get_locales() + end + + describe "handle_missing_translation/5" do + test "fallback to next locale if some translation is not available" do + Pleroma.Web.Gettext.with_locales ["x_unsupported", "en_test"] do + assert "xxYour account is awaiting approvalxx" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "approval pending email subject", + "Your account is awaiting approval" + ) + end + end + + test "duplicated locale in list should not result in infinite loops" do + Pleroma.Web.Gettext.with_locales ["x_unsupported", "x_unsupported", "en_test"] do + assert "xxYour account is awaiting approvalxx" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "approval pending email subject", + "Your account is awaiting approval" + ) + end + end + + test "direct interpolation" do + Pleroma.Web.Gettext.with_locales ["en_test"] do + assert "xxYour digest from some instancexx" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "digest email subject", + "Your digest from %{instance_name}", + instance_name: "some instance" + ) + end + end + + test "fallback with interpolation" do + Pleroma.Web.Gettext.with_locales ["x_unsupported", "en_test"] do + assert "xxYour digest from some instancexx" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "digest email subject", + "Your digest from %{instance_name}", + instance_name: "some instance" + ) + end + end + + test "fallback to msgid" do + Pleroma.Web.Gettext.with_locales ["x_unsupported"] do + assert "Your digest from some instance" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "digest email subject", + "Your digest from %{instance_name}", + instance_name: "some instance" + ) + end + end + end + + describe "handle_missing_plural_translation/7" do + test "direct interpolation" do + Pleroma.Web.Gettext.with_locales ["en_test"] do + assert "xx1 New Followerxx" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 1, + count: 1 + ) + + assert "xx5 New Followersxx" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 5, + count: 5 + ) + end + end + + test "fallback with interpolation" do + Pleroma.Web.Gettext.with_locales ["x_unsupported", "en_test"] do + assert "xx1 New Followerxx" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 1, + count: 1 + ) + + assert "xx5 New Followersxx" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 5, + count: 5 + ) + end + end + + test "fallback to msgid" do + Pleroma.Web.Gettext.with_locales ["x_unsupported"] do + assert "1 New Follower" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 1, + count: 1 + ) + + assert "5 New Followers" == + Pleroma.Web.Gettext.dpngettext( + "static_pages", + "new followers count header", + "%{count} New Follower", + "%{count} New Followers", + 5, + count: 5 + ) + end + end + end +end From e3107fee98bf33dc9a565a00804315c330811f73 Mon Sep 17 00:00:00 2001 From: Ilja Date: Tue, 5 Apr 2022 13:21:09 +0200 Subject: [PATCH 33/94] Fix eratic test for POST /api/pleroma/admin/reports/:id/notes It retrieved two ReportNotes and then checked one of them. But the order isn't guaranteed, while the test tested on the content of the first ReportNote. I made the test on the content more generic --- .../web/admin_api/controllers/report_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index 642b05f3f..2d526527b 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -369,7 +369,8 @@ test "it returns reports with notes", %{conn: conn, admin: admin} do refute is_nil(note) assert note["user"]["nickname"] == admin.nickname - assert note["content"] == "this is disgusting!" + # We use '=~' because the order of the notes isn't guaranteed + assert note["content"] =~ "this is disgusting" assert note["created_at"] assert response["total"] == 1 end From 33d821256e2a9323d80189a6ad5e07aeca28a9aa Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 17 Apr 2022 22:39:52 -0400 Subject: [PATCH 34/94] Fix incorrect fallback when English is set to first language --- lib/pleroma/web/gettext.ex | 19 +++++++++++++++++-- test/pleroma/web/gettext_test.exs | 11 +++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index 51e56939e..7afcd38f0 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -161,10 +161,25 @@ defp next_locale(locale, list) do end end + # We do not yet have a proper English translation. The "English" + # version is currently but the fallback msgid. However, this + # will not work if the user puts English as the first language, + # and at the same time specifies other languages, as gettext will + # think the English translation is missing, and call + # handle_missing_translation functions. This may result in + # text in other languages being shown even if English is preferred + # by the user. + # + # To prevent this, we do not allow fallbacking when the current + # locale missing a translation is English. + defp should_fallback?(locale) do + locale != "en" + end + def handle_missing_translation(locale, domain, msgctxt, msgid, bindings) do next = next_locale(locale, get_locales()) - if is_nil(next) do + if is_nil(next) or not should_fallback?(locale) do super(locale, domain, msgctxt, msgid, bindings) else {:ok, @@ -185,7 +200,7 @@ def handle_missing_plural_translation( ) do next = next_locale(locale, get_locales()) - if is_nil(next) do + if is_nil(next) or not should_fallback?(locale) do super(locale, domain, msgctxt, msgid, msgid_plural, n, bindings) else {:ok, diff --git a/test/pleroma/web/gettext_test.exs b/test/pleroma/web/gettext_test.exs index 9ede4827e..e186f1ab3 100644 --- a/test/pleroma/web/gettext_test.exs +++ b/test/pleroma/web/gettext_test.exs @@ -38,6 +38,17 @@ test "fallback to next locale if some translation is not available" do end end + test "putting en locale at the front should not make gettext fallback unexpectedly" do + Pleroma.Web.Gettext.with_locales ["en", "en_test"] do + assert "Your account is awaiting approval" == + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "approval pending email subject", + "Your account is awaiting approval" + ) + end + end + test "duplicated locale in list should not result in infinite loops" do Pleroma.Web.Gettext.with_locales ["x_unsupported", "x_unsupported", "en_test"] do assert "xxYour account is awaiting approvalxx" == From 932e5df19ed53121db85acb055e06a3fd32775ff Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 5 May 2022 18:39:34 -0400 Subject: [PATCH 35/94] Allow to skip cache in Cache plug Ref: fix-local-public --- lib/pleroma/web/plugs/cache.ex | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/plugs/cache.ex b/lib/pleroma/web/plugs/cache.ex index 935b2d834..b0e44db07 100644 --- a/lib/pleroma/web/plugs/cache.ex +++ b/lib/pleroma/web/plugs/cache.ex @@ -100,20 +100,19 @@ defp cache_resp(conn, opts) do should_cache = not Map.get(conn.assigns, :skip_cache, false) conn = - unless opts[:tracking_fun] do - if should_cache do + cond do + Map.get(conn.assigns, :skip_cache, false) -> + conn + + !opts[:tracking_fun] -> @cachex.put(:web_resp_cache, key, {content_type, body}, ttl: ttl) - end + conn - conn - else - tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) - - if should_cache do + true -> + tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) @cachex.put(:web_resp_cache, key, {content_type, body, tracking_fun_data}, ttl: ttl) - end - opts.tracking_fun.(conn, tracking_fun_data) + opts.tracking_fun.(conn, tracking_fun_data) end put_resp_header(conn, "x-cache", "MISS from Pleroma") From 3fd87b6a7521ea244de7882c9486ee6a8e3fd1e7 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 5 May 2022 19:20:32 -0400 Subject: [PATCH 36/94] Skip cache when /objects or /activities is authenticated Ref: fix-local-public --- lib/pleroma/web/plugs/cache.ex | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/plugs/cache.ex b/lib/pleroma/web/plugs/cache.ex index b0e44db07..935b2d834 100644 --- a/lib/pleroma/web/plugs/cache.ex +++ b/lib/pleroma/web/plugs/cache.ex @@ -100,19 +100,20 @@ defp cache_resp(conn, opts) do should_cache = not Map.get(conn.assigns, :skip_cache, false) conn = - cond do - Map.get(conn.assigns, :skip_cache, false) -> - conn - - !opts[:tracking_fun] -> + unless opts[:tracking_fun] do + if should_cache do @cachex.put(:web_resp_cache, key, {content_type, body}, ttl: ttl) - conn + end - true -> - tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) + conn + else + tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) + + if should_cache do @cachex.put(:web_resp_cache, key, {content_type, body, tracking_fun_data}, ttl: ttl) + end - opts.tracking_fun.(conn, tracking_fun_data) + opts.tracking_fun.(conn, tracking_fun_data) end put_resp_header(conn, "x-cache", "MISS from Pleroma") From 661d0ba481c49ee3cc0cd43051558de21f0398a9 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sun, 8 May 2022 18:10:40 +0000 Subject: [PATCH 37/94] Also use actor_type to determine if an account is a bot in antiFollowbotPolicy --- CHANGELOG.md | 1 + docs/configuration/cheatsheet.md | 2 + .../activity_pub/mrf/anti_followbot_policy.ex | 21 +++++-- .../mrf/anti_followbot_policy_test.exs | 57 +++++++++++++++---- 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cc902d58..2119c8e21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available. - AdminAPI: sort users so the newest are at the top. - ActivityPub Client-to-Server(C2S): Limitation on the type of Activity/Object are lifted as they are now passed through ObjectValidators +- MRF (`AntiFollowbotPolicy`): Bot accounts are now also considered followbots. Users can still allow bots to follow them by first following the bot. ### Added diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 3b8f8cc52..50281f451 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -125,6 +125,8 @@ To add configuration to your config file, you can copy it from the base config. * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections. * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines. * `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed. + * `Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy`: Drops follow requests from followbots. Users can still allow bots to follow them by first following the bot. + * `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)). * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 851e95d22..627f52168 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -24,7 +24,7 @@ defp score_displayname("federationbot"), do: 1.0 defp score_displayname("fedibot"), do: 1.0 defp score_displayname(_), do: 0.0 - defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do + defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do # nickname will be a binary string except when following a relay nick_score = if is_binary(nickname) do @@ -45,19 +45,32 @@ defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do 0.0 end - nick_score + name_score + # actor_type "Service" is a Bot account + actor_type_score = + if actor_type == "Service" do + 1.0 + else + 0.0 + end + + nick_score + name_score + actor_type_score end defp determine_if_followbot(_), do: 0.0 + defp bot_allowed?(%{"object" => target}, bot_actor) do + %User{} = user = normalize_by_ap_id(target) + + User.following?(user, bot_actor) + end + @impl true def filter(%{"type" => "Follow", "actor" => actor_id} = message) do %User{} = actor = normalize_by_ap_id(actor_id) score = determine_if_followbot(actor) - # TODO: scan biography data for keywords and score it somehow. - if score < 0.8 do + if score < 0.8 || bot_allowed?(message, actor) do {:ok, message} else {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} diff --git a/test/pleroma/web/activity_pub/mrf/anti_followbot_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_followbot_policy_test.exs index d5af3a9b6..14a6ae52b 100644 --- a/test/pleroma/web/activity_pub/mrf/anti_followbot_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/anti_followbot_policy_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do use Pleroma.DataCase, async: true import Pleroma.Factory + alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy describe "blocking based on attributes" do @@ -38,21 +39,55 @@ test "matches followbots by display name" do assert {:reject, "[AntiFollowbotPolicy]" <> _} = AntiFollowbotPolicy.filter(message) end + + test "matches followbots by actor_type" do + actor = insert(:user, %{actor_type: "Service"}) + target = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Follow", + "actor" => actor.ap_id, + "object" => target.ap_id, + "id" => "https://example.com/activities/1234" + } + + assert {:reject, "[AntiFollowbotPolicy]" <> _} = AntiFollowbotPolicy.filter(message) + end end - test "it allows non-followbots" do - actor = insert(:user) - target = insert(:user) + describe "it allows" do + test "non-followbots" do + actor = insert(:user) + target = insert(:user) - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "type" => "Follow", - "actor" => actor.ap_id, - "object" => target.ap_id, - "id" => "https://example.com/activities/1234" - } + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Follow", + "actor" => actor.ap_id, + "object" => target.ap_id, + "id" => "https://example.com/activities/1234" + } - {:ok, _} = AntiFollowbotPolicy.filter(message) + {:ok, _} = AntiFollowbotPolicy.filter(message) + end + + test "bots if the target follows the bots" do + actor = insert(:user, %{actor_type: "Service"}) + target = insert(:user) + + User.follow(target, actor) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Follow", + "actor" => actor.ap_id, + "object" => target.ap_id, + "id" => "https://example.com/activities/1234" + } + + {:ok, _} = AntiFollowbotPolicy.filter(message) + end end test "it gracefully handles nil display names" do From 8f140deb8f997ca9f6bdecd98812082800c851f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Wed, 18 May 2022 21:25:10 +0200 Subject: [PATCH 38/94] StealEmojiPolicy: fix String rejected_shortcodes * rejected_shortcodes is defined as a list of strings in the configuration description. As such, database-based configuration was led to handle those settings as strings, and not as the actually expected type, Regex. * This caused each message passing through this MRF, if a rejected shortcode was set and the emoji did not exist already on the instance, to fail federating, as an exception was raised, swiftly caught and mostly silenced. * This commit fixes the issue by introducing new behavior: strings are now handled as perfect matches for an emoji shortcode (meaning that if the emoji-to-be-pulled's shortcode is in the blacklist, it will be rejected), while still supporting Regex types as before. --- .../web/activity_pub/mrf/steal_emoji_policy.ex | 18 +++++++++++++++--- .../mrf/steal_emoji_policy_test.exs | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 0dd415732..61e95b49a 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -12,6 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) + defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do + shortcode == pattern + end + + defp shortcode_matches?(shortcode, pattern) do + String.match?(shortcode, pattern) + end + defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) @@ -72,7 +80,7 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa reject_emoji? = [:mrf_steal_emoji, :rejected_shortcodes] |> Config.get([]) - |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) + |> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end) !reject_emoji? end) @@ -122,8 +130,12 @@ def config_description do %{ key: :rejected_shortcodes, type: {:list, :string}, - description: "Regex-list of shortcodes to reject", - suggestions: [""] + description: """ + A list of patterns or matches to reject shortcodes with. + + Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + """, + suggestions: ["foo", ~r/foo/] }, %{ key: :size_limit, diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index 1b37e4c26..b0a7e8993 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -60,7 +60,7 @@ test "Steals emoji on unknown shortcode from allowed remote host", %{ |> File.exists?() end - test "reject shortcode", %{message: message} do + test "reject regex shortcode", %{message: message} do refute "firedfox" in installed() clear_config(:mrf_steal_emoji, @@ -74,6 +74,20 @@ test "reject shortcode", %{message: message} do refute "firedfox" in installed() end + test "reject string shortcode", %{message: message} do + refute "firedfox" in installed() + + clear_config(:mrf_steal_emoji, + hosts: ["example.org"], + size_limit: 284_468, + rejected_shortcodes: ["firedfox"] + ) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "firedfox" in installed() + end + test "reject if size is above the limit", %{message: message} do refute "firedfox" in installed() From 60eec9d0c50bf9753514d2e34ddec1634091ef84 Mon Sep 17 00:00:00 2001 From: lewdthewides Date: Thu, 12 May 2022 16:02:58 +0000 Subject: [PATCH 39/94] Instruct users to run 'git pull' as the pleroma user --- docs/administration/updating.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/administration/updating.md b/docs/administration/updating.md index ef2c9218c..01d3b9b0e 100644 --- a/docs/administration/updating.md +++ b/docs/administration/updating.md @@ -17,11 +17,11 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate" ## For from source installations (using git) 1. Go to the working directory of Pleroma (default is `/opt/pleroma`) -2. Run `git pull`. This pulls the latest changes from upstream. +2. Run `git pull` [^1]. This pulls the latest changes from upstream. 3. Run `mix deps.get` [^1]. This pulls in any new dependencies. 4. Stop the Pleroma service. 5. Run `mix ecto.migrate` [^1] [^2]. This task performs database migrations, if there were any. 6. Start the Pleroma service. -[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command. +[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `git` and `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command. [^2]: Prefix with `MIX_ENV=prod` to run it using the production config file. From e961cf26891e5d9a14e00c990cf6aa5a14095a22 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 15 Aug 2021 21:53:04 +0300 Subject: [PATCH 40/94] Initial meilisearch implementation, doesn't delete posts yet --- config/config.exs | 11 +- config/test.exs | 2 + lib/mix/tasks/pleroma/search/meilisearch.ex | 38 +++++++ lib/pleroma/activity.ex | 1 + lib/pleroma/activity/search.ex | 6 +- lib/pleroma/application.ex | 6 +- lib/pleroma/search/meilisearch.ex | 60 +++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++ .../controllers/search_controller.ex | 101 ++++++++++++++++++ 9 files changed, 220 insertions(+), 11 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex create mode 100644 lib/pleroma/search/meilisearch.ex diff --git a/config/config.exs b/config/config.exs index 00f9af797..7df0521f2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -850,17 +850,14 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]} + {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, + {Pleroma.Search, [max_running: 20, max_waiting: 50]} ] config :pleroma, :search, provider: Pleroma.Search.Builtin -config :pleroma, :telemetry, - slow_queries_logging: [ - enabled: false, - min_duration: 500_000, - exclude_sources: [nil, "oban_jobs"] - ] +config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search +config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/" # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index a5bf3a4d1..445975205 100644 --- a/config/test.exs +++ b/config/test.exs @@ -134,6 +134,8 @@ ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, logger: Pleroma.LoggerMock +config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search + # Reduce recompilation time # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2af8e5853 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + import Mix.Pleroma + + import Ecto.Query + + def run(["index"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + Pleroma.Repo.chunk_stream( + from(Pleroma.Object, + limit: 200, + where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + ), + 100, + :batches + ) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn activities -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(activities) + ) + end) + |> Stream.run() + end +end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4106feef6..10b1b0120 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,6 +368,7 @@ def restrict_deactivated_users(query) do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + def add_to_index(_activity), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 09671f621..7152b0e46 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -57,7 +57,7 @@ def maybe_restrict_blocked(query, %User{} = user) do def maybe_restrict_blocked(query, _), do: query - defp restrict_public(q) do + def restrict_public(q) do from([a, o] in q, where: fragment("?->>'type' = 'Create'", a.data), where: ^Pleroma.Constants.as_public() in a.recipients @@ -124,7 +124,7 @@ defp query_with(q, :rum, search_query, :websearch) do ) end - defp maybe_restrict_local(q, user) do + def maybe_restrict_local(q, user) do limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do @@ -137,7 +137,7 @@ defp maybe_restrict_local(q, user) do defp restrict_local(q), do: where(q, local: true) - defp maybe_fetch(activities, user, search_query) do + def maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d37454d2c..be03cdffb 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -307,7 +307,11 @@ defp http_children(_, _), do: [] def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] + [ + Pleroma.Web.RichMedia.Helpers, + Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, + Pleroma.Search + ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..92e0d3429 --- /dev/null +++ b/lib/pleroma/search/meilisearch.ex @@ -0,0 +1,60 @@ +defmodule Pleroma.Search.Meilisearch do + require Logger + + alias Pleroma.Activity + + import Pleroma.Activity.Search + import Ecto.Query + + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/search", + Jason.encode!(%{q: query, offset: offset, limit: limit}) + ) + + hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([activity], desc: activity.id) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end + end + + def add_to_index(activity) do + object = activity.object + + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + data = object.data + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + end + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 756096952..615bee428 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,6 +140,12 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + {:ok, activity} else %Activity{} = activity -> diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 86ad388fd..aaf52cdc5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper alias Pleroma.Web.MastodonAPI.AccountView @@ -64,6 +65,106 @@ defp search_options(params, user) do |> Enum.filter(&elem(&1, 1)) end + defp resource_search(_, "accounts", query, options) do + accounts = with_fallback(fn -> User.search(query, options) end) + + AccountView.render("index.json", + users: accounts, + for: options[:for_user], + embed_relationships: options[:embed_relationships] + ) + end + + defp resource_search(_, "statuses", query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) + + StatusView.render("index.json", + activities: statuses, + for: options[:for_user], + as: :activity + ) + end + + defp resource_search(:v2, "hashtags", query, options) do + tags_path = Endpoint.url() <> "/tag/" + + query + |> prepare_tags(options) + |> Enum.map(fn tag -> + %{name: tag, url: tags_path <> tag} + end) + end + + defp resource_search(:v1, "hashtags", query, options) do + prepare_tags(query, options) + end + + defp prepare_tags(query, options) do + tags = + query + |> preprocess_uri_query() + |> String.split(~r/[^#\w]+/u, trim: true) + |> Enum.uniq_by(&String.downcase/1) + + explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end) + + tags = + if Enum.any?(explicit_tags) do + explicit_tags + else + tags + end + + tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) + + tags = + if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do + add_joined_tag(tags) + else + tags + end + + Pleroma.Pagination.paginate(tags, options) + end + + defp add_joined_tag(tags) do + tags + |> Kernel.++([joined_tag(tags)]) + |> Enum.uniq_by(&String.downcase/1) + end + + # If `query` is a URI, returns last component of its path, otherwise returns `query` + defp preprocess_uri_query(query) do + if query =~ ~r/https?:\/\// do + query + |> String.trim_trailing("/") + |> URI.parse() + |> Map.get(:path) + |> String.split("/") + |> Enum.at(-1) + else + query + end + end + + defp joined_tag(tags) do + tags + |> Enum.map(fn tag -> String.capitalize(tag) end) + |> Enum.join() + end + + defp with_fallback(f, fallback \\ []) do + try do + f.() + rescue + error -> + Logger.error("#{__MODULE__} search error: #{inspect(error)}") + fallback + end + end + defp get_author(%{account_id: account_id}) when is_binary(account_id), do: User.get_cached_by_id(account_id) From 41db5c8653db15e2d537f27d2af35f8a5c066d34 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 10:18:01 +0300 Subject: [PATCH 41/94] Add logging to milisiearch index and make it use desc(id) --- lib/mix/tasks/pleroma/search/meilisearch.ex | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2af8e5853..1fece96e5 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,8 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - import Mix.Pleroma + require Logger + import Mix.Pleroma import Ecto.Query def run(["index"]) do @@ -12,12 +13,25 @@ def run(["index"]) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/ranking-rules", + Jason.encode!([ + "desc(id)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ) + Pleroma.Repo.chunk_stream( from(Pleroma.Object, - limit: 200, where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") ), - 100, + 200, :batches ) |> Stream.map(fn objects -> @@ -26,12 +40,14 @@ def run(["index"]) do %{id: object.id, source: data["source"], ap: data["id"]} end) end) - |> Stream.each(fn activities -> + |> Stream.each(fn objects -> {:ok, _} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!(activities) + Jason.encode!(objects) ) + + IO.puts("Indexed #{Enum.count(objects)} entries") end) |> Stream.run() end From e5ac2ffa0791c043309fedd56a38d5cb394712ec Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:24:31 +0300 Subject: [PATCH 42/94] Ensure only indexing public posts and implement clearing and delete --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++++- lib/pleroma/activity.ex | 1 + lib/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- lib/pleroma/web/common_api.ex | 7 +++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 1fece96e5..0b86fdece 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants import Mix.Pleroma import Ecto.Query @@ -29,7 +30,11 @@ def run(["index"]) do Pleroma.Repo.chunk_stream( from(Pleroma.Object, - where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) ), 200, :batches @@ -51,4 +56,12 @@ def run(["index"]) do end) |> Stream.run() end + + def run(["clear"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + end end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 10b1b0120..d1d2ad9d1 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -369,6 +369,7 @@ def restrict_deactivated_users(query) do defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 92e0d3429..dbe6b2d67 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants alias Pleroma.Activity @@ -41,7 +42,8 @@ def search(user, query, options \\ []) do def add_to_index(activity) do object = activity.object - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + Pleroma.Constants.as_public() in object.data["to"] do data = object.data endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -57,4 +59,17 @@ def add_to_index(activity) do end end end + + def remove_from_index(object) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + "#{endpoint}/indexes/objects/documents/#{object.id}", + "", + [], + [] + ) + end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 92afd5cb6..0b7be42e0 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -146,6 +146,13 @@ def delete(activity_id, user) do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + {:ok, delete} else {:find_activity, _} -> From abf82a63ec242885672e7add20ddfc9554d7f81d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:30:56 +0300 Subject: [PATCH 43/94] Make the indexing batch differently and more, show number indexed --- lib/mix/tasks/pleroma/search/meilisearch.ex | 65 ++++++++++++--------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 0b86fdece..2a6438528 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,33 +28,46 @@ def run(["index"]) do ]) ) - Pleroma.Repo.chunk_stream( - from(Pleroma.Object, - # Only index public posts which are notes and have some text - where: - fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) - ), - 200, - :batches - ) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} - end) - end) - |> Stream.each(fn objects -> - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) - ) + chunk_size = 100_000 - IO.puts("Indexed #{Enum.count(objects)} entries") - end) - |> Stream.run() + Pleroma.Repo.transaction( + fn -> + Pleroma.Repo.stream( + from(Pleroma.Object, + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + order_by: fragment("data->'published' DESC") + ), + timeout: :infinity + ) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn objects -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(objects) + ) + end) + |> Stream.run() + end, + timeout: :infinity + ) end def run(["clear"]) do From 7b3701e6b9de337b2d04cdca6d70fdcbb3cb48c9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:06:32 +0300 Subject: [PATCH 44/94] Make meilisearch sort on publish date converted to unix time --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a6438528..2dd9c0a62 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,7 +18,7 @@ def run(["index"]) do Pleroma.HTTP.post( "#{endpoint}/indexes/objects/settings/ranking-rules", Jason.encode!([ - "desc(id)", + "desc(published)", "typo", "words", "proximity", @@ -54,7 +54,15 @@ def run(["index"]) do |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} + + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } end) end) |> Stream.each(fn objects -> From c3a04166a0d93db56c054e203024c113958ecaeb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:30:14 +0300 Subject: [PATCH 45/94] Tweak search ordering to hopefully return newer results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++--- lib/pleroma/search/meilisearch.ex | 13 +++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2dd9c0a62..dcecbd7cf 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -39,7 +39,7 @@ def run(["index"]) do fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), - order_by: fragment("data->'published' DESC") + order_by: [desc: fragment("data->'published'")] ), timeout: :infinity ) @@ -66,11 +66,15 @@ def run(["index"]) do end) end) |> Stream.each(fn objects -> - {:ok, _} = + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", Jason.encode!(objects) ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to index: #{result}") + end end) |> Stream.run() end, @@ -83,6 +87,11 @@ def run(["clear"]) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + {:ok, result} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to clear: #{result}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index dbe6b2d67..9fdb0a07f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -32,7 +32,7 @@ def search(user, query, options \\ []) do |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) |> maybe_fetch(user, query) - |> order_by([activity], desc: activity.id) + |> order_by([object: obj], desc: obj.data["published"]) |> Pleroma.Repo.all() rescue _ -> maybe_fetch([], user, query) @@ -48,10 +48,19 @@ def add_to_index(activity) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + Jason.encode!([ + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } + ]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do From a586ce0ddd1e360dba6906b00830e2094e219596 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:57:53 +0300 Subject: [PATCH 46/94] Use content instead of source and scrub it --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++-------- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index dcecbd7cf..5270de255 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -37,7 +37,7 @@ def run(["index"]) do # Only index public posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and + fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] ), @@ -56,10 +56,11 @@ def run(["index"]) do data = object.data {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, content} = FastSanitize.strip_tags(data["content"]) %{ id: object.id, - source: data["source"], + content: content, ap: data["id"], published: published |> DateTime.to_unix() } @@ -87,11 +88,6 @@ def run(["clear"]) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) - - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to clear: #{result}") - end + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 9fdb0a07f..87fdeaf5e 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -56,7 +56,7 @@ def add_to_index(activity) do Jason.encode!([ %{ id: object.id, - source: data["source"], + content: data["content"] |> Pleroma.HTML.filter_tags(), ap: data["id"], published: published |> DateTime.to_unix() } From 52a872432d344bdb53cd397c20131bb1a1f10684 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 01:37:43 +0300 Subject: [PATCH 47/94] Make the chunk size smaller --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5270de255..44af25f3e 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,7 +28,7 @@ def run(["index"]) do ]) ) - chunk_size = 100_000 + chunk_size = 10_000 Pleroma.Repo.transaction( fn -> From b3401ba7bdd087fa0fcff684e07a2872acaa73ab Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 16:37:52 +0300 Subject: [PATCH 48/94] Also index incoming federated posts --- lib/pleroma/search/search.ex | 18 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++----- lib/pleroma/web/activity_pub/side_effects.ex | 7 +++++++ lib/pleroma/web/common_api.ex | 8 ++------ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/search/search.ex diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex new file mode 100644 index 000000000..e363abf19 --- /dev/null +++ b/lib/pleroma/search/search.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 615bee428..5f068cef0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,11 +140,8 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + # Add local posts to search index + Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 9c2f89e72..98911f28f 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -193,6 +193,7 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do # - Increase replies count # - Set up ActivityExpiration # - Set up notifications + # - Index incoming posts for search (if needed) @impl true def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), @@ -222,6 +223,8 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + meta = meta |> add_notifications(notifications) @@ -281,6 +284,7 @@ def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do # - Reduce the user note count # - Reduce the reply count # - Stream out the activity + # - Removes posts from search index (if needed) @impl true def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = @@ -320,6 +324,9 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, if result == :ok do Notification.create_notifications(object) + + Pleroma.Search.remove_from_index(object) + {:ok, object, meta} else {:error, result} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 0b7be42e0..b4887d424 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -146,12 +146,8 @@ def delete(activity_id, user) do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + # Remove from search index for local posts + Pleroma.Search.remove_from_index(object) {:ok, delete} else From 14ef6ce80f5a10b8ce13f6bf059ef63ba46eaee5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 18:47:41 +0300 Subject: [PATCH 49/94] Mark only content as searchable for meilisearch --- lib/mix/tasks/pleroma/search/meilisearch.ex | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 44af25f3e..ebd3cc81f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,6 +28,14 @@ def run(["index"]) do ]) ) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/searchable-attributes", + Jason.encode!([ + "content" + ]) + ) + chunk_size = 10_000 Pleroma.Repo.transaction( @@ -55,8 +63,14 @@ def run(["index"]) do Enum.map(objects, fn object -> data = object.data + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(data["content"]) + {:ok, content} = FastSanitize.strip_tags(content_str) %{ id: object.id, From 117f525fd6106ed7f16d4d6fcc687e1b53c0b81d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 19:38:03 +0300 Subject: [PATCH 50/94] Adjust content indexing to skip more unneeded stuff --- lib/mix/tasks/pleroma/search/meilisearch.ex | 45 ++++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index ebd3cc81f..3704e0bdc 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -52,13 +52,6 @@ def run(["index"]) do timeout: :infinity ) |> Stream.chunk_every(chunk_size) - |> Stream.transform(0, fn objects, acc -> - new_acc = acc + Enum.count(objects) - - IO.puts("Indexed #{new_acc} entries") - - {[objects], new_acc} - end) |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data @@ -70,15 +63,34 @@ def run(["index"]) do end {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(content_str) - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Only index if there is anything in the string. If there is a single symbol, + # it's probably a dot from mastodon posts with just the picture + if String.length(content) > 1 do + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + else + nil + end end) + |> Enum.filter(fn o -> not is_nil(o) end) + end) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} end) |> Stream.each(fn objects -> {:ok, result} = @@ -102,6 +114,9 @@ def run(["clear"]) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) + {:ok, _} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], + timeout: :infinity + ) end end From 0cf365490781fc7d77b43e69bb7de90bb3d1c044 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 22:53:18 +0300 Subject: [PATCH 51/94] Rework task indexing to share code with the main module The code in the main module now scrubs new posts too --- lib/mix/tasks/pleroma/search/meilisearch.ex | 35 +--------------- lib/pleroma/search/meilisearch.ex | 46 ++++++++++++++------- 2 files changed, 34 insertions(+), 47 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3704e0bdc..b5a394e34 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -51,40 +51,9 @@ def run(["index"]) do ), timeout: :infinity ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) |> Stream.chunk_every(chunk_size) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - - {:ok, published, _} = DateTime.from_iso8601(data["published"]) - - content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Only index if there is anything in the string. If there is a single symbol, - # it's probably a dot from mastodon posts with just the picture - if String.length(content) > 1 do - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } - else - nil - end - end) - |> Enum.filter(fn o -> not is_nil(o) end) - end) |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 87fdeaf5e..10468e36c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -39,28 +39,46 @@ def search(user, query, options \\ []) do end end - def add_to_index(activity) do - object = activity.object - - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + def object_to_search_data(object) do + if not is_nil(object) and object.data["type"] == "Note" and Pleroma.Constants.as_public() in object.data["to"] do data = object.data - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + if String.length(content) > 1 do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + end + end + end + + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity) + + if activity.data["type"] == "Create" and maybe_search_data do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([ - %{ - id: object.id, - content: data["content"] |> Pleroma.HTML.filter_tags(), - ap: data["id"], - published: published |> DateTime.to_unix() - } - ]) + Jason.encode!([maybe_search_data]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do From 5360cc109706024c5c1687b9031441c3b7da5a86 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 23:47:43 +0300 Subject: [PATCH 52/94] Make indexing logs rewrite themselves --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index b5a394e34..2485a441d 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -57,7 +57,9 @@ def run(["index"]) do |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{new_acc} entries") + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") {[objects], new_acc} end) @@ -76,6 +78,8 @@ def run(["index"]) do end, timeout: :infinity ) + + IO.write("\n") end def run(["clear"]) do From dbf556cdcf3ab02e31bbebb64451c742a5481e5b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 19:35:21 +0300 Subject: [PATCH 53/94] Implement meilisearch auth --- lib/mix/tasks/pleroma/search/meilisearch.ex | 75 ++++++++++++--------- lib/pleroma/search/meilisearch.ex | 69 ++++++++++++------- 2 files changed, 88 insertions(+), 56 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2485a441d..230be5aa1 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,32 +9,30 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + def run(["index"]) do start_pleroma() - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + meili_post!( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ] + ) - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/ranking-rules", - Jason.encode!([ - "desc(published)", - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]) - ) - - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/searchable-attributes", - Jason.encode!([ - "content" - ]) - ) + meili_post!( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) chunk_size = 10_000 @@ -64,14 +62,14 @@ def run(["index"]) do {[objects], new_acc} end) |> Stream.each(fn objects -> - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + result = + meili_post!( + "/indexes/objects/documents", + objects ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to index: #{result}") + if not Map.has_key?(result, "updateId") do + IO.puts("Failed to index: #{inspect(result)}") end end) |> Stream.run() @@ -85,11 +83,26 @@ def run(["index"]) do def run(["clear"]) do start_pleroma() + meili_delete!("/indexes/objects/documents") + end + + def run(["show-private-key", master_key]) do + start_pleroma() + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], - timeout: :infinity + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"X-Meili-API-Key", master_key}] ) + + decoded = Jason.decode!(result.body) + + if decoded["private"] do + IO.puts(decoded["private"]) + else + IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 10468e36c..8745d539d 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,20 +7,50 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query + defp meili_headers() do + private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + end + + def meili_post!(path, params) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers() + ) + + Jason.decode!(result.body) + end + + def meili_delete!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + timeout: :infinity + ) + end + def search(user, query, options \\ []) do limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/search", - Jason.encode!(%{q: query, offset: offset, limit: limit}) + result = + meili_post!( + "/indexes/objects/search", + %{q: query, offset: offset, limit: limit} ) - hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + hits = result["hits"] |> Enum.map(& &1["ap"]) try do hits @@ -73,30 +103,19 @@ def add_to_index(activity) do maybe_search_data = object_to_search_data(activity) if activity.data["type"] == "Create" and maybe_search_data do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!([maybe_search_data]) + result = + meili_post!( + "/indexes/objects/documents", + [maybe_search_data] ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + if not Map.has_key?(result, "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end def remove_from_index(object) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.request( - :delete, - "#{endpoint}/indexes/objects/documents/#{object.id}", - "", - [], - [] - ) + meili_delete!("/indexes/objects/documents/#{object.id}") end end From d5cc272a919de7199f83c0921d2db05df907776b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:02:34 +0300 Subject: [PATCH 54/94] Add a message with a count of posts to index --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 230be5aa1..557b06182 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ def run(["index"]) do Pleroma.Repo.transaction( fn -> - Pleroma.Repo.stream( + query = from(Pleroma.Object, # Only index public posts which are notes and have some text where: @@ -46,7 +46,13 @@ def run(["index"]) do fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] - ), + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, timeout: :infinity ) |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) From d1079f1aa307a40ae2adeff095b27adc87c5dccd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:21:46 +0300 Subject: [PATCH 55/94] Add the meilisearch.stats command --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 +++++++++- lib/pleroma/search/meilisearch.ex | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 557b06182..f2d9fe312 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] def run(["index"]) do start_pleroma() @@ -111,4 +111,12 @@ def run(["show-private-key", master_key]) do IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") end end + + def run(["stats"]) do + start_pleroma() + + result = meili_get!("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 8745d539d..1ad17bf9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,6 +13,18 @@ defp meili_headers() do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end + def meili_get!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, path), + meili_headers() + ) + + Jason.decode!(result.body) + end + def meili_post!(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) From 375154e5de99093fd79793d8e5ed979ff34c63c1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:27:16 +0300 Subject: [PATCH 56/94] Add search/meilisearch documentation --- docs/configuration/search.md | 99 ++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 docs/configuration/search.md diff --git a/docs/configuration/search.md b/docs/configuration/search.md new file mode 100644 index 000000000..14ec2bc63 --- /dev/null +++ b/docs/configuration/search.md @@ -0,0 +1,99 @@ +# Configuring search + +{! backend/administration/CLI_tasks/general_cli_task_info.include !} + +## Built-in search + +To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`: + +> config :pleroma, Pleroma.Search, module: Pleroma.Activity + +While it has no external dependencies, it has problems with performance and relevancy. + +## Meilisearch + +To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`: + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch + +You then need to set the address of the meilisearch instance, and optionally the private key for authentication. + +> config :pleroma, Pleroma.Search.Meilisearch, +> url: "http://127.0.0.1:7700/", +> private_key: "private key" + +Information about setting up meilisearch can be found in the +[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html). +You probably want to start it with `MEILI_NO_ANALYTICS=true` and `MEILI_NO_CENTRY=true` environment variables, +to disable analytics. + +### Private key authentication (optional) + +To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_, +you have to get the _private key_, which is actually used for authentication. + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.meilisearch show-private-key + ``` + +=== "From Source" + ```sh + mix pleroma.search.meilisearch show-private-key + ``` + +This is the key you actually put into your configuration file. + +### Initial indexing + +After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed. You'll only +have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM +consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 +million posts while idle and up to 7G while indexing initially, but your experience may be different). + +To start te initial indexing, run the `index` command: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.meilisearch index + ``` + +=== "From Source" + ```sh + mix pleroma.search.meilisearch index + ``` + +This will show you the total amount of posts to index, and then show you the amount of posts indexed currently, until the numbers eventually +become the same. The posts are indexed in big batches and meilisearch will take some time to actually index them, even after you have +inserted all the posts into it. Depending on the amount of posts, this may be as long as several hours. To get information about the status +of indexing and how many posts have actually been indexed, use the `stats` command: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.meilisearch stats + ``` + +=== "From Source" + ```sh + mix pleroma.search.meilisearch stats + ``` + +### Clearing the index + +In case you need to clear the index (for example, to re-index from scratch, if that needs to happen for some reason), you can +use the `clear` command: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.meilisearch clear + ``` + +=== "From Source" + ```sh + mix pleroma.search.meilisearch clear + ``` + +This will clear **all** the posts from the search index. Note, that deleted posts are also removed from index by the instance itself, so +there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information +that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size +depends on the amount of text in posts. From 41f35721529089045e2421991035ba362fc34166 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 21:15:15 +0300 Subject: [PATCH 57/94] Fix activity being passed to objec_to_search_data --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 1ad17bf9f..212bdd473 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -112,7 +112,7 @@ def object_to_search_data(object) do end def add_to_index(activity) do - maybe_search_data = object_to_search_data(activity) + maybe_search_data = object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do result = From 88891e9d764f43389a0b7daf6a39b6648e95d30d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 23:52:21 +0300 Subject: [PATCH 58/94] Add tests for local post indexing for meilisearch --- config/test.exs | 4 +- test/pleroma/search/meilisearch_test.exs | 108 +++++++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 test/pleroma/search/meilisearch_test.exs diff --git a/config/test.exs b/config/test.exs index 445975205..4d8d73502 100644 --- a/config/test.exs +++ b/config/test.exs @@ -134,7 +134,9 @@ ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, logger: Pleroma.LoggerMock -config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search +config :pleroma, Pleroma.Search, module: Pleroma.Activity + +config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil # Reduce recompilation time # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs new file mode 100644 index 000000000..6e13c8edf --- /dev/null +++ b/test/pleroma/search/meilisearch_test.exs @@ -0,0 +1,108 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.MeilisearchTest do + require Pleroma.Constants + + use Pleroma.DataCase + + import Pleroma.Factory + import Tesla.Mock + import Mock + + alias Pleroma.Web.CommonAPI + alias Pleroma.Search.Meilisearch + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "meilisearch" do + setup do: clear_config([Pleroma.Search, :module], Meilisearch) + + setup_with_mocks( + [ + {Meilisearch, [:passthrough], + [ + add_to_index: fn a -> passthrough([a]) end, + remove_from_index: fn a -> passthrough([a]) end + ]} + ], + context, + do: {:ok, context} + ) + + test "indexes a local post on creation" do + user = insert(:user) + + mock_global(fn + %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> + assert match?( + [%{"content" => "guys i just don't wanna leave the swamp"}], + Jason.decode!(body) + ) + + json(%{updateId: 1}) + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + assert_called(Meilisearch.add_to_index(activity)) + end + + test "doesn't index posts that are not public" do + user = insert(:user) + + Enum.each(["unlisted", "private", "direct"], fn visiblity -> + {:ok, _} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visiblity + }) + end) + + history = call_history(Meilisearch) + assert Enum.count(history) == 3 + + Enum.each(history, fn {_, _, return} -> + assert is_nil(return) + end) + end + + test "deletes posts from index when deleted locally" do + user = insert(:user) + + mock_global(fn + %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> + assert match?( + [%{"content" => "guys i just don't wanna leave the swamp"}], + Jason.decode!(body) + ) + + json(%{updateId: 1}) + + %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} -> + assert String.length(id) > 1 + json(%{updateId: 2}) + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + assert_called(Meilisearch.add_to_index(activity)) + + {:ok, _} = CommonAPI.delete(activity.id, user) + + assert_called(Meilisearch.remove_from_index(:_)) + end + end +end From b51972657197bf4c37ebca192be3d7c8a2409610 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 23:52:37 +0300 Subject: [PATCH 59/94] Add private_key: nil to default meilisearch options --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 7df0521f2..42373551d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -857,7 +857,7 @@ config :pleroma, :search, provider: Pleroma.Search.Builtin config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search -config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/" +config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. From 5ed17590915b31aa696befc1a65e3a953611335a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 28 Aug 2021 15:59:13 +0300 Subject: [PATCH 60/94] Reorder ranking rules for (maybe) better results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index f2d9fe312..cdf9ab0bd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,12 +18,12 @@ def run(["index"]) do "/indexes/objects/settings/ranking-rules", [ "desc(published)", - "typo", "words", + "exactness", "proximity", - "attribute", "wordsPosition", - "exactness" + "typo", + "attribute" ] ) From 0802c0666bbf208d0d7c89625b346c573621fca9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 8 Oct 2021 12:24:37 +0300 Subject: [PATCH 61/94] Move add_to_index / remove_from_index to Pleroma.Actitivy.Search --- lib/pleroma/activity.ex | 2 -- lib/pleroma/activity/search.ex | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index d1d2ad9d1..4106feef6 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,8 +368,6 @@ def restrict_deactivated_users(query) do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 7152b0e46..8352ba20a 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -45,6 +45,9 @@ def search(user, search_query, options \\ []) do end end + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end From 9ab0f130ccd71af3678e434db305bcb44336a121 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:14:39 +0300 Subject: [PATCH 62/94] Add more documentation about rum to meilisearch docs --- docs/configuration/search.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 14ec2bc63..e9743f1a4 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -12,6 +12,15 @@ While it has no external dependencies, it has problems with performance and rele ## Meilisearch +Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million +posts while idle and up to 7G while indexing initially). The disk usage for this additional index is also +around 4 gigabytes. Like [RUM](./cheatsheet.md#rum-indexing-for-full-text-search) indexes, it offers considerably +higher performance and ordering by timestamp in a reasonable amount of time. +Additionally, the search results seem to be more accurate. + +Due to high memory usage, it may be best to set it up on a different machine, if running pleroma on a low-resource +computer, and use private key authentication to secure the remote search instance. + To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`: > config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch From a937a98df5a118a3643a75175507233ed4ac9230 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:17:37 +0300 Subject: [PATCH 63/94] Don't try removing from index again in common_api It's already removed in the side effects of the pipeline --- lib/pleroma/web/common_api.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b4887d424..92afd5cb6 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -146,9 +146,6 @@ def delete(activity_id, user) do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Remove from search index for local posts - Pleroma.Search.remove_from_index(object) - {:ok, delete} else {:find_activity, _} -> From 37913c8ddf6910a4b53d905a1679d4f30c88c98d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:34:57 +0300 Subject: [PATCH 64/94] Use proper deleted object for removing from index --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 98911f28f..097afa30e 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -325,7 +325,7 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else From afcdd2f4379dc57730c348b5224b5821e952c5b6 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 00:38:00 +0300 Subject: [PATCH 65/94] Modify some meilisearch variables --- config/config.exs | 2 +- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 42373551d..c2b4ca996 100644 --- a/config/config.exs +++ b/config/config.exs @@ -851,7 +851,7 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, - {Pleroma.Search, [max_running: 20, max_waiting: 50]} + {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] config :pleroma, :search, provider: Pleroma.Search.Builtin diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 212bdd473..b8248e40c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -47,7 +47,7 @@ def meili_delete!(path) do Path.join(endpoint, path), "", meili_headers(), - timeout: :infinity + [] ) end From 51faa28568c869f8a63b3c07ddd78f1c1e04f6b7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 13:58:24 +0300 Subject: [PATCH 66/94] Set content-type to application/json --- lib/pleroma/search/meilisearch.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index b8248e40c..d94ab8b64 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,7 +10,8 @@ defmodule Pleroma.Search.Meilisearch do defp meili_headers() do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + [{"Content-Type", "application/json"}] ++ + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end def meili_get!(path) do From d99a2be3517af8941a3d8a59af9e0b297a587edb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 21:04:59 +0300 Subject: [PATCH 67/94] Only add local posts to index in activity_pub Remote ones are already added in another place --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5f068cef0..e6548a818 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when end) # Add local posts to search index - Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else From 9e7d7ebd48faa14ac8be37290f02d820a8062470 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 10 Nov 2021 21:25:12 +0300 Subject: [PATCH 68/94] Add a reindex option Signed-off-by: Ekaterina Vaartis --- lib/mix/tasks/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index cdf9ab0bd..2a3c3a8b9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -11,9 +11,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] - def run(["index"]) do + def run(["index" | args]) do start_pleroma() + is_reindex = "--reindex" in args + meili_post!( "/indexes/objects/settings/ranking-rules", [ @@ -68,6 +70,19 @@ def run(["index"]) do {[objects], new_acc} end) |> Stream.each(fn objects -> + objects = + objects + |> Enum.filter(fn o -> + if is_reindex do + result = meili_get!("/indexes/objects/documents/#{o.id}") + + # Filter out the already indexed documents. This is true when the document does not exist + result["errorCode"] == "document_not_found" + else + true + end + end) + result = meili_post!( "/indexes/objects/documents", From 86971fceaab2c5b0ebc4fc5e90bae3930648e78d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 13 Nov 2021 15:07:51 +0300 Subject: [PATCH 69/94] Support reindexing meilisearch >=0.24.0 It has has a different error code key --- lib/mix/tasks/pleroma/search/meilisearch.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a3c3a8b9..3b134ad3f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -76,8 +76,14 @@ def run(["index" | args]) do if is_reindex do result = meili_get!("/indexes/objects/documents/#{o.id}") + # With >= 0.24.0 the name for "errorCode" is just "code" + error_code_key = + if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), + do: "code", + else: "errorCode" + # Filter out the already indexed documents. This is true when the document does not exist - result["errorCode"] == "document_not_found" + result[error_code_key] == "document_not_found" else true end From 7a9d9cf45728a2c6f2a426053ddcff7b9fb0c2d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 20:15:12 +0300 Subject: [PATCH 70/94] Fix a typo in search docs --- docs/configuration/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index e9743f1a4..9adc7884f 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -60,7 +60,7 @@ have to do it one time, but it might take a while, depending on the amount of po consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). -To start te initial indexing, run the `index` command: +To start the initial indexing, run the `index` command: === "OTP" ```sh From 5dd908552cacb93d3622ff176499f24c96ff5431 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 20:24:05 +0300 Subject: [PATCH 71/94] Move the search.ex file so credo doesn't complain --- lib/pleroma/search.ex | 24 +++++++++++++++--------- lib/pleroma/search/search.ex | 18 ------------------ 2 files changed, 15 insertions(+), 27 deletions(-) delete mode 100644 lib/pleroma/search/search.ex diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 99bce632c..e363abf19 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,12 +1,18 @@ defmodule Pleroma.Search do - @type search_map :: %{ - statuses: [map], - accounts: [map], - hashtags: [map] - } + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) - @doc """ - Searches for stuff - """ - @callback search(map, map, keyword) :: search_map + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end end diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex deleted file mode 100644 index e363abf19..000000000 --- a/lib/pleroma/search/search.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) - end - - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) - end -end From 0769f06bd172a51b3875f4d56db31ceff4a79d06 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 21:42:18 +0300 Subject: [PATCH 72/94] Style fixes --- lib/mix/tasks/pleroma/search/meilisearch.ex | 3 ++- lib/pleroma/search/meilisearch.ex | 2 +- test/pleroma/search/meilisearch_test.exs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3b134ad3f..62ace7e39 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -82,7 +82,8 @@ def run(["index" | args]) do do: "code", else: "errorCode" - # Filter out the already indexed documents. This is true when the document does not exist + # Filter out the already indexed documents. + # This is true when the document does not exist result[error_code_key] == "document_not_found" else true diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index d94ab8b64..41f99ad9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query - defp meili_headers() do + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index 6e13c8edf..251388ea2 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -11,8 +11,8 @@ defmodule Pleroma.Search.MeilisearchTest do import Tesla.Mock import Mock - alias Pleroma.Web.CommonAPI alias Pleroma.Search.Meilisearch + alias Pleroma.Web.CommonAPI setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) From ea582fbf936feffce9596192722831426879edbd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 16 Nov 2021 21:54:26 +0300 Subject: [PATCH 73/94] Rename search.ex to database_search.ex and add search/2 --- lib/pleroma/{search.ex => search/database_search.ex} | 8 +++++++- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 ++-- .../web/mastodon_api/controllers/search_controller.ex | 4 +--- 4 files changed, 11 insertions(+), 7 deletions(-) rename lib/pleroma/{search.ex => search/database_search.ex} (68%) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search/database_search.ex similarity index 68% rename from lib/pleroma/search.ex rename to lib/pleroma/search/database_search.ex index e363abf19..be0e19be0 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,4 +1,4 @@ -defmodule Pleroma.Search do +defmodule Pleroma.Search.DatabaseSearch do def add_to_index(activity) do search_module = Pleroma.Config.get([Pleroma.Search, :module]) @@ -15,4 +15,10 @@ def remove_from_index(object) do Task.start(fn -> search_module.remove_from_index(object) end) end) end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e6548a818..92b1be62c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when end) # Add local posts to search index - if local, do: Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 097afa30e..d08d6aa70 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -223,7 +223,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -325,7 +325,7 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index aaf52cdc5..05fa4144d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -76,9 +76,7 @@ defp resource_search(_, "accounts", query, options) do end defp resource_search(_, "statuses", query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - - statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) StatusView.render("index.json", activities: statuses, From c12879841836035da0a4dc2117459801f16a27fb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 17 Nov 2021 22:29:49 +0300 Subject: [PATCH 74/94] Rename Activity.Search to Search.DatabaseSearch --- config/config.exs | 3 +- config/test.exs | 2 +- docs/configuration/search.md | 2 +- lib/pleroma/activity.ex | 2 +- lib/pleroma/activity/search.ex | 153 ----------------- lib/pleroma/search.ex | 24 +++ lib/pleroma/search/database_search.ex | 157 ++++++++++++++++-- lib/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 +- .../controllers/search_controller.ex | 2 +- .../database_search_test.ex} | 10 +- 12 files changed, 182 insertions(+), 181 deletions(-) delete mode 100644 lib/pleroma/activity/search.ex create mode 100644 lib/pleroma/search.ex rename test/pleroma/{activity/search_test.exs => search/database_search_test.ex} (81%) diff --git a/config/config.exs b/config/config.exs index c2b4ca996..731804503 100644 --- a/config/config.exs +++ b/config/config.exs @@ -856,7 +856,8 @@ config :pleroma, :search, provider: Pleroma.Search.Builtin -config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search +config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch + config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil # Import environment specific config. This must remain at the bottom diff --git a/config/test.exs b/config/test.exs index 4d8d73502..7fbababdf 100644 --- a/config/test.exs +++ b/config/test.exs @@ -134,7 +134,7 @@ ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, logger: Pleroma.LoggerMock -config :pleroma, Pleroma.Search, module: Pleroma.Activity +config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 9adc7884f..c7e77d9c2 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -6,7 +6,7 @@ To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`: -> config :pleroma, Pleroma.Search, module: Pleroma.Activity +> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch While it has no external dependencies, it has problems with performance and relevancy. diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4106feef6..abfe778d2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,7 +367,7 @@ def restrict_deactivated_users(query) do from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex deleted file mode 100644 index 8352ba20a..000000000 --- a/lib/pleroma/activity/search.ex +++ /dev/null @@ -1,153 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Search do - alias Pleroma.Activity - alias Pleroma.Object.Fetcher - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - - require Pleroma.Constants - - import Ecto.Query - - def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) - - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public() - |> query_with(index_type, search_query, search_function) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) - end - end - - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil - - def maybe_restrict_author(query, %User{} = author) do - Activity.Queries.by_author(query, author) - end - - def maybe_restrict_author(query, _), do: query - - def maybe_restrict_blocked(query, %User{} = user) do - Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) - end - - def maybe_restrict_blocked(query, _), do: query - - def restrict_public(q) do - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: ^Pleroma.Constants.as_public() in a.recipients - ) - end - - defp query_with(q, :gin, search_query, :plain) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :gin, search_query, :websearch) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :rum, search_query, :plain) do - from([a, o] in q, - where: - fragment( - "? @@ plainto_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - defp query_with(q, :rum, search_query, :websearch) do - from([a, o] in q, - where: - fragment( - "? @@ websearch_to_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - - case {limit, user} do - {:all, _} -> restrict_local(q) - {:unauthenticated, %User{}} -> q - {:unauthenticated, _} -> restrict_local(q) - {false, _} -> q - end - end - - defp restrict_local(q), do: where(q, local: true) - - def maybe_fetch(activities, user, search_query) do - with true <- Regex.match?(~r/https?:/, search_query), - {:ok, object} <- Fetcher.fetch_object_from_id(search_query), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), - true <- Visibility.visible_for_user?(activity, user) do - [activity | activities] - else - _ -> activities - end - end -end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..ae0b28c54 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index be0e19be0..5a8b8ca67 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,24 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Search.DatabaseSearch do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Activity + alias Pleroma.Object.Fetcher + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + require Pleroma.Constants + + import Ecto.Query + + def search(user, search_query, options \\ []) do + index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + search_function = + if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do + :websearch + else + :plain + end + + try do + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public() + |> query_with(index_type, search_query, search_function) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + |> maybe_fetch(user, search_query) + rescue + _ -> maybe_fetch([], user, search_query) + end end - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + def maybe_restrict_author(query, %User{} = author) do + Activity.Queries.by_author(query, author) end - def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + def maybe_restrict_author(query, _), do: query - search_module.search(options[:for_user], query, options) + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + + def restrict_public(q) do + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + + defp query_with(q, :gin, search_query, :plain) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :gin, search_query, :websearch) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :rum, search_query, :plain) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + + defp query_with(q, :rum, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "? @@ websearch_to_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + + def maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end + end + + defp restrict_local(q), do: where(q, local: true) + + def maybe_fetch(activities, user, search_query) do + with true <- Regex.match?(~r/https?:/, search_query), + {:ok, object} <- Fetcher.fetch_object_from_id(search_query), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + [activity | activities] + else + _ -> activities + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 41f99ad9f..fa9e27b03 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do alias Pleroma.Activity - import Pleroma.Activity.Search + import Pleroma.Search.DatabaseSearch import Ecto.Query defp meili_headers do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 92b1be62c..e6548a818 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when end) # Add local posts to search index - if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index d08d6aa70..097afa30e 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -223,7 +223,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -325,7 +325,7 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, if result == :ok do Notification.create_notifications(object) - Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 05fa4144d..751d46cdf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -76,7 +76,7 @@ defp resource_search(_, "accounts", query, options) do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/search/database_search_test.ex similarity index 81% rename from test/pleroma/activity/search_test.exs rename to test/pleroma/search/database_search_test.ex index 657fbc627..2387ac29b 100644 --- a/test/pleroma/activity/search_test.exs +++ b/test/pleroma/search/database_search_test.ex @@ -2,8 +2,8 @@ # Copyright ยฉ 2017-2021 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Activity.SearchTest do - alias Pleroma.Activity.Search +defmodule Pleroma.Search.DatabaseSearchTest do + alias Pleroma.Search.DatabaseSearch alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -13,7 +13,7 @@ test "it finds something" do user = insert(:user) {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) - [result] = Search.search(nil, "wednesday") + [result] = DatabaseSearch.search(nil, "wednesday") assert result.id == post.id end @@ -28,7 +28,7 @@ test "using plainto_tsquery on postgres < 11" do {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) # plainto doesn't understand complex queries - assert [result] = Search.search(nil, "wednesday -dudes") + assert [result] = DatabaseSearch.search(nil, "wednesday -dudes") assert result.id == post.id end @@ -38,7 +38,7 @@ test "using websearch_to_tsquery" do {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) - assert [result] = Search.search(nil, "wednesday -dudes") + assert [result] = DatabaseSearch.search(nil, "wednesday -dudes") assert result.id == other_post.id end From bac70a2bc155e8c714b0ea3cf83acb9583b71ec0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 22 Nov 2021 21:39:54 +0300 Subject: [PATCH 75/94] Implement suggestions from the Meilisearch MR - Index unlisted posts - Move version check outside of the streaming and only do it once - Use a PUT request instead of checking manually if there is need to insert - Add error handling, sort of --- lib/mix/tasks/pleroma/search/meilisearch.ex | 84 +++++++++----------- lib/pleroma/search/meilisearch.ex | 85 ++++++++++++++------- 2 files changed, 93 insertions(+), 76 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 62ace7e39..6730a99a9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,38 +3,40 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - require Logger require Pleroma.Constants import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] - def run(["index" | args]) do + def run(["index"]) do start_pleroma() - is_reindex = "--reindex" in args + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "words", + "exactness", + "proximity", + "wordsPosition", + "typo", + "attribute" + ] + ) - meili_post!( - "/indexes/objects/settings/ranking-rules", - [ - "desc(published)", - "words", - "exactness", - "proximity", - "wordsPosition", - "typo", - "attribute" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) - meili_post!( - "/indexes/objects/settings/searchable-attributes", - [ - "content" - ] - ) + IO.puts("Created indices. Starting to insert posts.") chunk_size = 10_000 @@ -42,11 +44,11 @@ def run(["index" | args]) do fn -> query = from(Pleroma.Object, - # Only index public posts which are notes and have some text + # Only index public and unlisted posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'content') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), order_by: [desc: fragment("data->'published'")] ) @@ -70,34 +72,18 @@ def run(["index" | args]) do {[objects], new_acc} end) |> Stream.each(fn objects -> - objects = - objects - |> Enum.filter(fn o -> - if is_reindex do - result = meili_get!("/indexes/objects/documents/#{o.id}") - - # With >= 0.24.0 the name for "errorCode" is just "code" - error_code_key = - if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), - do: "code", - else: "errorCode" - - # Filter out the already indexed documents. - # This is true when the document does not exist - result[error_code_key] == "document_not_found" - else - true - end - end) - result = - meili_post!( + meili_put( "/indexes/objects/documents", objects ) - if not Map.has_key?(result, "updateId") do - IO.puts("Failed to index: #{inspect(result)}") + with {:ok, res} <- result do + if not Map.has_key?(res, "updateId") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") end end) |> Stream.run() @@ -137,7 +123,7 @@ def run(["show-private-key", master_key]) do def run(["stats"]) do start_pleroma() - result = meili_get!("/indexes/objects/stats") + {:ok, result} = meili_get("/indexes/objects/stats") IO.puts("Number of entries: #{result["numberOfDocuments"]}") IO.puts("Indexing? #{result["isIndexing"]}") end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index fa9e27b03..21b44de86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -14,29 +14,50 @@ defp meili_headers do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end - def meili_get!(path) do + def meili_get(path) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.get( Path.join(endpoint, path), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end - def meili_post!(path, params) do + def meili_post(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.post( Path.join(endpoint, path), Jason.encode!(params), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_put(path, params) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.request( + :put, + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers(), + [] + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end def meili_delete!(path) do @@ -57,34 +78,40 @@ def search(user, query, options \\ []) do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - result = - meili_post!( + res = + meili_post( "/indexes/objects/search", %{q: query, offset: offset, limit: limit} ) - hits = result["hits"] |> Enum.map(& &1["ap"]) + with {:ok, result} <- res do + hits = result["hits"] |> Enum.map(& &1["ap"]) - try do - hits - |> Activity.create_by_object_ap_id() - |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> maybe_fetch(user, query) - |> order_by([object: obj], desc: obj.data["published"]) - |> Pleroma.Repo.all() - rescue - _ -> maybe_fetch([], user, query) + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([object: obj], desc: obj.data["published"]) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end end end def object_to_search_data(object) do + # Only index public or unlisted Notes if not is_nil(object) and object.data["type"] == "Note" and - Pleroma.Constants.as_public() in object.data["to"] do + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + String.length(object.data["content"]) > 1 do data = object.data content_str = @@ -117,13 +144,17 @@ def add_to_index(activity) do if activity.data["type"] == "Create" and maybe_search_data do result = - meili_post!( + meili_put( "/indexes/objects/documents", [maybe_search_data] ) - if not Map.has_key?(result, "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + with {:ok, res} <- result, + true <- Map.has_key?(res, "updateId") do + # Do nothing + else + _ -> + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end From 58cc5d13a2a1e16be3d9125f5f47b9240fb8a74a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 17:46:23 +0300 Subject: [PATCH 76/94] Add config description for meilisearch --- config/description.exs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/config/description.exs b/config/description.exs index 48e0c59a8..8d38f60ed 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3429,5 +3429,40 @@ ] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Search, + type: :group, + description: "General search settings.", + children: [ + %{ + key: :module, + type: :keyword, + description: "Selected search module.", + suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Search.Meilisearch, + type: :group, + description: "Meilisearch settings.", + children: [ + %{ + key: :url, + type: :string, + description: "Meilisearch URL.", + suggestion: ["http://127.0.0.1:7700/"] + }, + %{ + key: :private_key, + type: :string, + description: + "Private key for meilisearch authentication, or `nil` to disable private key authentication.", + suggestion: [nil] + } + ] } ] From 426cff3372be0e2176cc93c07aead9b170a53b00 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 18:16:33 +0300 Subject: [PATCH 77/94] Update search.md documentation with meilisearch indexing steps --- docs/configuration/search.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index c7e77d9c2..7dbbd3e17 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -60,6 +60,15 @@ have to do it one time, but it might take a while, depending on the amount of po consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). +The sequence of actions is as follows: + +1. First, change the configuration to use `Pleroma.Search.Meilisearch` as the search backend +2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything +3. Start the initial indexing process (as described below with `index`), + and wait until the task says it sent everything from the database to index +4. Wait until everything is actually indexed (by checking with `stats` as described below), + at this point you don't have to do anything, just wait a while. + To start the initial indexing, run the `index` command: === "OTP" From cc3319ac1d7aed4a8010c7869cb7a4d8c7dd0dd7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 18:48:52 +0300 Subject: [PATCH 78/94] Make chunk size configurable --- config/config.exs | 5 ++++- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 731804503..c49e16fe2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -858,7 +858,10 @@ config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch -config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil +config :pleroma, Pleroma.Search.Meilisearch, + url: "http://127.0.0.1:7700/", + private_key: nil, + initial_indexing_chunk_size: 100_000 # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 6730a99a9..021552f7b 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ def run(["index"]) do IO.puts("Created indices. Starting to insert posts.") - chunk_size = 10_000 + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) Pleroma.Repo.transaction( fn -> From a4914add8c549180c73b819f3974f3ca9aacd65f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 19:05:59 +0300 Subject: [PATCH 79/94] Don't support meilisearch < 0.24.0, since it breaks things --- lib/mix/tasks/pleroma/search/meilisearch.ex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 021552f7b..5098668ad 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -14,17 +14,29 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.24.0") do + raise "Meilisearch <0.24.0 not supported" + end + {:ok, _} = meili_post( "/indexes/objects/settings/ranking-rules", [ - "desc(published)", + "published:desc", "words", "exactness", "proximity", - "wordsPosition", "typo", - "attribute" + "attribute", + "sort" ] ) From 80e52f4d86b0120e0b9b8420e50e45bcef851b39 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 19:27:22 +0300 Subject: [PATCH 80/94] Add description for initial_indexing_chunk_size --- config/description.exs | 8 ++++++++ docs/configuration/search.md | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 8d38f60ed..2d068556f 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3462,6 +3462,14 @@ description: "Private key for meilisearch authentication, or `nil` to disable private key authentication.", suggestion: [nil] + }, + %{ + key: :initial_indexing_chunk_size, + type: :int, + description: + "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <> + " since there's a limit on maximum insert size", + suggestion: [100_000] } ] } diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 7dbbd3e17..a785a18ad 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -25,11 +25,15 @@ To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pl > config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch -You then need to set the address of the meilisearch instance, and optionally the private key for authentication. +You then need to set the address of the meilisearch instance, and optionally the private key for authentication. You might +also want to change the `initial_indexing_chunk_size` to be smaller if you're server is not very powerful, but not higher than `100_000`, +because meilisearch will refuse to process it if it's too big. However, in general you want this to be as big as possible, because meilisearch +indexes faster when it can process many posts in a single batch. > config :pleroma, Pleroma.Search.Meilisearch, > url: "http://127.0.0.1:7700/", -> private_key: "private key" +> private_key: "private key", +> initial_indexing_chunk_size: 100_000 Information about setting up meilisearch can be found in the [official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html). From 54b2a86f47f365c8a1b773e003ff5f841c0d2865 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 22:38:50 +0300 Subject: [PATCH 81/94] Add a search backend behaviour --- lib/pleroma/search/database_search.ex | 5 +++++ lib/pleroma/search/meilisearch.ex | 4 ++++ lib/pleroma/search/search_backend.ex | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 lib/pleroma/search/search_backend.ex diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 5a8b8ca67..3735a5fab 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -13,6 +13,8 @@ defmodule Pleroma.Search.DatabaseSearch do import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + def search(user, search_query, options \\ []) do index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) @@ -45,7 +47,10 @@ def search(user, search_query, options \\ []) do end end + @impl true def add_to_index(_activity), do: nil + + @impl true def remove_from_index(_object), do: nil def maybe_restrict_author(query, %User{} = author) do diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 21b44de86..33bbf8392 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Search.DatabaseSearch import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -139,6 +141,7 @@ def object_to_search_data(object) do end end + @impl true def add_to_index(activity) do maybe_search_data = object_to_search_data(activity.object) @@ -159,6 +162,7 @@ def add_to_index(activity) do end end + @impl true def remove_from_index(object) do meili_delete!("/indexes/objects/documents/#{object.id}") end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex new file mode 100644 index 000000000..ed6bfd329 --- /dev/null +++ b/lib/pleroma/search/search_backend.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.Search.SearchBackend do + @doc """ + Add the object associated with the activity to the search index. + + The whole activity is passed, to allow filtering on things such as scope. + """ + @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil + + @doc """ + Remove the object from the index. + + Just the object, as opposed to the whole activity, is passed, since the object + is what contains the actual content and there is no need for fitlering when removing + from index. + """ + @callback remove_from_index(object :: Pleroma.Object.t()) :: nil +end From 5def4a7d49659d7f3dca4049dd990c9e6a6ccb39 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 15:23:11 +0300 Subject: [PATCH 82/94] Use oban for search indexing --- config/config.exs | 3 ++- lib/pleroma/search.ex | 15 ++++--------- lib/pleroma/workers/search_indexing_worker.ex | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 lib/pleroma/workers/search_indexing_worker.ex diff --git a/config/config.exs b/config/config.exs index c49e16fe2..05beb9ba0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -568,7 +568,8 @@ remote_fetcher: 2, attachments_cleanup: 1, new_users_digest: 1, - mute_expire: 5 + mute_expire: 5, + search_indexing: 1 ], plugins: [Oban.Plugins.Pruner], crontab: [ diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index ae0b28c54..af858fc46 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,19 +1,12 @@ defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Workers.SearchIndexingWorker - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + def add_to_index(activity) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) end def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) end def search(query, options) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex new file mode 100644 index 000000000..43b7bad1e --- /dev/null +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -0,0 +1,21 @@ +defmodule Pleroma.Workers.SearchIndexingWorker do + use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + + @impl Oban.Worker + + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do + activity = Pleroma.Activity.get_by_id_with_object(activity_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.add_to_index(activity) + end + + def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do + object = Pleroma.Object.get_by_id(object_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.remove_from_index(object) + end +end From 7aebff799b57d27d97f7830fd7c3ef27875c33fe Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:31:32 +0300 Subject: [PATCH 83/94] Fix meilisearch tests and jobs for oban --- lib/pleroma/workers/search_indexing_worker.ex | 4 +++ test/pleroma/search/meilisearch_test.exs | 35 ++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 43b7bad1e..70a8d42d0 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -9,6 +9,8 @@ def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) + + :ok end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do @@ -17,5 +19,7 @@ def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) d search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) + + :ok end end diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index 251388ea2..da614577f 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Search.MeilisearchTest do require Pleroma.Constants use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory import Tesla.Mock @@ -13,6 +14,7 @@ defmodule Pleroma.Search.MeilisearchTest do alias Pleroma.Search.Meilisearch alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.SearchIndexingWorker setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -27,7 +29,8 @@ defmodule Pleroma.Search.MeilisearchTest do {Meilisearch, [:passthrough], [ add_to_index: fn a -> passthrough([a]) end, - remove_from_index: fn a -> passthrough([a]) end + remove_from_index: fn a -> passthrough([a]) end, + meili_put: fn u, a -> passthrough([u, a]) end ]} ], context, @@ -38,7 +41,7 @@ test "indexes a local post on creation" do user = insert(:user) mock_global(fn - %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> + %{method: :put, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> assert match?( [%{"content" => "guys i just don't wanna leave the swamp"}], Jason.decode!(body) @@ -53,6 +56,15 @@ test "indexes a local post on creation" do visibility: "public" }) + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued( + worker: SearchIndexingWorker, + args: args + ) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_called(Meilisearch.add_to_index(activity)) end @@ -60,26 +72,25 @@ test "doesn't index posts that are not public" do user = insert(:user) Enum.each(["unlisted", "private", "direct"], fn visiblity -> - {:ok, _} = + {:ok, activity} = CommonAPI.post(user, %{ status: "guys i just don't wanna leave the swamp", visibility: visiblity }) + + Meilisearch.add_to_index(activity) + assert_not_called(Meilisearch.meili_put(:_)) end) history = call_history(Meilisearch) assert Enum.count(history) == 3 - - Enum.each(history, fn {_, _, return} -> - assert is_nil(return) - end) end test "deletes posts from index when deleted locally" do user = insert(:user) mock_global(fn - %{method: :post, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> + %{method: :put, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> assert match?( [%{"content" => "guys i just don't wanna leave the swamp"}], Jason.decode!(body) @@ -98,10 +109,16 @@ test "deletes posts from index when deleted locally" do visibility: "public" }) - assert_called(Meilisearch.add_to_index(activity)) + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) {:ok, _} = CommonAPI.delete(activity.id, user) + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + assert_called(Meilisearch.remove_from_index(:_)) end end From 7f53aa400b45ee3d3621049d91b0d083cfba7287 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:52:06 +0300 Subject: [PATCH 84/94] Don't try removing deleted users and such from index as posts --- lib/pleroma/search.ex | 8 ++++---- lib/pleroma/web/activity_pub/side_effects.ex | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index af858fc46..3b266e59b 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,12 +1,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker - def add_to_index(activity) do - SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) + def add_to_index(%Pleroma.Activity{id: activity_id}) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) end - def remove_from_index(object) do - SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) + def remove_from_index(%Pleroma.Object{id: object_id}) do + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) end def search(query, options) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 097afa30e..517dd0a4f 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -325,7 +325,10 @@ def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + # Only remove from index when deleting actual objects, not users or anything else + with %Pleroma.Object{} <- deleted_object do + Pleroma.Search.remove_from_index(deleted_object) + end {:ok, object, meta} else From 774b4e165ac670d8c5f266805786f6ec3cd96090 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 17:17:43 +0300 Subject: [PATCH 85/94] Change search_indexing = 10 and retries for indexing = 2 --- config/config.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 05beb9ba0..cf5f9cf27 100644 --- a/config/config.exs +++ b/config/config.exs @@ -569,7 +569,7 @@ attachments_cleanup: 1, new_users_digest: 1, mute_expire: 5, - search_indexing: 1 + search_indexing: 10 ], plugins: [Oban.Plugins.Pruner], crontab: [ @@ -580,7 +580,8 @@ config :pleroma, :workers, retries: [ federator_incoming: 5, - federator_outgoing: 5 + federator_outgoing: 5, + search_indexing: 2 ] config :pleroma, Pleroma.Formatter, From 09ea3bb694c3b3dc24f650a084dacc253519914f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 21:09:53 +0300 Subject: [PATCH 86/94] Actually, unlisted posts are indexed --- test/pleroma/search/meilisearch_test.exs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index da614577f..04a2d75d9 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -71,19 +71,23 @@ test "indexes a local post on creation" do test "doesn't index posts that are not public" do user = insert(:user) - Enum.each(["unlisted", "private", "direct"], fn visiblity -> + Enum.each(["private", "direct"], fn visibility -> {:ok, activity} = CommonAPI.post(user, %{ status: "guys i just don't wanna leave the swamp", - visibility: visiblity + visibility: visibility }) - Meilisearch.add_to_index(activity) + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) + assert_not_called(Meilisearch.meili_put(:_)) end) history = call_history(Meilisearch) - assert Enum.count(history) == 3 + assert Enum.count(history) == 2 end test "deletes posts from index when deleted locally" do From b7462040cceec9f735757a23a449608bf17a3e1d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:29:17 +0300 Subject: [PATCH 87/94] Change the meilisearch key auth to conform to 0.25.0 --- docs/configuration/search.md | 6 +++--- lib/mix/tasks/pleroma/search/meilisearch.ex | 14 ++++++++------ lib/pleroma/search/meilisearch.ex | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index a785a18ad..82217e5ee 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -47,15 +47,15 @@ you have to get the _private key_, which is actually used for authentication. === "OTP" ```sh - ./bin/pleroma_ctl search.meilisearch show-private-key + ./bin/pleroma_ctl search.meilisearch show-keys ``` === "From Source" ```sh - mix pleroma.search.meilisearch show-private-key + mix pleroma.search.meilisearch show-keys ``` -This is the key you actually put into your configuration file. +You will see a "Default Admin API Key", this is the key you actually put into your configuration file. ### Initial indexing diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5098668ad..db56876fa 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -22,7 +22,7 @@ def run(["index"]) do ) # The ranking rule syntax was changed but nothing about that is mentioned in the changelog - if not Version.match?(meili_version, ">= 0.24.0") do + if not Version.match?(meili_version, ">= 0.25.0") do raise "Meilisearch <0.24.0 not supported" end @@ -112,7 +112,7 @@ def run(["clear"]) do meili_delete!("/indexes/objects/documents") end - def run(["show-private-key", master_key]) do + def run(["show-keys", master_key]) do start_pleroma() endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -120,15 +120,17 @@ def run(["show-private-key", master_key]) do {:ok, result} = Pleroma.HTTP.get( Path.join(endpoint, "/keys"), - [{"X-Meili-API-Key", master_key}] + [{"Authorization", "Bearer #{master_key}"}] ) decoded = Jason.decode!(result.body) - if decoded["private"] do - IO.puts(decoded["private"]) + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) else - IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 33bbf8392..0f9182ffc 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,7 +13,7 @@ defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] end def meili_get(path) do From 563b964690437c5a630842caae6e272040b25d74 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:45:49 +0300 Subject: [PATCH 88/94] Change updateId to uid because apparently that's the new name --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index db56876fa..d4a83c3cd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -91,7 +91,7 @@ def run(["index"]) do ) with {:ok, res} <- result do - if not Map.has_key?(res, "updateId") do + if not Map.has_key?(res, "uid") do IO.puts("\nFailed to index: #{inspect(result)}") end else diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 0f9182ffc..3db65f261 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -153,7 +153,7 @@ def add_to_index(activity) do ) with {:ok, res} <- result, - true <- Map.has_key?(res, "updateId") do + true <- Map.has_key?(res, "uid") do # Do nothing else _ -> From 69d5d1a01bd43593dc0e194929c068740945843e Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 23 Mar 2022 11:36:01 +0300 Subject: [PATCH 89/94] Update meilisearch docs --- docs/configuration/search.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 82217e5ee..f131948a7 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -37,8 +37,10 @@ indexes faster when it can process many posts in a single batch. Information about setting up meilisearch can be found in the [official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html). -You probably want to start it with `MEILI_NO_ANALYTICS=true` and `MEILI_NO_CENTRY=true` environment variables, -to disable analytics. +You probably want to start it with `MEILI_NO_ANALYTICS=true` environment variable to disable analytics. +At least version 0.25.0 is required, but you are strongly adviced to use at least 0.26.0, as it introduces +the `--enable-auto-batching` option which drastically improves performance. Without this option, the search +is hardly usable on a somewhat big instance. ### Private key authentication (optional) From 1ecdb19de5fa54119c6a744c766bbd6c77d1b746 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 30 Jun 2022 16:28:31 +0100 Subject: [PATCH 90/94] Refactor ES on top of search behaviour --- config/config.exs | 18 +- lib/mix/tasks/pleroma/search.ex | 64 ----- lib/mix/tasks/pleroma/search/meilisearch.ex | 144 ---------- lib/pleroma/application.ex | 11 + .../document_mappings/activity.ex | 19 -- .../document_mappings/hashtag.ex | 21 -- .../elasticsearch/document_mappings/user.ex | 17 -- lib/pleroma/elasticsearch/store.ex | 256 ------------------ lib/pleroma/hashtag.ex | 1 - lib/pleroma/search/builtin.ex | 138 ---------- lib/pleroma/search/elasticsearch.ex | 112 ++------ lib/pleroma/search/elasticsearch/cluster.ex | 4 + .../document_mappings/activity.ex | 55 ++++ .../search/elasticsearch/hashtag_parser.ex | 34 --- lib/pleroma/search/elasticsearch/store.ex | 52 ++++ .../search/elasticsearch/user_paser.ex | 57 ---- lib/pleroma/user.ex | 1 - lib/pleroma/web/activity_pub/pipeline.ex | 1 - lib/pleroma/web/activity_pub/side_effects.ex | 21 +- .../web/activity_pub/side_effects/handling.ex | 2 +- lib/pleroma/web/common_api.ex | 8 +- .../controllers/search_controller.ex | 37 ++- mix.exs | 1 + priv/es-mappings/activity.json | 34 +-- .../web/activity_pub/pipeline_test.exs | 1 - 25 files changed, 212 insertions(+), 897 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/search.ex delete mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex delete mode 100644 lib/pleroma/elasticsearch/document_mappings/activity.ex delete mode 100644 lib/pleroma/elasticsearch/document_mappings/hashtag.ex delete mode 100644 lib/pleroma/elasticsearch/document_mappings/user.ex delete mode 100644 lib/pleroma/elasticsearch/store.ex delete mode 100644 lib/pleroma/search/builtin.ex create mode 100644 lib/pleroma/search/elasticsearch/cluster.ex create mode 100644 lib/pleroma/search/elasticsearch/document_mappings/activity.ex delete mode 100644 lib/pleroma/search/elasticsearch/hashtag_parser.ex create mode 100644 lib/pleroma/search/elasticsearch/store.ex delete mode 100644 lib/pleroma/search/elasticsearch/user_paser.ex diff --git a/config/config.exs b/config/config.exs index cf5f9cf27..727a2b0cb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -856,8 +856,6 @@ {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] -config :pleroma, :search, provider: Pleroma.Search.Builtin - config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch config :pleroma, Pleroma.Search.Meilisearch, @@ -865,6 +863,22 @@ private_key: nil, initial_indexing_chunk_size: 100_000 +config :pleroma, Pleroma.Search.Elasticsearch.Cluster, + url: "http://localhost:9200", + username: "elastic", + password: "changeme", + api: Elasticsearch.API.HTTP, + json_library: Jason, + indexes: %{ + activities: %{ + settings: "priv/es-mappings/activity.json", + store: Pleroma.Search.Elasticsearch.Store, + sources: [Pleroma.Activity], + bulk_page_size: 5000, + bulk_wait_interval: 15_000 + } + } + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex deleted file mode 100644 index 1fd880eab..000000000 --- a/lib/mix/tasks/pleroma/search.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Search do - use Mix.Task - import Mix.Pleroma - import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Hashtag - - @shortdoc "Manages elasticsearch" - - def run(["import", "activities" | _rest]) do - start_pleroma() - - from(a in Activity, where: not ilike(a.actor, "%/relay")) - |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> Activity.with_preloaded_object() - |> Activity.with_preloaded_user_actor() - |> get_all(:activities) - end - - def run(["import", "users" | _rest]) do - start_pleroma() - - from(u in User, where: u.nickname not in ["internal.fetch", "relay"]) - |> get_all(:users) - end - - def run(["import", "hashtags" | _rest]) do - start_pleroma() - - from(h in Hashtag) - |> Pleroma.Repo.all() - |> Pleroma.Elasticsearch.bulk_post(:hashtags) - end - - defp get_all(query, index, max_id \\ nil) do - params = %{limit: 1000} - - params = - if max_id == nil do - params - else - Map.put(params, :max_id, max_id) - end - - res = - query - |> Pagination.fetch_paginated(params) - - if res == [] do - :ok - else - res - |> Pleroma.Elasticsearch.bulk_post(index) - - get_all(query, index, List.last(res).id) - end - end -end diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex deleted file mode 100644 index d4a83c3cd..000000000 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ /dev/null @@ -1,144 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - require Pleroma.Constants - - import Mix.Pleroma - import Ecto.Query - - import Pleroma.Search.Meilisearch, - only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] - - def run(["index"]) do - start_pleroma() - - meili_version = - ( - {:ok, result} = meili_get("/version") - - result["pkgVersion"] - ) - - # The ranking rule syntax was changed but nothing about that is mentioned in the changelog - if not Version.match?(meili_version, ">= 0.25.0") do - raise "Meilisearch <0.24.0 not supported" - end - - {:ok, _} = - meili_post( - "/indexes/objects/settings/ranking-rules", - [ - "published:desc", - "words", - "exactness", - "proximity", - "typo", - "attribute", - "sort" - ] - ) - - {:ok, _} = - meili_post( - "/indexes/objects/settings/searchable-attributes", - [ - "content" - ] - ) - - IO.puts("Created indices. Starting to insert posts.") - - chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) - - Pleroma.Repo.transaction( - fn -> - query = - from(Pleroma.Object, - # Only index public and unlisted posts which are notes and have some text - where: - fragment("data->>'type' = 'Note'") and - (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or - fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), - order_by: [desc: fragment("data->'published'")] - ) - - count = query |> Pleroma.Repo.aggregate(:count, :data) - IO.puts("Entries to index: #{count}") - - Pleroma.Repo.stream( - query, - timeout: :infinity - ) - |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) - |> Stream.filter(fn o -> not is_nil(o) end) - |> Stream.chunk_every(chunk_size) - |> Stream.transform(0, fn objects, acc -> - new_acc = acc + Enum.count(objects) - - # Reset to the beginning of the line and rewrite it - IO.write("\r") - IO.write("Indexed #{new_acc} entries") - - {[objects], new_acc} - end) - |> Stream.each(fn objects -> - result = - meili_put( - "/indexes/objects/documents", - objects - ) - - with {:ok, res} <- result do - if not Map.has_key?(res, "uid") do - IO.puts("\nFailed to index: #{inspect(result)}") - end - else - e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") - end - end) - |> Stream.run() - end, - timeout: :infinity - ) - - IO.write("\n") - end - - def run(["clear"]) do - start_pleroma() - - meili_delete!("/indexes/objects/documents") - end - - def run(["show-keys", master_key]) do - start_pleroma() - - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, result} = - Pleroma.HTTP.get( - Path.join(endpoint, "/keys"), - [{"Authorization", "Bearer #{master_key}"}] - ) - - decoded = Jason.decode!(result.body) - - if decoded["results"] do - Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> - IO.puts("#{desc}: #{key}") - end) - else - IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") - end - end - - def run(["stats"]) do - start_pleroma() - - {:ok, result} = meili_get("/indexes/objects/stats") - IO.puts("Number of entries: #{result["numberOfDocuments"]}") - IO.puts("Indexing? #{result["isIndexing"]}") - end -end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index be03cdffb..b709e737b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -105,6 +105,7 @@ def start(_type, _args) do {Oban, Config.get(Oban)}, Pleroma.Web.Endpoint ] ++ + elasticsearch_children() ++ task_children(@mix_env) ++ dont_run_in_test(@mix_env) ++ shout_child(shout_enabled?()) @@ -303,6 +304,16 @@ defp http_children(Tesla.Adapter.Gun, _) do defp http_children(_, _), do: [] + def elasticsearch_children do + config = Config.get([Pleroma.Search, :module]) + + if config == Pleroma.Search.Elasticsearch do + [Pleroma.Search.Elasticsearch.Cluster] + else + [] + end + end + @spec limiters_setup() :: :ok def limiters_setup do config = Config.get(ConcurrentLimiter, []) diff --git a/lib/pleroma/elasticsearch/document_mappings/activity.ex b/lib/pleroma/elasticsearch/document_mappings/activity.ex deleted file mode 100644 index a028c6fad..000000000 --- a/lib/pleroma/elasticsearch/document_mappings/activity.ex +++ /dev/null @@ -1,19 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Elasticsearch.DocumentMappings.Activity do - alias Pleroma.Object - - def id(obj), do: obj.id - - def encode(%{object: %{data: %{"type" => "Note"}}} = activity) do - %{ - _timestamp: activity.inserted_at, - user: activity.user_actor.nickname, - content: activity.object.data["content"], - instance: URI.parse(activity.user_actor.ap_id).host, - hashtags: Object.hashtags(activity.object) - } - end -end diff --git a/lib/pleroma/elasticsearch/document_mappings/hashtag.ex b/lib/pleroma/elasticsearch/document_mappings/hashtag.ex deleted file mode 100644 index 7391983f6..000000000 --- a/lib/pleroma/elasticsearch/document_mappings/hashtag.ex +++ /dev/null @@ -1,21 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Elasticsearch.DocumentMappings.Hashtag do - def id(obj), do: obj.id - - def encode(%{timestamp: _} = hashtag) do - %{ - hashtag: hashtag.name, - timestamp: hashtag.timestamp - } - end - - def encode(hashtag) do - %{ - hashtag: hashtag.name, - timestamp: hashtag.inserted_at - } - end -end diff --git a/lib/pleroma/elasticsearch/document_mappings/user.ex b/lib/pleroma/elasticsearch/document_mappings/user.ex deleted file mode 100644 index d5cfca656..000000000 --- a/lib/pleroma/elasticsearch/document_mappings/user.ex +++ /dev/null @@ -1,17 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Elasticsearch.DocumentMappings.User do - def id(obj), do: obj.id - - def encode(%{actor_type: "Person"} = user) do - %{ - timestamp: user.inserted_at, - instance: URI.parse(user.ap_id).host, - nickname: user.nickname, - bio: user.bio, - display_name: user.name - } - end -end diff --git a/lib/pleroma/elasticsearch/store.ex b/lib/pleroma/elasticsearch/store.ex deleted file mode 100644 index 98c88a7c7..000000000 --- a/lib/pleroma/elasticsearch/store.ex +++ /dev/null @@ -1,256 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Elasticsearch do - alias Pleroma.Activity - alias Pleroma.User - alias Pleroma.Object - alias Pleroma.Elasticsearch.DocumentMappings - alias Pleroma.Config - require Logger - - defp url do - Config.get([:elasticsearch, :url]) - end - - defp enabled? do - Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch - end - - def delete_by_id(:activity, id) do - if enabled?() do - Elastix.Document.delete(url(), "activities", "activity", id) - end - end - - def put_by_id(:activity, id) do - id - |> Activity.get_by_id_with_object() - |> maybe_put_into_elasticsearch() - end - - def maybe_put_into_elasticsearch({:ok, item}) do - maybe_put_into_elasticsearch(item) - end - - def maybe_put_into_elasticsearch( - %{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity - ) do - if enabled?() do - actor = Pleroma.Activity.user_actor(activity) - - activity - |> Map.put(:user_actor, actor) - |> put() - end - end - - def maybe_put_into_elasticsearch(%User{actor_type: "Person"} = user) do - if enabled?() do - put(user) - end - end - - def maybe_put_into_elasticsearch(_) do - {:ok, :skipped} - end - - def maybe_bulk_post(data, type) do - if enabled?() do - bulk_post(data, type) - end - end - - def put(%Activity{} = activity) do - with {:ok, _} <- - Elastix.Document.index( - url(), - "activities", - "activity", - DocumentMappings.Activity.id(activity), - DocumentMappings.Activity.encode(activity) - ) do - activity - |> Map.get(:object) - |> Object.hashtags() - |> Enum.map(fn x -> - %{id: x, name: x, timestamp: DateTime.to_iso8601(DateTime.utc_now())} - end) - |> bulk_post(:hashtags) - else - {:error, %{reason: err}} -> - Logger.error("Could not put activity: #{err}") - :skipped - end - end - - def put(%User{} = user) do - with {:ok, _} <- - Elastix.Document.index( - url(), - "users", - "user", - DocumentMappings.User.id(user), - DocumentMappings.User.encode(user) - ) do - :ok - else - {:error, %{reason: err}} -> - Logger.error("Could not put user: #{err}") - :skipped - end - end - - def bulk_post(data, :activities) do - d = - data - |> Enum.filter(fn x -> - t = - x.object - |> Map.get(:data, %{}) - |> Map.get("type", "") - - t == "Note" - end) - |> Enum.map(fn d -> - [ - %{index: %{_id: DocumentMappings.Activity.id(d)}}, - DocumentMappings.Activity.encode(d) - ] - end) - |> List.flatten() - - with {:ok, %{body: %{"errors" => false}}} <- - Elastix.Bulk.post( - url(), - d, - index: "activities", - type: "activity" - ) do - :ok - else - {:error, %{reason: err}} -> - Logger.error("Could not bulk put activity: #{err}") - :skipped - - {:ok, %{body: _}} -> - :skipped - end - end - - def bulk_post(data, :users) do - d = - data - |> Enum.filter(fn x -> x.actor_type == "Person" end) - |> Enum.map(fn d -> - [ - %{index: %{_id: DocumentMappings.User.id(d)}}, - DocumentMappings.User.encode(d) - ] - end) - |> List.flatten() - - with {:ok, %{body: %{"errors" => false}}} <- - Elastix.Bulk.post( - url(), - d, - index: "users", - type: "user" - ) do - :ok - else - {:error, %{reason: err}} -> - Logger.error("Could not bulk put users: #{err}") - :skipped - - {:ok, %{body: _}} -> - :skipped - end - end - - def bulk_post(data, :hashtags) when is_list(data) do - d = - data - |> Enum.map(fn d -> - [ - %{index: %{_id: DocumentMappings.Hashtag.id(d)}}, - DocumentMappings.Hashtag.encode(d) - ] - end) - |> List.flatten() - - with {:ok, %{body: %{"errors" => false}}} <- - Elastix.Bulk.post( - url(), - d, - index: "hashtags", - type: "hashtag" - ) do - :ok - else - {:error, %{reason: err}} -> - Logger.error("Could not bulk put hashtags: #{err}") - :skipped - - {:ok, %{body: _}} -> - :skipped - end - end - - def bulk_post(_, :hashtags), do: {:ok, nil} - - def search(_, _, _, :skip), do: [] - - def search(:raw, index, type, q) do - with {:ok, raw_results} <- Elastix.Search.search(url(), index, [type], q) do - results = - raw_results - |> Map.get(:body, %{}) - |> Map.get("hits", %{}) - |> Map.get("hits", []) - - {:ok, results} - else - {:error, e} -> - Logger.error(e) - {:error, e} - end - end - - def search(:activities, q) do - with {:ok, results} <- search(:raw, "activities", "activity", q) do - results - |> Enum.map(fn result -> result["_id"] end) - |> Pleroma.Activity.all_by_ids_with_object() - |> Enum.sort(&(&1.inserted_at >= &2.inserted_at)) - else - e -> - Logger.error(e) - [] - end - end - - def search(:users, q) do - with {:ok, results} <- search(:raw, "users", "user", q) do - results - |> Enum.map(fn result -> result["_id"] end) - |> Pleroma.User.get_all_by_ids() - else - e -> - Logger.error(e) - [] - end - end - - def search(:hashtags, q) do - with {:ok, results} <- search(:raw, "hashtags", "hashtag", q) do - results - |> Enum.map(fn result -> result["_source"]["hashtag"] end) - else - e -> - Logger.error(e) - [] - end - end -end diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex index cdbfeab02..53e2e9c89 100644 --- a/lib/pleroma/hashtag.ex +++ b/lib/pleroma/hashtag.ex @@ -61,7 +61,6 @@ def get_or_create_by_names(names) when is_list(names) do {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} end) |> Repo.transaction() do - Pleroma.Elasticsearch.maybe_bulk_post(hashtags, :hashtags) {:ok, hashtags} else {:error, _name, value, _changes_so_far} -> {:error, value} diff --git a/lib/pleroma/search/builtin.ex b/lib/pleroma/search/builtin.ex deleted file mode 100644 index 3cbe2207a..000000000 --- a/lib/pleroma/search/builtin.ex +++ /dev/null @@ -1,138 +0,0 @@ -defmodule Pleroma.Search.Builtin do - @behaviour Pleroma.Search - - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Activity - alias Pleroma.Web.MastodonAPI.AccountView - alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.Endpoint - - require Logger - - @impl Pleroma.Search - def search(_conn, %{q: query} = params, options) do - version = Keyword.get(options, :version) - timeout = Keyword.get(Repo.config(), :timeout, 15_000) - query = String.trim(query) - default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []} - - default_values - |> Enum.map(fn {resource, default_value} -> - if params[:type] in [nil, resource] do - {resource, fn -> resource_search(version, resource, query, options) end} - else - {resource, fn -> default_value end} - end - end) - |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end, - timeout: timeout, - on_timeout: :kill_task - ) - |> Enum.reduce(default_values, fn - {:ok, {resource, result}}, acc -> - Map.put(acc, resource, result) - - _error, acc -> - acc - end) - end - - defp resource_search(_, "accounts", query, options) do - accounts = with_fallback(fn -> User.search(query, options) end) - - AccountView.render("index.json", - users: accounts, - for: options[:for_user], - embed_relationships: options[:embed_relationships] - ) - end - - defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) - - StatusView.render("index.json", - activities: statuses, - for: options[:for_user], - as: :activity - ) - end - - defp resource_search(:v2, "hashtags", query, options) do - tags_path = Endpoint.url() <> "/tag/" - - query - |> prepare_tags(options) - |> Enum.map(fn tag -> - %{name: tag, url: tags_path <> tag} - end) - end - - defp resource_search(:v1, "hashtags", query, options) do - prepare_tags(query, options) - end - - defp prepare_tags(query, options) do - tags = - query - |> preprocess_uri_query() - |> String.split(~r/[^#\w]+/u, trim: true) - |> Enum.uniq_by(&String.downcase/1) - - explicit_tags = Enum.filter(tags, fn tag -> String.starts_with?(tag, "#") end) - - tags = - if Enum.any?(explicit_tags) do - explicit_tags - else - tags - end - - tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) - - tags = - if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do - add_joined_tag(tags) - else - tags - end - - Pleroma.Pagination.paginate(tags, options) - end - - # If `query` is a URI, returns last component of its path, otherwise returns `query` - defp preprocess_uri_query(query) do - if query =~ ~r/https?:\/\// do - query - |> String.trim_trailing("/") - |> URI.parse() - |> Map.get(:path) - |> String.split("/") - |> Enum.at(-1) - else - query - end - end - - defp add_joined_tag(tags) do - tags - |> Kernel.++([joined_tag(tags)]) - |> Enum.uniq_by(&String.downcase/1) - end - - defp joined_tag(tags) do - tags - |> Enum.map(fn tag -> String.capitalize(tag) end) - |> Enum.join() - end - - defp with_fallback(f, fallback \\ []) do - try do - f.() - rescue - error -> - Logger.error("#{__MODULE__} search error: #{inspect(error)}") - fallback - end - end -end diff --git a/lib/pleroma/search/elasticsearch.ex b/lib/pleroma/search/elasticsearch.ex index 76d2c3277..7c7ca82c8 100644 --- a/lib/pleroma/search/elasticsearch.ex +++ b/lib/pleroma/search/elasticsearch.ex @@ -3,24 +3,22 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Search.Elasticsearch do - @behaviour Pleroma.Search + @behaviour Pleroma.Search.SearchBackend alias Pleroma.Activity alias Pleroma.Object.Fetcher - alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Search.Elasticsearch.Parsers - alias Pleroma.Web.Endpoint - def es_query(:activity, query) do + def es_query(:activity, query, offset, limit) do must = Parsers.Activity.parse(query) if must == [] do :skip else %{ - size: 50, + size: limit, + from: offset, terminate_after: 50, timeout: "5s", sort: [ @@ -36,50 +34,6 @@ def es_query(:activity, query) do end end - def es_query(:user, query) do - must = Parsers.User.parse(query) - - if must == [] do - :skip - else - %{ - size: 50, - terminate_after: 50, - timeout: "5s", - sort: [ - "_score" - ], - query: %{ - bool: %{ - must: must - } - } - } - end - end - - def es_query(:hashtag, query) do - must = Parsers.Hashtag.parse(query) - - if must == [] do - :skip - else - %{ - size: 50, - terminate_after: 50, - timeout: "5s", - sort: [ - "_score" - ], - query: %{ - bool: %{ - must: Parsers.Hashtag.parse(query) - } - } - } - end - end - defp maybe_fetch(:activity, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), @@ -90,8 +44,10 @@ defp maybe_fetch(:activity, search_query) do end end - @impl Pleroma.Search - def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) do + def search(user, query, options) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + parsed_query = query |> String.trim() @@ -104,30 +60,13 @@ def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) d activity_task = Task.async(fn -> - q = es_query(:activity, parsed_query) + q = es_query(:activity, parsed_query, offset, limit) - Pleroma.Elasticsearch.search(:activities, q) + Pleroma.Search.Elasticsearch.Store.search(:activities, q) |> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end) end) - user_task = - Task.async(fn -> - q = es_query(:user, parsed_query) - - Pleroma.Elasticsearch.search(:users, q) - |> Enum.filter(fn x -> Pleroma.User.visible_for(x, user) == :visible end) - end) - - hashtag_task = - Task.async(fn -> - q = es_query(:hashtag, parsed_query) - - Pleroma.Elasticsearch.search(:hashtags, q) - end) - activity_results = Task.await(activity_task) - user_results = Task.await(user_task) - hashtag_results = Task.await(hashtag_task) direct_activity = Task.await(activity_fetch_task) activity_results = @@ -137,25 +76,16 @@ def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) d [direct_activity | activity_results] end - %{ - "accounts" => - AccountView.render("index.json", - users: user_results, - for: user - ), - "hashtags" => - Enum.map(hashtag_results, fn x -> - %{ - url: Endpoint.url() <> "/tag/" <> x, - name: x - } - end), - "statuses" => - StatusView.render("index.json", - activities: activity_results, - for: user, - as: :activity - ) - } + activity_results + end + + @impl true + def add_to_index(activity) do + Elasticsearch.put_document(Pleroma.Search.Elasticsearch.Cluster, activity, "activities") + end + + @impl true + def remove_from_index(object) do + Elasticsearch.delete_document(Pleroma.Search.Elasticsearch.Cluster, object, "activities") end end diff --git a/lib/pleroma/search/elasticsearch/cluster.ex b/lib/pleroma/search/elasticsearch/cluster.ex new file mode 100644 index 000000000..4f76c4ebc --- /dev/null +++ b/lib/pleroma/search/elasticsearch/cluster.ex @@ -0,0 +1,4 @@ +defmodule Pleroma.Search.Elasticsearch.Cluster do + @moduledoc false + use Elasticsearch.Cluster, otp_app: :pleroma +end diff --git a/lib/pleroma/search/elasticsearch/document_mappings/activity.ex b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex new file mode 100644 index 000000000..edd8e03c1 --- /dev/null +++ b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex @@ -0,0 +1,55 @@ +# Akkoma: A lightweight social networking server +# Copyright ยฉ 2022-2022 Akkoma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defimpl Elasticsearch.Document, for: Pleroma.Activity do + alias Pleroma.Object + require Pleroma.Constants + + def id(obj), do: obj.id + def routing(_), do: false + + def object_to_search_data(object) do + # Only index public or unlisted Notes + if not is_nil(object) and object.data["type"] == "Note" and + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + String.length(object.data["content"]) > 1 do + data = object.data + + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + if String.length(content) > 1 do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + _timestamp: published, + content: content, + instance: URI.parse(object.data["actor"]).host, + hashtags: Object.hashtags(object), + user: Pleroma.User.get_cached_by_ap_id(object.data["actor"]).nickname + } + else + %{} + end + else + %{} + end + end + + def encode(activity) do + object = Pleroma.Object.normalize(activity) + object_to_search_data(object) + end +end diff --git a/lib/pleroma/search/elasticsearch/hashtag_parser.ex b/lib/pleroma/search/elasticsearch/hashtag_parser.ex deleted file mode 100644 index 911dc651c..000000000 --- a/lib/pleroma/search/elasticsearch/hashtag_parser.ex +++ /dev/null @@ -1,34 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Search.Elasticsearch.Parsers.Hashtag do - defp to_es(term) when is_binary(term) do - %{ - term: %{ - hashtag: %{ - value: String.downcase(term) - } - } - } - end - - defp to_es({:quoted, term}), do: to_es(term) - - defp to_es({:filter, ["hashtag", query]}) do - %{ - term: %{ - hashtag: %{ - value: String.downcase(query) - } - } - } - end - - defp to_es({:filter, _}), do: nil - - def parse(q) do - Enum.map(q, &to_es/1) - |> Enum.filter(fn x -> x != nil end) - end -end diff --git a/lib/pleroma/search/elasticsearch/store.ex b/lib/pleroma/search/elasticsearch/store.ex new file mode 100644 index 000000000..895b76d7f --- /dev/null +++ b/lib/pleroma/search/elasticsearch/store.ex @@ -0,0 +1,52 @@ +# Akkoma: A lightweight social networking server +# Copyright ยฉ 2022-2022 Akkoma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.Elasticsearch.Store do + @behaviour Elasticsearch.Store + alias Pleroma.Search.Elasticsearch.Cluster + require Logger + + alias Pleroma.Repo + + @impl true + def stream(schema) do + Repo.stream(schema) + end + + @impl true + def transaction(fun) do + {:ok, result} = Repo.transaction(fun, timeout: :infinity) + result + end + + def search(_, _, _, :skip), do: [] + + def search(:raw, index, q) do + with {:ok, raw_results} <- Elasticsearch.post(Cluster, "/#{index}/_search", q) do + results = + raw_results + |> Map.get("hits", %{}) + |> Map.get("hits", []) + + {:ok, results} + else + {:error, e} -> + Logger.error(e) + {:error, e} + end + end + + def search(:activities, q) do + with {:ok, results} <- search(:raw, "activities", q) do + results + |> Enum.map(fn result -> result["_id"] end) + |> Pleroma.Activity.all_by_ids_with_object() + |> Enum.sort(&(&1.inserted_at >= &2.inserted_at)) + else + e -> + Logger.error(e) + [] + end + end +end diff --git a/lib/pleroma/search/elasticsearch/user_paser.ex b/lib/pleroma/search/elasticsearch/user_paser.ex deleted file mode 100644 index 4176c6141..000000000 --- a/lib/pleroma/search/elasticsearch/user_paser.ex +++ /dev/null @@ -1,57 +0,0 @@ -# Akkoma: A lightweight social networking server -# Copyright ยฉ 2022-2022 Akkoma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Search.Elasticsearch.Parsers.User do - defp to_es(term) when is_binary(term) do - %{ - bool: %{ - minimum_should_match: 1, - should: [ - %{ - match: %{ - bio: %{ - query: term, - operator: "AND" - } - } - }, - %{ - term: %{ - nickname: %{ - value: term - } - } - }, - %{ - match: %{ - display_name: %{ - query: term, - operator: "AND" - } - } - } - ] - } - } - end - - defp to_es({:quoted, term}), do: to_es(term) - - defp to_es({:filter, ["user", query]}) do - %{ - term: %{ - nickname: %{ - value: query - } - } - } - end - - defp to_es({:filter, _}), do: nil - - def parse(q) do - Enum.map(q, &to_es/1) - |> Enum.filter(fn x -> x != nil end) - end -end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9a50ee3ec..dc6c661ea 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1095,7 +1095,6 @@ def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do was_superuser_before_update = User.superuser?(user) with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do - Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user) set_cache(user) end |> maybe_remove_report_notifications(was_superuser_before_update) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 214647dbf..d4e507287 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -28,7 +28,6 @@ def common_pipeline(object, meta) do case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do {:ok, {:ok, activity, meta}} -> side_effects().handle_after_transaction(meta) - side_effects().handle_after_transaction(activity) {:ok, activity, meta} {:ok, value} -> diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 517dd0a4f..e2371b693 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors +# Copyright ยฉ 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.SideEffects do @@ -272,6 +272,7 @@ def handle(%{data: %{"type" => "Undo", "object" => undone_object}} = object, met def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do reacted_object = Object.get_by_ap_id(object.data["object"]) Utils.add_emoji_reaction_to_object(object, reacted_object) + Notification.create_notifications(object) {:ok, object, meta} @@ -547,24 +548,6 @@ defp add_notifications(meta, notifications) do end @impl true - def handle_after_transaction(%Pleroma.Activity{data: %{"type" => "Create"}} = activity) do - Pleroma.Elasticsearch.put_by_id(:activity, activity.id) - end - - def handle_after_transaction(%Pleroma.Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => id} - }) do - Pleroma.Elasticsearch.delete_by_id(:activity, id) - end - - def handle_after_transaction(%Pleroma.Activity{}) do - :ok - end - - def handle_after_transaction(%Pleroma.Object{}) do - :ok - end - def handle_after_transaction(meta) do meta |> send_notifications() diff --git a/lib/pleroma/web/activity_pub/side_effects/handling.ex b/lib/pleroma/web/activity_pub/side_effects/handling.ex index a82305155..eb012f576 100644 --- a/lib/pleroma/web/activity_pub/side_effects/handling.ex +++ b/lib/pleroma/web/activity_pub/side_effects/handling.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors +# Copyright ยฉ 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.SideEffects.Handling do diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 92afd5cb6..856fa95b9 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -396,13 +396,7 @@ def listen(user, data) do def post(user, %{status: _} = data) do with {:ok, draft} <- ActivityDraft.create(user, data) do - activity = ActivityPub.create(draft.changes, draft.preview?) - - unless draft.preview? do - Pleroma.Elasticsearch.maybe_put_into_elasticsearch(activity) - end - - activity + ActivityPub.create(draft.changes, draft.preview?) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 751d46cdf..e4acba226 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2021 Pleroma Authors +# Copyright ยฉ 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.SearchController do @@ -8,7 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper + alias Pleroma.Web.Endpoint alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.Plugs.RateLimiter @@ -42,13 +44,34 @@ def account_search(%{assigns: %{user: user}} = conn, %{q: query} = params) do def search2(conn, params), do: do_search(:v2, conn, params) def search(conn, params), do: do_search(:v1, conn, params) - defp do_search(version, %{assigns: %{user: user}} = conn, params) do - options = - search_options(params, user) - |> Keyword.put(:version, version) + defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do + query = String.trim(query) + options = search_options(params, user) + timeout = Keyword.get(Repo.config(), :timeout, 15_000) + default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []} - search_provider = Pleroma.Config.get([:search, :provider]) - json(conn, search_provider.search(conn, params, options)) + result = + default_values + |> Enum.map(fn {resource, default_value} -> + if params[:type] in [nil, resource] do + {resource, fn -> resource_search(version, resource, query, options) end} + else + {resource, fn -> default_value end} + end + end) + |> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end, + timeout: timeout, + on_timeout: :kill_task + ) + |> Enum.reduce(default_values, fn + {:ok, {resource, result}}, acc -> + Map.put(acc, resource, result) + + _error, acc -> + acc + end) + + json(conn, result) end defp search_options(params, user) do diff --git a/mix.exs b/mix.exs index 564db2d75..558e71262 100644 --- a/mix.exs +++ b/mix.exs @@ -203,6 +203,7 @@ defp deps do {:nimble_parsec, "~> 1.0", override: true}, {:phoenix_live_dashboard, "~> 0.6.2"}, {:ecto_psql_extras, "~> 0.6"}, + {:elasticsearch, "~> 1.0.0"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/priv/es-mappings/activity.json b/priv/es-mappings/activity.json index e476fd59f..052633496 100644 --- a/priv/es-mappings/activity.json +++ b/priv/es-mappings/activity.json @@ -1,20 +1,22 @@ { - "properties": { - "_timestamp": { - "type": "date", - "index": true - }, - "instance": { - "type": "keyword" - }, - "content": { - "type": "text" - }, - "hashtags": { - "type": "keyword" - }, - "user": { - "type": "text" + "mappings": { + "properties": { + "_timestamp": { + "type": "date", + "index": true + }, + "instance": { + "type": "keyword" + }, + "content": { + "type": "text" + }, + "hashtags": { + "type": "keyword" + }, + "user": { + "type": "text" + } } } } diff --git a/test/pleroma/web/activity_pub/pipeline_test.exs b/test/pleroma/web/activity_pub/pipeline_test.exs index 30fd5651b..e606fa3d1 100644 --- a/test/pleroma/web/activity_pub/pipeline_test.exs +++ b/test/pleroma/web/activity_pub/pipeline_test.exs @@ -28,7 +28,6 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do SideEffectsMock |> expect(:handle, fn o, m -> {:ok, o, m} end) |> expect(:handle_after_transaction, fn m -> m end) - |> expect(:handle_after_transaction, fn m -> m end) :ok end From 635a3c223ab90fd3475ca5bc05db04c0c9cc5365 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 30 Jun 2022 16:53:21 +0100 Subject: [PATCH 91/94] Add elasticsearch tests --- .../document_mappings/activity.ex | 6 + lib/pleroma/telemetry/logger.ex | 63 +-------- ...earch_test.ex => database_search_test.exs} | 0 test/pleroma/search/elasticsearch_test.exs | 120 ++++++++++++++++++ test/support/elasticsearch_mock.ex | 14 ++ 5 files changed, 141 insertions(+), 62 deletions(-) rename test/pleroma/search/{database_search_test.ex => database_search_test.exs} (100%) create mode 100644 test/pleroma/search/elasticsearch_test.exs create mode 100644 test/support/elasticsearch_mock.ex diff --git a/lib/pleroma/search/elasticsearch/document_mappings/activity.ex b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex index edd8e03c1..3a84e991b 100644 --- a/lib/pleroma/search/elasticsearch/document_mappings/activity.ex +++ b/lib/pleroma/search/elasticsearch/document_mappings/activity.ex @@ -53,3 +53,9 @@ def encode(activity) do object_to_search_data(object) end end + +defimpl Elasticsearch.Document, for: Pleroma.Object do + def id(obj), do: obj.id + def routing(_), do: false + def encode(_), do: nil +end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 35e245237..50f7fcf2a 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -12,8 +12,7 @@ defmodule Pleroma.Telemetry.Logger do [:pleroma, :connection_pool, :reclaim, :stop], [:pleroma, :connection_pool, :provision_failure], [:pleroma, :connection_pool, :client, :dead], - [:pleroma, :connection_pool, :client, :add], - [:pleroma, :repo, :query] + [:pleroma, :connection_pool, :client, :add] ] def attach do :telemetry.attach_many( @@ -93,64 +92,4 @@ def handle_event( end def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok - - def handle_event( - [:pleroma, :repo, :query] = _name, - %{query_time: query_time} = measurements, - %{source: source} = metadata, - config - ) do - logging_config = Pleroma.Config.get([:telemetry, :slow_queries_logging], []) - - if logging_config[:enabled] && - logging_config[:min_duration] && - query_time > logging_config[:min_duration] and - (is_nil(logging_config[:exclude_sources]) or - source not in logging_config[:exclude_sources]) do - log_slow_query(measurements, metadata, config) - else - :ok - end - end - - defp log_slow_query( - %{query_time: query_time} = _measurements, - %{source: _source, query: query, params: query_params, repo: repo} = _metadata, - _config - ) do - sql_explain = - with {:ok, %{rows: explain_result_rows}} <- - repo.query("EXPLAIN " <> query, query_params, log: false) do - Enum.map_join(explain_result_rows, "\n", & &1) - end - - {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) - - pleroma_stacktrace = - Enum.filter(stacktrace, fn - {__MODULE__, _, _, _} -> - false - - {mod, _, _, _} -> - mod - |> to_string() - |> String.starts_with?("Elixir.Pleroma.") - end) - - Logger.warn(fn -> - """ - Slow query! - - Total time: #{round(query_time / 1_000)} ms - - #{query} - - #{inspect(query_params, limit: :infinity)} - - #{sql_explain} - - #{Exception.format_stacktrace(pleroma_stacktrace)} - """ - end) - end end diff --git a/test/pleroma/search/database_search_test.ex b/test/pleroma/search/database_search_test.exs similarity index 100% rename from test/pleroma/search/database_search_test.ex rename to test/pleroma/search/database_search_test.exs diff --git a/test/pleroma/search/elasticsearch_test.exs b/test/pleroma/search/elasticsearch_test.exs new file mode 100644 index 000000000..cc5eb6792 --- /dev/null +++ b/test/pleroma/search/elasticsearch_test.exs @@ -0,0 +1,120 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ElasticsearchTest do + require Pleroma.Constants + + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + import Tesla.Mock + import Mock + + alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.SearchIndexingWorker + + describe "elasticsearch" do + setup do + clear_config([Pleroma.Search, :module], Pleroma.Search.Elasticsearch) + clear_config([Pleroma.Search.Elasticsearch.Cluster, :api], Pleroma.ElasticsearchMock) + end + + setup_with_mocks( + [ + {Pleroma.Search.Elasticsearch, [:passthrough], + [ + add_to_index: fn a -> passthrough([a]) end, + remove_from_index: fn a -> passthrough([a]) end + ]}, + {Elasticsearch, [:passthrough], + [ + put_document: fn _, _, _ -> :ok end, + delete_document: fn _, _, _ -> :ok end + ]} + ], + context, + do: {:ok, context} + ) + + test "indexes a local post on creation" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued( + worker: SearchIndexingWorker, + args: args + ) + + assert :ok = perform_job(SearchIndexingWorker, args) + + assert_called(Pleroma.Search.Elasticsearch.add_to_index(activity)) + end + + test "doesn't index posts that are not public" do + user = insert(:user) + + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) + + assert_not_called(Elasticsearch.put_document(:_)) + end) + + history = call_history(Pleroma.Search.Elasticsearch) + assert Enum.count(history) == 2 + end + + test "deletes posts from index when deleted locally" do + user = insert(:user) + + mock_global(fn + %{method: :put, url: "http://127.0.0.1:7700/indexes/objects/documents", body: body} -> + assert match?( + [%{"content" => "guys i just don't wanna leave the swamp"}], + Jason.decode!(body) + ) + + json(%{updateId: 1}) + + %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} -> + assert String.length(id) > 1 + json(%{updateId: 2}) + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + assert_enqueued(worker: SearchIndexingWorker, args: args) + assert :ok = perform_job(SearchIndexingWorker, args) + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + + assert_called(Pleroma.Search.Elasticsearch.remove_from_index(:_)) + end + end +end diff --git a/test/support/elasticsearch_mock.ex b/test/support/elasticsearch_mock.ex new file mode 100644 index 000000000..6e203f2ef --- /dev/null +++ b/test/support/elasticsearch_mock.ex @@ -0,0 +1,14 @@ +defmodule Pleroma.ElasticsearchMock do + @behaviour Elasticsearch.API + + @impl true + def request(_config, :get, "/posts/1", _data, _opts) do + {:ok, + %HTTPoison.Response{ + status_code: 404, + body: %{ + "status" => "not_found" + } + }} + end +end From bc9e76cce78eaaa16d651432d23499a1cd9dfbd9 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 30 Jun 2022 17:36:57 +0100 Subject: [PATCH 92/94] Add documentation for ES search --- config/description.exs | 85 +++++++++++ docs/configuration/search.md | 40 +++++ lib/mix/tasks/pleroma/search/elasticsearch.ex | 9 ++ lib/mix/tasks/pleroma/search/meilisearch.ex | 144 ++++++++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 lib/mix/tasks/pleroma/search/elasticsearch.ex create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex diff --git a/config/description.exs b/config/description.exs index 2d068556f..ac3faa346 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3472,5 +3472,90 @@ suggestion: [100_000] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Search.Elasticsearch.Cluster, + type: :group, + description: "Elasticsearch settings.", + children: [ + %{ + key: :url, + type: :string, + description: "Elasticsearch URL.", + suggestion: ["http://127.0.0.1:9200/"] + }, + %{ + key: :username, + type: :string, + description: "Username to connect to ES. Set to nil if your cluster is unauthenticated.", + suggestion: ["elastic"] + }, + %{ + key: :password, + type: :string, + description: "Password to connect to ES. Set to nil if your cluster is unauthenticated.", + suggestion: ["changeme"] + }, + %{ + key: :api, + type: :module, + description: + "The API module used by Elasticsearch. Should always be Elasticsearch.API.HTTP", + suggestion: [Elasticsearch.API.HTTP] + }, + %{ + key: :json_library, + type: :module, + description: + "The JSON module used to encode/decode when communicating with Elasticsearch", + suggestion: [Jason] + }, + %{ + key: :indexes, + type: :map, + description: "The indices to set up in Elasticsearch", + children: [ + %{ + key: :activities, + type: :map, + description: "Config for the index to use for activities", + children: [ + %{ + key: :settings, + type: :string, + description: + "Path to the file containing index settings for the activities index. Should contain a mapping.", + suggestion: ["priv/es-mappings/activity.json"] + }, + %{ + key: :store, + type: :module, + description: "The internal store module", + suggestion: [Pleroma.Search.Elasticsearch.Store] + }, + %{ + key: :sources, + type: {:list, :module}, + description: "The internal types to use for this index", + suggestion: [[Pleroma.Activity]] + }, + %{ + key: :bulk_page_size, + type: :int, + description: "Size for bulk put requests, mostly used on building the index", + suggestion: [5000] + }, + %{ + key: :bulk_wait_interval, + type: :int, + description: "Time to wait between bulk put requests (in ms)", + suggestion: [15_000] + } + ] + } + ] + } + ] } ] diff --git a/docs/configuration/search.md b/docs/configuration/search.md index f131948a7..7c1093ab9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -121,3 +121,43 @@ This will clear **all** the posts from the search index. Note, that deleted post there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size depends on the amount of text in posts. + +## Elasticsearch + +As with meilisearch, this can be rather memory-hungry, but it is very good at what it does. + +To use [elasticsearch](https://www.elastic.co/), set the search module to `Pleroma.Search.Elasticsearch`: + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.Elasticsearch + +You then need to set the URL and authentication credentials if relevant. + +> config :pleroma, Pleroma.Search.Elasticsearch.Cluster, +> url: "http://127.0.0.1:9200/", +> username: "elastic", +> password: "changeme", + +### Initial indexing + +After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed. You'll only +have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. + +The sequence of actions is as follows: + +1. First, change the configuration to use `Pleroma.Search.Elasticsearch` as the search backend +2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything +3. Start the initial indexing process (as described below with `index`), + and wait until the task says it sent everything from the database to index +4. Wait until the index tasks exits + +To start the initial indexing, run the `build` command: + +=== "OTP" +```sh +./bin/pleroma_ctl search.elasticsearch index activities --cluster Pleroma.Search.Elasticsearch.Cluster +``` + +=== "From Source" +```sh +mix elasticsearch.build activities --cluster Pleroma.Search.Elasticsearch.Cluster +``` \ No newline at end of file diff --git a/lib/mix/tasks/pleroma/search/elasticsearch.ex b/lib/mix/tasks/pleroma/search/elasticsearch.ex new file mode 100644 index 000000000..1d7d7a29a --- /dev/null +++ b/lib/mix/tasks/pleroma/search/elasticsearch.ex @@ -0,0 +1,9 @@ +defmodule Mix.Tasks.Pleroma.Search.Elasticsearch do + alias Mix.Tasks.Elasticsearch.Build + import Mix.Pleroma + + def run(["index" | args]) do + start_pleroma() + Build.run(args) + end +end diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..d4a83c3cd --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,144 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + require Pleroma.Constants + + import Mix.Pleroma + import Ecto.Query + + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] + + def run(["index"]) do + start_pleroma() + + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.25.0") do + raise "Meilisearch <0.24.0 not supported" + end + + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "published:desc", + "words", + "exactness", + "proximity", + "typo", + "attribute", + "sort" + ] + ) + + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) + + IO.puts("Created indices. Starting to insert posts.") + + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) + + Pleroma.Repo.transaction( + fn -> + query = + from(Pleroma.Object, + # Only index public and unlisted posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), + order_by: [desc: fragment("data->'published'")] + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, + timeout: :infinity + ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.each(fn objects -> + result = + meili_put( + "/indexes/objects/documents", + objects + ) + + with {:ok, res} <- result do + if not Map.has_key?(res, "uid") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") + end + end) + |> Stream.run() + end, + timeout: :infinity + ) + + IO.write("\n") + end + + def run(["clear"]) do + start_pleroma() + + meili_delete!("/indexes/objects/documents") + end + + def run(["show-keys", master_key]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"Authorization", "Bearer #{master_key}"}] + ) + + decoded = Jason.decode!(result.body) + + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) + else + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") + end + end + + def run(["stats"]) do + start_pleroma() + + {:ok, result} = meili_get("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end +end From 2937495712d760ca8a9388e69cc2336636830743 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 30 Jun 2022 18:44:31 +0000 Subject: [PATCH 93/94] fix ES import from live (#29) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/29 --- config/config.exs | 2 +- docs/configuration/search.md | 6 +++--- lib/mix/pleroma.ex | 13 +++++++++++- lib/mix/tasks/pleroma/search.ex | 20 +++++++++++++++++++ lib/mix/tasks/pleroma/search/elasticsearch.ex | 9 --------- 5 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search.ex delete mode 100644 lib/mix/tasks/pleroma/search/elasticsearch.ex diff --git a/config/config.exs b/config/config.exs index 727a2b0cb..b00fd18e3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -874,7 +874,7 @@ settings: "priv/es-mappings/activity.json", store: Pleroma.Search.Elasticsearch.Store, sources: [Pleroma.Activity], - bulk_page_size: 5000, + bulk_page_size: 1000, bulk_wait_interval: 15_000 } } diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 7c1093ab9..e1f23b505 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -154,10 +154,10 @@ To start the initial indexing, run the `build` command: === "OTP" ```sh -./bin/pleroma_ctl search.elasticsearch index activities --cluster Pleroma.Search.Elasticsearch.Cluster +./bin/pleroma_ctl search import activities ``` === "From Source" ```sh -mix elasticsearch.build activities --cluster Pleroma.Search.Elasticsearch.Cluster -``` \ No newline at end of file +mix pleroma.search import activities +``` diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 2b6c7d6bb..02c40850a 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -57,7 +57,8 @@ def start_pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + elasticsearch_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -136,4 +137,14 @@ defp http_children(Tesla.Adapter.Gun) do end defp http_children(_), do: [] + + def elasticsearch_children do + config = Pleroma.Config.get([Pleroma.Search, :module]) + + if config == Pleroma.Search.Elasticsearch do + [Pleroma.Search.Elasticsearch.Cluster] + else + [] + end + end end diff --git a/lib/mix/tasks/pleroma/search.ex b/lib/mix/tasks/pleroma/search.ex new file mode 100644 index 000000000..102bc5b63 --- /dev/null +++ b/lib/mix/tasks/pleroma/search.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search do + use Mix.Task + import Mix.Pleroma + + @shortdoc "Manages elasticsearch" + + def run(["import", "activities" | _rest]) do + start_pleroma() + + Elasticsearch.Index.Bulk.upload( + Pleroma.Search.Elasticsearch.Cluster, + "activities", + Pleroma.Config.get([Pleroma.Search.Elasticsearch.Cluster, :indexes, :activities]) + ) + end +end diff --git a/lib/mix/tasks/pleroma/search/elasticsearch.ex b/lib/mix/tasks/pleroma/search/elasticsearch.ex deleted file mode 100644 index 1d7d7a29a..000000000 --- a/lib/mix/tasks/pleroma/search/elasticsearch.ex +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Mix.Tasks.Pleroma.Search.Elasticsearch do - alias Mix.Tasks.Elasticsearch.Build - import Mix.Pleroma - - def run(["index" | args]) do - start_pleroma() - Build.run(args) - end -end From 3f7c64f30baa67105271a72f0d9b3e11ddd37fc8 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Fri, 1 Jul 2022 11:14:55 +0000 Subject: [PATCH 94/94] Fix broken attachments from owncast (#31) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/31 --- .../article_note_page_validator.ex | 1 + .../owncast-note-with-attachment.json | 31 +++++++++++++++++++ .../article_note_page_validator_test.exs | 11 +++++++ 3 files changed, 43 insertions(+) create mode 100644 test/fixtures/owncast-note-with-attachment.json diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 815995895..a9f395c5e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -108,6 +108,7 @@ defp fix(data) do |> fix_replies() |> fix_source() |> fix_misskey_content() + |> Transmogrifier.fix_attachments() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() end diff --git a/test/fixtures/owncast-note-with-attachment.json b/test/fixtures/owncast-note-with-attachment.json new file mode 100644 index 000000000..68cb6bbf7 --- /dev/null +++ b/test/fixtures/owncast-note-with-attachment.json @@ -0,0 +1,31 @@ +{ + "attachment": { + "content": "Live stream preview", + "type": "Image", + "url": "https://owncast.localhost.localdomain/preview.gif?us=KjfNX387gm" + }, + "attributedTo": "https://owncast.localhost.localdomain/federation/user/streamer", + "audience": "https://www.w3.org/ns/activitystreams#Public", + "content": "

I've gone live!

#owncast #streaming

https://owncast.localhost.localdomain", + "id": "https://owncast.localhost.localdomain/federation/KjBNuq8ng", + "published": "2022-04-17T15:42:03Z", + "tag": [ + { + "href": "https://directory.owncast.online/tags/owncast", + "name": "#owncast", + "type": "Hashtag" + }, + { + "href": "https://directory.owncast.online/tags/streaming", + "name": "#streaming", + "type": "Hashtag" + }, + { + "href": "https://directory.owncast.online/tags/owncast", + "name": "#owncast", + "type": "Hashtag" + } + ], + "to": "https://www.w3.org/ns/activitystreams#Public", + "type": "Note" +} diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 2bd1e46c1..717a704d4 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -52,5 +52,16 @@ test "a note with a remote replies collection should validate", _ do %{valid?: true, changes: %{replies: ["https://bookwyrm.com/user/TestUser/status/18"]}} = ArticleNotePageValidator.cast_and_validate(note) end + + test "a note with an attachment should work", _ do + insert(:user, %{ap_id: "https://owncast.localhost.localdomain/federation/user/streamer"}) + + note = + "test/fixtures/owncast-note-with-attachment.json" + |> File.read!() + |> Jason.decode!() + + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) + end end end