From f86bf164300b7311160660970469efad5cfed3df Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 2 Jan 2023 02:15:38 +0000 Subject: [PATCH] Add language support on /api/v1/statuses --- lib/pleroma/iso639.ex | 11 +++++++++++ .../article_note_page_validator.ex | 16 ++++++++++++++++ lib/pleroma/web/common_api/activity_draft.ex | 15 +++++++++++++++ .../web/mastodon_api/views/status_view.ex | 14 ++++++++++++-- priv/language-codes.json | 1 + .../controllers/status_controller_test.exs | 12 +++++++++--- 6 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 lib/pleroma/iso639.ex create mode 100644 priv/language-codes.json diff --git a/lib/pleroma/iso639.ex b/lib/pleroma/iso639.ex new file mode 100644 index 000000000..a80fab7e9 --- /dev/null +++ b/lib/pleroma/iso639.ex @@ -0,0 +1,11 @@ +defmodule Pleroma.ISO639 do + @file "priv/language-codes.json" + @data File.read!(@file) + |> Jason.decode!() + + for %{"alpha2" => alpha2} <- @data do + def valid_alpha2?(unquote(alpha2)), do: true + end + + def valid_alpha2?(_alpha2), do: false +end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 0d45421e2..1bdf2a950 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do field(:replies, {:array, ObjectValidators.ObjectID}, default: []) field(:source, :map) + field(:content_map, :map) end def cast_and_apply(data) do @@ -146,6 +147,20 @@ defp fix_source(%{"source" => source} = object) when is_binary(source) do defp fix_source(object), do: object + defp fix_content_map_languages(%{"contentMap" => content_map} = object) + when is_map(content_map) do + # Only allow valid languages + content_map = + content_map + |> Enum.reject(fn {lang, content} -> + !Pleroma.ISO639.valid_alpha2?(lang) + end) + + Map.put(object, "contentMap", content_map) + end + + defp fix_content_map_languages(object), do: object + defp fix(data) do data |> CommonFixes.fix_actor() @@ -158,6 +173,7 @@ defp fix(data) do |> Transmogrifier.fix_attachments() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() + |> fix_content_map_languages() end def changeset(struct, data) do diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 8b0eaaadf..adddea669 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -22,6 +22,8 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do attachments: [], in_reply_to: nil, in_reply_to_conversation: nil, + language: nil, + content_map: %{}, quote_id: nil, quote: nil, visibility: nil, @@ -58,6 +60,7 @@ def create(user, params) do |> with_valid(&visibility/1) |> with_valid("e_id/1) |> content() + |> with_valid(&language/1) |> with_valid(&to_and_cc/1) |> with_valid(&context/1) |> sensitive() @@ -133,6 +136,17 @@ defp quote_id(%{params: %{quote_id: %Activity{} = quote}} = draft) do defp quote_id(draft), do: draft + defp language(%{params: %{language: language}, content_html: content} = draft) + when is_binary(language) do + if Pleroma.ISO639.valid_alpha2?(language) do + %__MODULE__{draft | content_map: %{language => content}} + else + add_error(draft, dgettext("errors", "Invalid language")) + end + end + + defp language(draft), do: draft + defp visibility(%{params: params} = draft) do case CommonAPI.get_visibility(params, draft.in_reply_to, draft.in_reply_to_conversation) do {visibility, "direct"} when visibility != "direct" -> @@ -224,6 +238,7 @@ defp object(draft) do "mediaType" => Utils.get_content_type(draft.params[:content_type]) }) |> Map.put("generator", draft.params[:generator]) + |> Map.put("contentMap", draft.content_map) %__MODULE__{draft | object: object} end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b9a7e57f5..48756e78b 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -169,6 +169,7 @@ def render( |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) {pinned?, pinned_at} = pin_data(object, user) + lang = language(object) %{ id: to_string(activity.id), @@ -199,7 +200,7 @@ def render( mentions: mentions, tags: reblogged[:tags] || [], application: build_application(object.data["generator"]), - language: nil, + language: lang, emojis: [], pleroma: %{ local: activity.local, @@ -357,6 +358,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} {pinned?, pinned_at} = pin_data(object, user) quote = Activity.get_quoted_activity_from_object(object) + lang = language(object) %{ id: to_string(activity.id), @@ -391,7 +393,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} mentions: mentions, tags: build_tags(tags), application: build_application(object.data["generator"]), - language: nil, + language: lang, emojis: build_emojis(object.data["emoji"]), quote_id: if(quote, do: quote.id, else: nil), quote: maybe_render_quote(quote, opts), @@ -784,4 +786,12 @@ defp get_source_content_type(%{"mediaType" => type} = _source) do defp get_source_content_type(_source) do Utils.get_content_type(nil) end + + defp language(%Object{data: %{"contentMap" => contentMap}}) when is_map(contentMap) do + contentMap + |> Map.keys() + |> Enum.at(0) + end + + defp language(_), do: nil end diff --git a/priv/language-codes.json b/priv/language-codes.json new file mode 100644 index 000000000..dda9a2b75 --- /dev/null +++ b/priv/language-codes.json @@ -0,0 +1 @@ +[{"English": "Afar", "alpha2": "aa"},{"English": "Abkhazian", "alpha2": "ab"},{"English": "Avestan", "alpha2": "ae"},{"English": "Afrikaans", "alpha2": "af"},{"English": "Akan", "alpha2": "ak"},{"English": "Amharic", "alpha2": "am"},{"English": "Aragonese", "alpha2": "an"},{"English": "Arabic", "alpha2": "ar"},{"English": "Assamese", "alpha2": "as"},{"English": "Avaric", "alpha2": "av"},{"English": "Aymara", "alpha2": "ay"},{"English": "Azerbaijani", "alpha2": "az"},{"English": "Bashkir", "alpha2": "ba"},{"English": "Belarusian", "alpha2": "be"},{"English": "Bulgarian", "alpha2": "bg"},{"English": "Bihari languages", "alpha2": "bh"},{"English": "Bislama", "alpha2": "bi"},{"English": "Bambara", "alpha2": "bm"},{"English": "Bengali", "alpha2": "bn"},{"English": "Tibetan", "alpha2": "bo"},{"English": "Breton", "alpha2": "br"},{"English": "Bosnian", "alpha2": "bs"},{"English": "Catalan; Valencian", "alpha2": "ca"},{"English": "Chechen", "alpha2": "ce"},{"English": "Chamorro", "alpha2": "ch"},{"English": "Corsican", "alpha2": "co"},{"English": "Cree", "alpha2": "cr"},{"English": "Czech", "alpha2": "cs"},{"English": "Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic", "alpha2": "cu"},{"English": "Chuvash", "alpha2": "cv"},{"English": "Welsh", "alpha2": "cy"},{"English": "Danish", "alpha2": "da"},{"English": "German", "alpha2": "de"},{"English": "Divehi; Dhivehi; Maldivian", "alpha2": "dv"},{"English": "Dzongkha", "alpha2": "dz"},{"English": "Ewe", "alpha2": "ee"},{"English": "Greek, Modern (1453-)", "alpha2": "el"},{"English": "English", "alpha2": "en"},{"English": "Esperanto", "alpha2": "eo"},{"English": "Spanish; Castilian", "alpha2": "es"},{"English": "Estonian", "alpha2": "et"},{"English": "Basque", "alpha2": "eu"},{"English": "Persian", "alpha2": "fa"},{"English": "Fulah", "alpha2": "ff"},{"English": "Finnish", "alpha2": "fi"},{"English": "Fijian", "alpha2": "fj"},{"English": "Faroese", "alpha2": "fo"},{"English": "French", "alpha2": "fr"},{"English": "Western Frisian", "alpha2": "fy"},{"English": "Irish", "alpha2": "ga"},{"English": "Gaelic; Scottish Gaelic", "alpha2": "gd"},{"English": "Galician", "alpha2": "gl"},{"English": "Guarani", "alpha2": "gn"},{"English": "Gujarati", "alpha2": "gu"},{"English": "Manx", "alpha2": "gv"},{"English": "Hausa", "alpha2": "ha"},{"English": "Hebrew", "alpha2": "he"},{"English": "Hindi", "alpha2": "hi"},{"English": "Hiri Motu", "alpha2": "ho"},{"English": "Croatian", "alpha2": "hr"},{"English": "Haitian; Haitian Creole", "alpha2": "ht"},{"English": "Hungarian", "alpha2": "hu"},{"English": "Armenian", "alpha2": "hy"},{"English": "Herero", "alpha2": "hz"},{"English": "Interlingua (International Auxiliary Language Association)", "alpha2": "ia"},{"English": "Indonesian", "alpha2": "id"},{"English": "Interlingue; Occidental", "alpha2": "ie"},{"English": "Igbo", "alpha2": "ig"},{"English": "Sichuan Yi; Nuosu", "alpha2": "ii"},{"English": "Inupiaq", "alpha2": "ik"},{"English": "Ido", "alpha2": "io"},{"English": "Icelandic", "alpha2": "is"},{"English": "Italian", "alpha2": "it"},{"English": "Inuktitut", "alpha2": "iu"},{"English": "Japanese", "alpha2": "ja"},{"English": "Javanese", "alpha2": "jv"},{"English": "Georgian", "alpha2": "ka"},{"English": "Kongo", "alpha2": "kg"},{"English": "Kikuyu; Gikuyu", "alpha2": "ki"},{"English": "Kuanyama; Kwanyama", "alpha2": "kj"},{"English": "Kazakh", "alpha2": "kk"},{"English": "Kalaallisut; Greenlandic", "alpha2": "kl"},{"English": "Central Khmer", "alpha2": "km"},{"English": "Kannada", "alpha2": "kn"},{"English": "Korean", "alpha2": "ko"},{"English": "Kanuri", "alpha2": "kr"},{"English": "Kashmiri", "alpha2": "ks"},{"English": "Kurdish", "alpha2": "ku"},{"English": "Komi", "alpha2": "kv"},{"English": "Cornish", "alpha2": "kw"},{"English": "Kirghiz; Kyrgyz", "alpha2": "ky"},{"English": "Latin", "alpha2": "la"},{"English": "Luxembourgish; Letzeburgesch", "alpha2": "lb"},{"English": "Ganda", "alpha2": "lg"},{"English": "Limburgan; Limburger; Limburgish", "alpha2": "li"},{"English": "Lingala", "alpha2": "ln"},{"English": "Lao", "alpha2": "lo"},{"English": "Lithuanian", "alpha2": "lt"},{"English": "Luba-Katanga", "alpha2": "lu"},{"English": "Latvian", "alpha2": "lv"},{"English": "Malagasy", "alpha2": "mg"},{"English": "Marshallese", "alpha2": "mh"},{"English": "Maori", "alpha2": "mi"},{"English": "Macedonian", "alpha2": "mk"},{"English": "Malayalam", "alpha2": "ml"},{"English": "Mongolian", "alpha2": "mn"},{"English": "Marathi", "alpha2": "mr"},{"English": "Malay", "alpha2": "ms"},{"English": "Maltese", "alpha2": "mt"},{"English": "Burmese", "alpha2": "my"},{"English": "Nauru", "alpha2": "na"},{"English": "Bokm\u00e5l, Norwegian; Norwegian Bokm\u00e5l", "alpha2": "nb"},{"English": "Ndebele, North; North Ndebele", "alpha2": "nd"},{"English": "Nepali", "alpha2": "ne"},{"English": "Ndonga", "alpha2": "ng"},{"English": "Dutch; Flemish", "alpha2": "nl"},{"English": "Norwegian Nynorsk; Nynorsk, Norwegian", "alpha2": "nn"},{"English": "Norwegian", "alpha2": "no"},{"English": "Ndebele, South; South Ndebele", "alpha2": "nr"},{"English": "Navajo; Navaho", "alpha2": "nv"},{"English": "Chichewa; Chewa; Nyanja", "alpha2": "ny"},{"English": "Occitan (post 1500)", "alpha2": "oc"},{"English": "Ojibwa", "alpha2": "oj"},{"English": "Oromo", "alpha2": "om"},{"English": "Oriya", "alpha2": "or"},{"English": "Ossetian; Ossetic", "alpha2": "os"},{"English": "Panjabi; Punjabi", "alpha2": "pa"},{"English": "Pali", "alpha2": "pi"},{"English": "Polish", "alpha2": "pl"},{"English": "Pushto; Pashto", "alpha2": "ps"},{"English": "Portuguese", "alpha2": "pt"},{"English": "Quechua", "alpha2": "qu"},{"English": "Romansh", "alpha2": "rm"},{"English": "Rundi", "alpha2": "rn"},{"English": "Romanian; Moldavian; Moldovan", "alpha2": "ro"},{"English": "Russian", "alpha2": "ru"},{"English": "Kinyarwanda", "alpha2": "rw"},{"English": "Sanskrit", "alpha2": "sa"},{"English": "Sardinian", "alpha2": "sc"},{"English": "Sindhi", "alpha2": "sd"},{"English": "Northern Sami", "alpha2": "se"},{"English": "Sango", "alpha2": "sg"},{"English": "Sinhala; Sinhalese", "alpha2": "si"},{"English": "Slovak", "alpha2": "sk"},{"English": "Slovenian", "alpha2": "sl"},{"English": "Samoan", "alpha2": "sm"},{"English": "Shona", "alpha2": "sn"},{"English": "Somali", "alpha2": "so"},{"English": "Albanian", "alpha2": "sq"},{"English": "Serbian", "alpha2": "sr"},{"English": "Swati", "alpha2": "ss"},{"English": "Sotho, Southern", "alpha2": "st"},{"English": "Sundanese", "alpha2": "su"},{"English": "Swedish", "alpha2": "sv"},{"English": "Swahili", "alpha2": "sw"},{"English": "Tamil", "alpha2": "ta"},{"English": "Telugu", "alpha2": "te"},{"English": "Tajik", "alpha2": "tg"},{"English": "Thai", "alpha2": "th"},{"English": "Tigrinya", "alpha2": "ti"},{"English": "Turkmen", "alpha2": "tk"},{"English": "Tagalog", "alpha2": "tl"},{"English": "Tswana", "alpha2": "tn"},{"English": "Tonga (Tonga Islands)", "alpha2": "to"},{"English": "Turkish", "alpha2": "tr"},{"English": "Tsonga", "alpha2": "ts"},{"English": "Tatar", "alpha2": "tt"},{"English": "Twi", "alpha2": "tw"},{"English": "Tahitian", "alpha2": "ty"},{"English": "Uighur; Uyghur", "alpha2": "ug"},{"English": "Ukrainian", "alpha2": "uk"},{"English": "Urdu", "alpha2": "ur"},{"English": "Uzbek", "alpha2": "uz"},{"English": "Venda", "alpha2": "ve"},{"English": "Vietnamese", "alpha2": "vi"},{"English": "Volap\u00fck", "alpha2": "vo"},{"English": "Walloon", "alpha2": "wa"},{"English": "Wolof", "alpha2": "wo"},{"English": "Xhosa", "alpha2": "xh"},{"English": "Yiddish", "alpha2": "yi"},{"English": "Yoruba", "alpha2": "yo"},{"English": "Zhuang; Chuang", "alpha2": "za"},{"English": "Chinese", "alpha2": "zh"},{"English": "Zulu", "alpha2": "zu"}] \ No newline at end of file diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 45da22d91..d964340cb 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -67,11 +67,17 @@ test "posting a status", %{conn: conn} do |> post("/api/v1/statuses", %{ "status" => "cofe", "spoiler_text" => "2hu", - "sensitive" => "0" + "sensitive" => "0", + "language" => "ja" }) - assert %{"content" => "cofe", "id" => id, "spoiler_text" => "2hu", "sensitive" => false} = - json_response_and_validate_schema(conn_one, 200) + assert %{ + "content" => "cofe", + "id" => id, + "spoiler_text" => "2hu", + "sensitive" => false, + "language" => "ja" + } = json_response_and_validate_schema(conn_one, 200) assert Activity.get_by_id(id)