From e9f1897cfdb32c890e9eaf2e894128be5c7e1123 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 18 Aug 2022 03:14:48 +0000 Subject: [PATCH] parser MFM server-side (#172) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/172 --- .../article_note_page_validator.ex | 7 ++-- lib/pleroma/web/common_api/utils.ex | 8 ++--- mix.exs | 5 ++- mix.lock | 1 + priv/scrubbers/default.ex | 34 +++++++++++++++++-- .../article_note_page_validator_test.exs | 12 ++++--- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index f2779432e..28053ea3a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -108,6 +108,8 @@ defp remote_mention_resolver( end # https://github.com/misskey-dev/misskey/pull/8787 + # Misskey has an awful tendency to drop all custom formatting when it sends remotely + # So this basically reprocesses their MFM source defp fix_misskey_content( %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object ) @@ -119,7 +121,7 @@ defp fix_misskey_content( {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler) - put_in(object, ["source", "content"], linked) + Map.put(object, "content", linked) end defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do @@ -132,9 +134,10 @@ defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_bina object |> Map.put("source", %{ - "content" => linked, + "content" => content, "mediaType" => "text/x.misskeymarkdown" }) + |> Map.put("content", linked) |> Map.delete("_misskey_content") end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 61af71acd..15016eb47 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -285,11 +285,11 @@ def format_input(text, "text/html", options) do def format_input(text, "text/x.misskeymarkdown", options) do text + |> Formatter.markdown_to_html() + |> MfmParser.Parser.parse() + |> MfmParser.Encoder.to_html() |> Formatter.linkify(options) - |> Formatter.html_escape("text/x.misskeymarkdown") - |> (fn {text, mentions, tags} -> - {String.replace(text, ~r/\r?\n/, "
"), mentions, tags} - end).() + |> Formatter.html_escape("text/html") end def format_input(text, "text/markdown", options) do diff --git a/mix.exs b/mix.exs index c6bd0e28f..e7f491997 100644 --- a/mix.exs +++ b/mix.exs @@ -129,7 +129,7 @@ defp deps do override: true}, {:bcrypt_elixir, "~> 2.2"}, {:trailing_format_plug, "~> 0.0.7"}, - {:fast_sanitize, "~> 0.2.0"}, + {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, {:phoenix_html, "~> 3.1", override: true}, {:calendar, "~> 1.0"}, @@ -191,6 +191,9 @@ defp deps do {:ecto_psql_extras, "~> 0.6"}, {:elasticsearch, git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, + {:mfm_parser, + git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", + ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index 2d3c9f33e..1c3b550e7 100644 --- a/mix.lock +++ b/mix.lock @@ -67,6 +67,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "5054e0ba1ebcbd9a7916aec219528e3e58057241", [ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 153b0be45..68ac06e32 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -56,8 +56,36 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:u, []) Meta.allow_tag_with_these_attributes(:ul, []) - Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "quote-inline"]) - Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tags_with_style_attributes([:span]) + + Meta.allow_tag_with_this_attribute_values(:span, "class", [ + "h-card", + "quote-inline", + "mfm", + "_mfm_tada_", + "_mfm_jelly_", + "_mfm_twitch_", + "_mfm_shake_", + "_mfm_spin_", + "_mfm_jump_", + "_mfm_bounce_", + "_mfm_flip_", + "_mfm_x2_", + "_mfm_x3_", + "_mfm_x4_", + "_mfm_blur_", + "_mfm_rainbow_", + "_mfm_rotate_" + ]) + + Meta.allow_tag_with_these_attributes(:span, [ + "data-x", + "data-y", + "data-h", + "data-v", + "data-left", + "data-right" + ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) @@ -101,4 +129,6 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:small, []) Meta.strip_everything_not_covered() + + defp scrub_css(value), do: value end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index f419770f2..c766414a6 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -96,9 +96,8 @@ test "a misskey MFM status with a content field should work and be linked", _ do %{ valid?: true, changes: %{ - content: "this does not get replaced", + content: content, source: %{ - "content" => content, "mediaType" => "text/x.misskeymarkdown" } } @@ -114,7 +113,9 @@ test "a misskey MFM status with a content field should work and be linked", _ do "@full_tag_remote_user" assert content =~ "@oops_not_a_mention" - assert content =~ "$[jelly mfm goes here]

## aaa" + + assert content =~ + "mfm goes here

aaa" end test "a misskey MFM status with a _misskey_content field should work and be linked", _ do @@ -133,9 +134,10 @@ test "a misskey MFM status with a _misskey_content field should work and be link %{ valid?: true, changes: %{ + content: content, source: %{ - "content" => content, - "mediaType" => "text/x.misskeymarkdown" + "mediaType" => "text/x.misskeymarkdown", + "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa" } } } = changes