parser MFM server-side (#172)

Reviewed-on: AkkomaGang/akkoma#172

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -108,6 +108,8 @@ defp remote_mention_resolver(
   end
 
   # https://github.com/misskey-dev/misskey/pull/8787
+  # Misskey has an awful tendency to drop all custom formatting when it sends remotely
+  # So this basically reprocesses their MFM source
   defp fix_misskey_content(
          %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object
        )
@@ -119,7 +121,7 @@ defp fix_misskey_content(
     {linked, _, _} =
       Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler)
 
-    put_in(object, ["source", "content"], linked)
+    Map.put(object, "content", linked)
   end
 
   defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do
@@ -132,9 +134,10 @@ defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_bina
 
     object
     |> Map.put("source", %{
-      "content" => linked,
+      "content" => content,
       "mediaType" => "text/x.misskeymarkdown"
     })
+    |> Map.put("content", linked)
     |> Map.delete("_misskey_content")
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -285,11 +285,11 @@ def format_input(text, "text/html", options) do
 
   def format_input(text, "text/x.misskeymarkdown", options) do
     text
+    |> Formatter.markdown_to_html()
+    |> MfmParser.Parser.parse()
+    |> MfmParser.Encoder.to_html()
     |> Formatter.linkify(options)
-    |> Formatter.html_escape("text/x.misskeymarkdown")
+    |> Formatter.html_escape("text/html")
-    |> (fn {text, mentions, tags} ->
-          {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
-        end).()
   end
 
   def format_input(text, "text/markdown", options) do

mix.exs

@@ -129,7 +129,7 @@ defp deps do
        override: true},
       {:bcrypt_elixir, "~> 2.2"},
       {:trailing_format_plug, "~> 0.0.7"},
-      {:fast_sanitize, "~> 0.2.0"},
+      {:fast_sanitize, "~> 0.2.3"},
       {:html_entities, "~> 0.5", override: true},
       {:phoenix_html, "~> 3.1", override: true},
       {:calendar, "~> 1.0"},
@@ -191,6 +191,9 @@ defp deps do
       {:ecto_psql_extras, "~> 0.6"},
       {:elasticsearch,
        git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"},
+      {:mfm_parser,
+       git: "https://akkoma.dev/AkkomaGang/mfm-parser.git",
+       ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"},
 
       # indirect dependency version override
       {:plug, "~> 1.10.4", override: true},

mix.lock

@@ -67,6 +67,7 @@
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
   "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "5054e0ba1ebcbd9a7916aec219528e3e58057241", [ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"]},
   "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
   "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"},

priv/scrubbers/default.ex

@@ -56,8 +56,36 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_these_attributes(:u, [])
   Meta.allow_tag_with_these_attributes(:ul, [])
 
-  Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "quote-inline"])
+  Meta.allow_tags_with_style_attributes([:span])
-  Meta.allow_tag_with_these_attributes(:span, [])
+
+  Meta.allow_tag_with_this_attribute_values(:span, "class", [
+    "h-card",
+    "quote-inline",
+    "mfm",
+    "_mfm_tada_",
+    "_mfm_jelly_",
+    "_mfm_twitch_",
+    "_mfm_shake_",
+    "_mfm_spin_",
+    "_mfm_jump_",
+    "_mfm_bounce_",
+    "_mfm_flip_",
+    "_mfm_x2_",
+    "_mfm_x3_",
+    "_mfm_x4_",
+    "_mfm_blur_",
+    "_mfm_rainbow_",
+    "_mfm_rotate_"
+  ])
+
+  Meta.allow_tag_with_these_attributes(:span, [
+    "data-x",
+    "data-y",
+    "data-h",
+    "data-v",
+    "data-left",
+    "data-right"
+  ])
 
   Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"])
 
@@ -101,4 +129,6 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_these_attributes(:small, [])
 
   Meta.strip_everything_not_covered()
+
+  defp scrub_css(value), do: value
 end

test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs

@@ -96,9 +96,8 @@ test "a misskey MFM status with a content field should work and be linked", _ do
       %{
         valid?: true,
         changes: %{
-          content: "this does not get replaced",
+          content: content,
           source: %{
-            "content" => content,
             "mediaType" => "text/x.misskeymarkdown"
           }
         }
@@ -114,7 +113,9 @@ test "a misskey MFM status with a content field should work and be linked", _ do
                "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{full_tag_remote_user.id}\" href=\"#{full_tag_remote_user.ap_id}\" rel=\"ugc\">@<span>full_tag_remote_user</span></a></span>"
 
       assert content =~ "@oops_not_a_mention"
-      assert content =~ "$[jelly mfm goes here] <br><br>## aaa"
+
+      assert content =~
+               "<span class=\"mfm\" style=\"display: inline-block; animation: 1s linear 0s infinite normal both running mfm-rubberBand;\">mfm goes here</span> </p>aaa"
     end
 
     test "a misskey MFM status with a _misskey_content field should work and be linked", _ do
@@ -133,9 +134,10 @@ test "a misskey MFM status with a _misskey_content field should work and be link
       %{
         valid?: true,
         changes: %{
+          content: content,
           source: %{
-            "content" => content,
+            "mediaType" => "text/x.misskeymarkdown",
-            "mediaType" => "text/x.misskeymarkdown"
+            "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa"
           }
         }
       } = changes