[#58] pre-link MFM content (#59)

Reviewed-on: AkkomaGang/akkoma#59

docs/development/setting_up_akkoma_dev.md

@@ -36,6 +36,33 @@ config :logger, :console,
   level: :info
 ```
 
+## Testing with HTTPS
+
+If you end up developing alongside other software like misskey,
+you will not be able to federate without an SSL certificate. You should
+be able to use the snakeoil certificate that comes standard with most
+distributions or generate one from scratch, then force elixir to accept it.
+
+HTTP clients are none too keen to accept self-signed certs, but we can do
+this:
+
+```elixir
+config :pleroma, :http,
+  adapter: [
+    pools: %{
+      default: [
+        conn_opts: [
+          transport_opts: [
+            verify: :verify_none
+          ]
+        ]
+      ]
+    }
+  ]
+```
+
+Now your SSL requests will work. Hooray.
+
 ## Testing
 
 1. Create a `test.secret.exs` file with the content as shown below

lib/pleroma/application.ex

@@ -58,9 +58,6 @@ def start(_type, _args) do
     Pleroma.Docs.JSON.compile()
     limiters_setup()
 
-    Logger.info("Starting Finch")
-    Finch.start_link(name: MyFinch)
-
     # Define workers and child supervisors to be supervised
     children =
       [
@@ -70,6 +67,7 @@ def start(_type, _args) do
         Pleroma.Web.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
+        http_children() ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
@@ -276,4 +274,13 @@ def limiters_setup do
       ConcurrentLimiter.new(module, max_running, max_waiting)
     end)
   end
+
+  defp http_children do
+    config =
+      [:http, :adapter]
+      |> Config.get([])
+      |> Keyword.put(:name, MyFinch)
+
+    [{Finch, config}]
+  end
 end

lib/pleroma/formatter.ex

@@ -133,7 +133,7 @@ def html_escape(text, "text/html") do
     HTML.filter_tags(text)
   end
 
-  def html_escape(text, "text/plain") do
+  def html_escape(text, format) when format in ["text/plain", "text/x.misskeymarkdown"] do
     Regex.split(@link_regex, text, include_captures: true)
     |> Enum.map_every(2, fn chunk ->
       {:safe, part} = Phoenix.HTML.html_escape(chunk)

lib/pleroma/http.ex

@@ -65,7 +65,6 @@ def request(method, url, body, headers, options) when is_binary(url) do
     options = put_in(options[:adapter], adapter_opts)
     params = options[:params] || []
     request = build_request(method, headers, options, url, body, params)
-
     client = Tesla.client([Tesla.Middleware.FollowRedirects])
 
     request(client, request)

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -81,12 +82,22 @@ defp fix_replies(%{"replies" => %{"first" => first}} = data) do
   defp fix_replies(data), do: data
 
   # https://github.com/misskey-dev/misskey/pull/8787
-  defp fix_misskey_content(%{"source" => %{"mediaType" => "text/x.misskeymarkdown"}} = object),
+  defp fix_misskey_content(
-    do: object
+         %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object
+       ) do
+    {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown")
+    Map.put(object, "content", linked)
+  end
 
   defp fix_misskey_content(%{"_misskey_content" => content} = object) do
+    {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown")
+
     object
-    |> Map.put("source", %{"content" => content, "mediaType" => "text/x.misskeymarkdown"})
+    |> Map.put("source", %{
+      "content" => content,
+      "mediaType" => "text/x.misskeymarkdown"
+    })
+    |> Map.put("content", linked)
     |> Map.delete("_misskey_content")
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -259,7 +259,8 @@ def format_input(text, format, options \\ [])
   @doc """
   Formatting text to plain text, BBCode, HTML, or Markdown
   """
-  def format_input(text, "text/plain", options) do
+  def format_input(text, format, options)
+      when format in ["text/plain", "text/x.misskeymarkdown"] do
     text
     |> Formatter.html_escape("text/plain")
     |> Formatter.linkify(options)
@@ -291,15 +292,6 @@ def format_input(text, "text/markdown", options) do
     |> Formatter.html_escape("text/html")
   end
 
-  def format_input(text, "text/x.misskeymarkdown", options) do
-    text
-    |> Formatter.html_escape("text/plain")
-    |> Formatter.linkify(options)
-    |> (fn {text, mentions, tags} ->
-          {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
-        end).()
-  end
-
   def format_naive_asctime(date) do
     date |> DateTime.from_naive!("Etc/UTC") |> format_asctime
   end

test/fixtures/misskey/mfm_underscore_format.json

@@ -0,0 +1,31 @@
+{
+  "id": "https://misskey.local.live/notes/92j1n3owja",
+  "type": "Note",
+  "attributedTo": "https://misskey.local.live/users/92hzkskwgy",
+  "summary": null,
+  "content": "<p><a href=\"https://akkoma.local.live/users/akkoma_user\" class=\"u-url mention\">@akkoma_user@akkoma.local.live</a><span> linkifylink </span><a href=\"https://misskey.local.live/tags/dancedance\" rel=\"tag\">#dancedance</a><span> </span><i><span> mfm goes here</span></i><span> <br><br>## aaa</span></p>",
+  "_misskey_content": "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa",
+  "published": "2022-07-10T15:37:36.368Z",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "https://misskey.local.live/users/92hzkskwgy/followers",
+    "http://localhost:4001/users/akkoma_user"
+  ],
+  "inReplyTo": null,
+  "attachment": [],
+  "sensitive": false,
+  "tag": [
+    {
+      "type": "Hashtag",
+      "href": "https://misskey.local.live/tags/dancedance",
+      "name": "#dancedance"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:4001/users/akkoma_user",
+      "name": "@akkoma_user"
+    }
+  ]
+}

test/fixtures/misskey/mfm_x_format.json

@@ -0,0 +1,34 @@
+{
+  "id": "https://misskey.local.live/notes/92j1n3owja",
+  "type": "Note",
+  "attributedTo": "https://misskey.local.live/users/92hzkskwgy",
+  "summary": null,
+  "content": "<p><a href=\"https://akkoma.local.live/users/akkoma_user\" class=\"u-url mention\">@akkoma_user@akkoma.local.live</a><span> linkifylink </span><a href=\"https://misskey.local.live/tags/dancedance\" rel=\"tag\">#dancedance</a><span> </span><i><span> mfm goes here</span></i><span> <br><br>## aaa</span></p>",
+  "source": {
+    "content": "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa",
+    "mediaType": "text/x.misskeymarkdown"
+  },
+  "published": "2022-07-10T15:37:36.368Z",
+  "to": [
+    "https://www.w3.org/ns/activitystreams#Public"
+  ],
+  "cc": [
+    "https://misskey.local.live/users/92hzkskwgy/followers",
+    "http://localhost:4001/users/akkoma_user"
+  ],
+  "inReplyTo": null,
+  "attachment": [],
+  "sensitive": false,
+  "tag": [
+    {
+      "type": "Hashtag",
+      "href": "https://misskey.local.live/tags/dancedance",
+      "name": "#dancedance"
+    },
+    {
+      "type": "Mention",
+      "href": "http://localhost:4001/users/akkoma_user",
+      "name": "@akkoma_user"
+    }
+  ]
+}

test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs

@@ -10,6 +10,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
 
   import Pleroma.Factory
 
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    :ok
+  end
+
   describe "Notes" do
     setup do
       user = insert(:user)
@@ -63,5 +69,55 @@ test "a note with an attachment should work", _ do
 
       %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
     end
+
+    test "a misskey MFM status with a content field should work and be linked", _ do
+      local_user = insert(:user, %{nickname: "akkoma_user"})
+
+      insert(:user, %{ap_id: "https://misskey.local.live/users/92hzkskwgy"})
+
+      note =
+        "test/fixtures/misskey/mfm_x_format.json"
+        |> File.read!()
+        |> Jason.decode!()
+
+      expected_content =
+        "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{local_user.id}\" href=\"#{local_user.ap_id}\" rel=\"ugc\">@<span>akkoma_user</span></a></span> linkifylink <a class=\"hashtag\" data-tag=\"dancedance\" href=\"http://localhost:4001/tag/dancedance\" rel=\"tag ugc\">#dancedance</a> $[jelly mfm goes here] <br><br>## aaa"
+
+      %{
+        valid?: true,
+        changes: %{
+          content: ^expected_content,
+          source: %{
+            "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa",
+            "mediaType" => "text/x.misskeymarkdown"
+          }
+        }
+      } = ArticleNotePageValidator.cast_and_validate(note)
+    end
+
+    test "a misskey MFM status with a _misskey_content field should work and be linked", _ do
+      local_user = insert(:user, %{nickname: "akkoma_user"})
+
+      insert(:user, %{ap_id: "https://misskey.local.live/users/92hzkskwgy"})
+
+      note =
+        "test/fixtures/misskey/mfm_underscore_format.json"
+        |> File.read!()
+        |> Jason.decode!()
+
+      expected_content =
+        "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{local_user.id}\" href=\"#{local_user.ap_id}\" rel=\"ugc\">@<span>akkoma_user</span></a></span> linkifylink <a class=\"hashtag\" data-tag=\"dancedance\" href=\"http://localhost:4001/tag/dancedance\" rel=\"tag ugc\">#dancedance</a> $[jelly mfm goes here] <br><br>## aaa"
+
+      %{
+        valid?: true,
+        changes: %{
+          content: ^expected_content,
+          source: %{
+            "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa",
+            "mediaType" => "text/x.misskeymarkdown"
+          }
+        }
+      } = ArticleNotePageValidator.cast_and_validate(note)
+    end
   end
 end