From 14b942443f46012ab5bfb125a7d87a857486771d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 9 Jul 2022 23:18:00 +0100 Subject: [PATCH 1/5] Actually make finch take options --- docs/development/setting_up_akkoma_dev.md | 27 +++++++++++++++++++++++ lib/pleroma/application.ex | 13 ++++++++--- lib/pleroma/http.ex | 1 - 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/development/setting_up_akkoma_dev.md b/docs/development/setting_up_akkoma_dev.md index 33ec24f6c..a0b5e441b 100644 --- a/docs/development/setting_up_akkoma_dev.md +++ b/docs/development/setting_up_akkoma_dev.md @@ -36,6 +36,33 @@ config :logger, :console, level: :info ``` +## Testing with HTTPS + +If you end up developing alongside other software like misskey, +you will not be able to federate without an SSL certificate. You should +be able to use the snakeoil certificate that comes standard with most +distributions or generate one from scratch, then force elixir to accept it. + +HTTP clients are none too keep to accept self-signed certs, but we can do +this: + +```elixir +config :pleroma, :http, + adapter: [ + pools: %{ + default: [ + conn_opts: [ + transport_opts: [ + verify: :verify_none + ] + ] + ] + } + ] +``` + +Now your SSL requests will work. Hooray. + ## Testing 1. Create a `test.secret.exs` file with the content as shown below diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 0f4f5a358..33ba86406 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -58,9 +58,6 @@ def start(_type, _args) do Pleroma.Docs.JSON.compile() limiters_setup() - Logger.info("Starting Finch") - Finch.start_link(name: MyFinch) - # Define workers and child supervisors to be supervised children = [ @@ -70,6 +67,7 @@ def start(_type, _args) do Pleroma.Web.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ + http_children() ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -276,4 +274,13 @@ def limiters_setup do ConcurrentLimiter.new(module, max_running, max_waiting) end) end + + defp http_children do + config = + [:http, :adapter] + |> Config.get([]) + |> Keyword.put(:name, MyFinch) + + [{Finch, config}] + end end diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index 01f307d17..d8028651c 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -65,7 +65,6 @@ def request(method, url, body, headers, options) when is_binary(url) do options = put_in(options[:adapter], adapter_opts) params = options[:params] || [] request = build_request(method, headers, options, url, body, params) - client = Tesla.client([Tesla.Middleware.FollowRedirects]) request(client, request) -- 2.43.0 From 291543d1cb881f330e15c81f7421545a79255295 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 9 Jul 2022 23:18:25 +0100 Subject: [PATCH 2/5] Mix format --- lib/pleroma/application.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 33ba86406..fcb1d6571 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -277,10 +277,10 @@ def limiters_setup do defp http_children do config = - [:http, :adapter] - |> Config.get([]) - |> Keyword.put(:name, MyFinch) - + [:http, :adapter] + |> Config.get([]) + |> Keyword.put(:name, MyFinch) + [{Finch, config}] end end -- 2.43.0 From 705a8a1fe95150c9dd995ed814de9847890dca52 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 10 Jul 2022 17:18:53 +0100 Subject: [PATCH 3/5] pre-link MFM content --- lib/pleroma/formatter.ex | 2 +- .../article_note_page_validator.ex | 16 +++++-- lib/pleroma/web/common_api/utils.ex | 12 +---- .../misskey/mfm_underscore_format.json | 31 ++++++++++++ test/fixtures/misskey/mfm_x_format.json | 34 +++++++++++++ .../article_note_page_validator_test.exs | 48 +++++++++++++++++++ 6 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 test/fixtures/misskey/mfm_underscore_format.json create mode 100644 test/fixtures/misskey/mfm_x_format.json diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index ae37946ab..78325095a 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -133,7 +133,7 @@ def html_escape(text, "text/html") do HTML.filter_tags(text) end - def html_escape(text, "text/plain") do + def html_escape(text, format) when format in ["text/plain", "text/x.misskeymarkdown"] do Regex.split(@link_regex, text, include_captures: true) |> Enum.map_every(2, fn chunk -> {:safe, part} = Phoenix.HTML.html_escape(chunk) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index a9f395c5e..17e80fa92 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object.Fetcher + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.Transmogrifier @@ -81,12 +82,21 @@ defp fix_replies(%{"replies" => %{"first" => first}} = data) do defp fix_replies(data), do: data # https://github.com/misskey-dev/misskey/pull/8787 - defp fix_misskey_content(%{"source" => %{"mediaType" => "text/x.misskeymarkdown"}} = object), - do: object + defp fix_misskey_content( + %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object + ) do + {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown") + put_in(object, ["source", "content"], linked) + end defp fix_misskey_content(%{"_misskey_content" => content} = object) do + {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown") + object - |> Map.put("source", %{"content" => content, "mediaType" => "text/x.misskeymarkdown"}) + |> Map.put("source", %{ + "content" => linked, + "mediaType" => "text/x.misskeymarkdown" + }) |> Map.delete("_misskey_content") end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 8fde9ae15..f5bc3acf5 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -259,7 +259,8 @@ def format_input(text, format, options \\ []) @doc """ Formatting text to plain text, BBCode, HTML, or Markdown """ - def format_input(text, "text/plain", options) do + def format_input(text, format, options) + when format in ["text/plain", "text/x.misskeymarkdown"] do text |> Formatter.html_escape("text/plain") |> Formatter.linkify(options) @@ -291,15 +292,6 @@ def format_input(text, "text/markdown", options) do |> Formatter.html_escape("text/html") end - def format_input(text, "text/x.misskeymarkdown", options) do - text - |> Formatter.html_escape("text/plain") - |> Formatter.linkify(options) - |> (fn {text, mentions, tags} -> - {String.replace(text, ~r/\r?\n/, "
"), mentions, tags} - end).() - end - def format_naive_asctime(date) do date |> DateTime.from_naive!("Etc/UTC") |> format_asctime end diff --git a/test/fixtures/misskey/mfm_underscore_format.json b/test/fixtures/misskey/mfm_underscore_format.json new file mode 100644 index 000000000..e031a954f --- /dev/null +++ b/test/fixtures/misskey/mfm_underscore_format.json @@ -0,0 +1,31 @@ +{ + "id": "https://misskey.local.live/notes/92j1n3owja", + "type": "Note", + "attributedTo": "https://misskey.local.live/users/92hzkskwgy", + "summary": null, + "content": "

@akkoma_user@akkoma.local.live linkifylink #dancedance mfm goes here

## aaa

", + "_misskey_content": "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "published": "2022-07-10T15:37:36.368Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://misskey.local.live/users/92hzkskwgy/followers", + "http://localhost:4001/users/akkoma_user" + ], + "inReplyTo": null, + "attachment": [], + "sensitive": false, + "tag": [ + { + "type": "Hashtag", + "href": "https://misskey.local.live/tags/dancedance", + "name": "#dancedance" + }, + { + "type": "Mention", + "href": "http://localhost:4001/users/akkoma_user", + "name": "@akkoma_user" + } + ] +} diff --git a/test/fixtures/misskey/mfm_x_format.json b/test/fixtures/misskey/mfm_x_format.json new file mode 100644 index 000000000..31d6e5368 --- /dev/null +++ b/test/fixtures/misskey/mfm_x_format.json @@ -0,0 +1,34 @@ +{ + "id": "https://misskey.local.live/notes/92j1n3owja", + "type": "Note", + "attributedTo": "https://misskey.local.live/users/92hzkskwgy", + "summary": null, + "content": "

@akkoma_user@akkoma.local.live linkifylink #dancedance mfm goes here

## aaa

", + "source": { + "content": "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "mediaType": "text/x.misskeymarkdown" + }, + "published": "2022-07-10T15:37:36.368Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://misskey.local.live/users/92hzkskwgy/followers", + "http://localhost:4001/users/akkoma_user" + ], + "inReplyTo": null, + "attachment": [], + "sensitive": false, + "tag": [ + { + "type": "Hashtag", + "href": "https://misskey.local.live/tags/dancedance", + "name": "#dancedance" + }, + { + "type": "Mention", + "href": "http://localhost:4001/users/akkoma_user", + "name": "@akkoma_user" + } + ] +} diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 717a704d4..7d424efb6 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -10,6 +10,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest import Pleroma.Factory + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + describe "Notes" do setup do user = insert(:user) @@ -63,5 +69,47 @@ test "a note with an attachment should work", _ do %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + + test "a misskey MFM status with a content field should work and be linked", _ do + local_user = insert(:user, %{nickname: "akkoma_user"}) + + insert(:user, %{ap_id: "https://misskey.local.live/users/92hzkskwgy"}) + + note = + "test/fixtures/misskey/mfm_x_format.json" + |> File.read!() + |> Jason.decode!() + + expected_content = + "@akkoma_user linkifylink #dancedance $[jelly mfm goes here]

## aaa" + + %{ + valid?: true, + changes: %{ + source: %{"content" => ^expected_content, "mediaType" => "text/x.misskeymarkdown"} + } + } = ArticleNotePageValidator.cast_and_validate(note) + end + + test "a misskey MFM status with a _misskey_content field should work and be linked", _ do + local_user = insert(:user, %{nickname: "akkoma_user"}) + + insert(:user, %{ap_id: "https://misskey.local.live/users/92hzkskwgy"}) + + note = + "test/fixtures/misskey/mfm_underscore_format.json" + |> File.read!() + |> Jason.decode!() + + expected_content = + "@akkoma_user linkifylink #dancedance $[jelly mfm goes here]

## aaa" + + %{ + valid?: true, + changes: %{ + source: %{"content" => ^expected_content, "mediaType" => "text/x.misskeymarkdown"} + } + } = ArticleNotePageValidator.cast_and_validate(note) + end end end -- 2.43.0 From 838b02533730ea5b270afb6233f6e564ae896189 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 10 Jul 2022 17:20:00 +0100 Subject: [PATCH 4/5] fix typo --- docs/development/setting_up_akkoma_dev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/setting_up_akkoma_dev.md b/docs/development/setting_up_akkoma_dev.md index a0b5e441b..7184be485 100644 --- a/docs/development/setting_up_akkoma_dev.md +++ b/docs/development/setting_up_akkoma_dev.md @@ -43,7 +43,7 @@ you will not be able to federate without an SSL certificate. You should be able to use the snakeoil certificate that comes standard with most distributions or generate one from scratch, then force elixir to accept it. -HTTP clients are none too keep to accept self-signed certs, but we can do +HTTP clients are none too keen to accept self-signed certs, but we can do this: ```elixir -- 2.43.0 From e71a7893ec4ffeeec89f4a7cb92ff029534a581a Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 10 Jul 2022 17:47:24 +0100 Subject: [PATCH 5/5] only link in `content` field --- .../object_validators/article_note_page_validator.ex | 5 +++-- .../article_note_page_validator_test.exs | 12 ++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 17e80fa92..e11335170 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -86,7 +86,7 @@ defp fix_misskey_content( %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object ) do {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown") - put_in(object, ["source", "content"], linked) + Map.put(object, "content", linked) end defp fix_misskey_content(%{"_misskey_content" => content} = object) do @@ -94,9 +94,10 @@ defp fix_misskey_content(%{"_misskey_content" => content} = object) do object |> Map.put("source", %{ - "content" => linked, + "content" => content, "mediaType" => "text/x.misskeymarkdown" }) + |> Map.put("content", linked) |> Map.delete("_misskey_content") end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 7d424efb6..62526d952 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -86,7 +86,11 @@ test "a misskey MFM status with a content field should work and be linked", _ do %{ valid?: true, changes: %{ - source: %{"content" => ^expected_content, "mediaType" => "text/x.misskeymarkdown"} + content: ^expected_content, + source: %{ + "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "mediaType" => "text/x.misskeymarkdown" + } } } = ArticleNotePageValidator.cast_and_validate(note) end @@ -107,7 +111,11 @@ test "a misskey MFM status with a _misskey_content field should work and be link %{ valid?: true, changes: %{ - source: %{"content" => ^expected_content, "mediaType" => "text/x.misskeymarkdown"} + content: ^expected_content, + source: %{ + "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "mediaType" => "text/x.misskeymarkdown" + } } } = ArticleNotePageValidator.cast_and_validate(note) end -- 2.43.0