From e9e17e5df34051bce60232890ea042582af31f8c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 00:27:51 -0500 Subject: [PATCH 01/16] Upgrade Earmark to v1.4.10 --- lib/pleroma/earmark_renderer.ex | 256 ------------------ lib/pleroma/formatter.ex | 8 + .../audio_video_validator.ex | 3 +- lib/pleroma/web/common_api/utils.ex | 3 +- mix.exs | 2 +- mix.lock | 2 +- test/pleroma/formatter_test.exs | 7 + test/pleroma/web/common_api/utils_test.exs | 75 +++++ 8 files changed, 95 insertions(+), 261 deletions(-) delete mode 100644 lib/pleroma/earmark_renderer.ex diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex deleted file mode 100644 index 6211a3b4a..000000000 --- a/lib/pleroma/earmark_renderer.ex +++ /dev/null @@ -1,256 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only -# -# This file is derived from Earmark, under the following copyright: -# Copyright © 2014 Dave Thomas, The Pragmatic Programmers -# SPDX-License-Identifier: Apache-2.0 -# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex -defmodule Pleroma.EarmarkRenderer do - @moduledoc false - - alias Earmark.Block - alias Earmark.Context - alias Earmark.HtmlRenderer - alias Earmark.Options - - import Earmark.Inline, only: [convert: 3] - import Earmark.Helpers.HtmlHelpers - import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2] - import Earmark.Context, only: [append: 2, set_value: 2] - import Earmark.Options, only: [get_mapper: 1] - - @doc false - def render(blocks, %Context{options: %Options{}} = context) do - messages = get_messages(context) - - {contexts, html} = - get_mapper(context.options).( - blocks, - &render_block(&1, put_in(context.options.messages, [])) - ) - |> Enum.unzip() - - all_messages = - contexts - |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end) - - {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()} - end - - ############# - # Paragraph # - ############# - defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do - lines = convert(lines, lnb, context) - add_attrs(lines, "

#{lines.value}

", attrs, [], lnb) - end - - ######## - # Html # - ######## - defp render_block(%Block.Html{html: html}, context) do - {context, html} - end - - defp render_block(%Block.HtmlComment{lines: lines}, context) do - {context, lines} - end - - defp render_block(%Block.HtmlOneline{html: html}, context) do - {context, html} - end - - ######### - # Ruler # - ######### - defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do - add_attrs(context, "
", attrs, [], lnb) - end - - ########### - # Heading # - ########### - defp render_block( - %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs}, - context - ) do - converted = convert(content, lnb, context) - html = "#{converted.value}" - add_attrs(converted, html, attrs, [], lnb) - end - - ############## - # Blockquote # - ############## - - defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, body} = render(blocks, context) - html = "
#{body}
" - add_attrs(context1, html, attrs, [], lnb) - end - - ######### - # Table # - ######### - - defp render_block( - %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs}, - context - ) do - {context1, html} = add_attrs(context, "", attrs, [], lnb) - context2 = set_value(context1, html) - - context3 = - if header do - append(add_trs(append(context2, ""), [header], "th", aligns, lnb), "") - else - # Maybe an error, needed append(context, html) - context2 - end - - context4 = append(add_trs(append(context3, ""), rows, "td", aligns, lnb), "") - - {context4, [context4.value, "
"]} - end - - ######## - # Code # - ######## - - defp render_block( - %Block.Code{lnb: lnb, language: language, attrs: attrs} = block, - %Context{options: options} = context - ) do - class = - if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: "" - - tag = ~s[
]
-    lines = options.render_code.(block)
-    html = ~s[#{tag}#{lines}
] - add_attrs(context, html, attrs, [], lnb) - end - - ######### - # Lists # - ######### - - defp render_block( - %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start}, - context - ) do - {context1, content} = render(items, context) - html = "<#{type}#{start}>#{content}" - add_attrs(context1, html, attrs, [], lnb) - end - - # format a single paragraph list item, and remove the para tags - defp render_block( - %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs}, - context - ) - when length(blocks) == 1 do - {context1, content} = render(blocks, context) - content = Regex.replace(~r{}, content, "") - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - # format a spaced list item - defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, content} = render(blocks, context) - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - ################## - # Footnote Block # - ################## - - defp render_block(%Block.FnList{blocks: footnotes}, context) do - items = - Enum.map(footnotes, fn note -> - blocks = append_footnote_link(note) - %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks} - end) - - {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context) - {context1, Enum.join([~s[
    ], "
    ", html, "
    "])} - end - - ####################################### - # Isolated IALs are rendered as paras # - ####################################### - - defp render_block(%Block.Ial{verbatim: verbatim}, context) do - {context, "

    {:#{verbatim}}

    "} - end - - #################### - # IDDef is ignored # - #################### - - defp render_block(%Block.IdDef{}, context), do: {context, ""} - - ##################################### - # And here are the inline renderers # - ##################################### - - defdelegate br, to: HtmlRenderer - defdelegate codespan(text), to: HtmlRenderer - defdelegate em(text), to: HtmlRenderer - defdelegate strong(text), to: HtmlRenderer - defdelegate strikethrough(text), to: HtmlRenderer - - defdelegate link(url, text), to: HtmlRenderer - defdelegate link(url, text, title), to: HtmlRenderer - - defdelegate image(path, alt, title), to: HtmlRenderer - - defdelegate footnote_link(ref, backref, number), to: HtmlRenderer - - # Table rows - defp add_trs(context, rows, tag, aligns, lnb) do - numbered_rows = - rows - |> Enum.zip(Stream.iterate(lnb, &(&1 + 1))) - - numbered_rows - |> Enum.reduce(context, fn {row, lnb}, ctx -> - append(add_tds(append(ctx, ""), row, tag, aligns, lnb), "") - end) - end - - defp add_tds(context, row, tag, aligns, lnb) do - Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb)) - end - - defp add_td_fn(row, tag, aligns, lnb) do - fn n, ctx -> - style = - case Enum.at(aligns, n - 1, :default) do - :default -> "" - align -> " style=\"text-align: #{align}\"" - end - - col = Enum.at(row, n - 1) - converted = convert(col, lnb, set_messages(ctx, [])) - append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}") - end - end - - ############################### - # Append Footnote Return Link # - ############################### - - defdelegate append_footnote_link(note), to: HtmlRenderer - defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer - - defdelegate render_code(lines), to: HtmlRenderer - - defp code_classes(language, prefix) do - ["" | String.split(prefix || "")] - |> Enum.map(fn pfx -> "#{pfx}#{language}" end) - |> Enum.join(" ") - end -end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 0c450eae4..b0e4a84ae 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -138,6 +138,14 @@ def html_escape(text, "text/plain") do |> Enum.join("") end + def minify({text, mentions, hashtags}, type) do + {minify(text, type), mentions, hashtags} + end + + def minify(text, "text/html") do + String.replace(text, "\n", "") + end + def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 16973e5db..eaf94797a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do use Ecto.Schema - alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes @@ -96,7 +95,7 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) when is_binary(content) do content = content - |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) + |> Earmark.as_html!() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1c74ea787..b434a069e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,8 +294,9 @@ def format_input(text, "text/html", options) do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + |> Earmark.as_html!() |> Formatter.linkify(options) + |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end diff --git a/mix.exs b/mix.exs index 72a6346b5..feb7eefa3 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defp deps do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.3"}, + {:earmark, "1.4.10"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index 6b551a012..29439a438 100644 --- a/mix.lock +++ b/mix.lock @@ -27,7 +27,7 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, + "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, diff --git a/test/pleroma/formatter_test.exs b/test/pleroma/formatter_test.exs index 5781a3f01..ceedd1b6d 100644 --- a/test/pleroma/formatter_test.exs +++ b/test/pleroma/formatter_test.exs @@ -307,4 +307,11 @@ test "it escapes HTML in plain text" do assert Formatter.html_escape(text, "text/plain") == expected end + + test "it minifies html" do + text = "

    \nhello

    \n

    \nworld

    \n" + expected = "

    hello

    world

    " + + assert Formatter.minify(text, "text/html") == expected + end end diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 4d6c9ea26..39ea08ca8 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -168,6 +168,81 @@ test "works for text/markdown with mentions" do end end + describe "format_input/3 with markdown" do + test "Paragraph" do + code = ~s[Hello\n\nWorld!] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    Hello

    World!

    " + end + + test "raw HTML" do + code = ~s[OwO] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    #{code}

    " + end + + test "rulers" do + code = ~s[before\n\n-----\n\nafter] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    before

    after

    " + end + + test "headings" do + code = ~s[# h1\n## h2\n### h3\n] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    h1

    h2

    h3

    ] + end + + test "blockquote" do + code = ~s[> whoms't are you quoting?] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    whoms’t are you quoting?

    " + end + + test "code" do + code = ~s[`mix`] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    mix

    ] + + code = ~s[``mix``] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    mix

    ] + + code = ~s[```\nputs "Hello World"\n```] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[
    puts "Hello World"
    ] + end + + test "lists" do + code = ~s[- one\n- two\n- three\n- four] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "" + + code = ~s[1. one\n2. two\n3. three\n4. four\n] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "
    1. one
    2. two
    3. three
    4. four
    " + end + + test "delegated renderers" do + code = ~s[a
    b] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    #{code}

    " + + code = ~s[*aaaa~*] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + + code = ~s[**aaaa~**] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + + # strikethrought + code = ~s[aaaa~] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + end + end + describe "context_to_conversation_id" do test "creates a mapping object" do conversation_id = Utils.context_to_conversation_id("random context") From ba71bbf6101847292346ba3b1fbe78ce4c385919 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 01:53:25 -0500 Subject: [PATCH 02/16] Improve Formatter.minify/2 --- lib/pleroma/formatter.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index b0e4a84ae..61906dda6 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -143,7 +143,10 @@ def minify({text, mentions, hashtags}, type) do end def minify(text, "text/html") do - String.replace(text, "\n", "") + text + |> String.replace(">\n", ">") + |> String.replace("> ", ">") + |> String.replace(" <", "<") end def truncate(text, max_length \\ 200, omission \\ "...") do From c4f4e48e574362d1ec86eaf11a382e81ca97cb35 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 02:08:41 -0500 Subject: [PATCH 03/16] Remove some N/A tests --- test/pleroma/web/common_api/utils_test.exs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 39ea08ca8..c6abbbe84 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -187,12 +187,6 @@ test "rulers" do assert result == "

    before

    after

    " end - test "headings" do - code = ~s[# h1\n## h2\n### h3\n] - {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[

    h1

    h2

    h3

    ] - end - test "blockquote" do code = ~s[> whoms't are you quoting?] {result, [], []} = Utils.format_input(code, "text/markdown") @@ -224,10 +218,6 @@ test "lists" do end test "delegated renderers" do - code = ~s[a
    b] - {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    #{code}

    " - code = ~s[*aaaa~*] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] @@ -236,7 +226,7 @@ test "delegated renderers" do {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] - # strikethrought + # strikethrough code = ~s[aaaa~] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] From b2548cfcdabdcb90bfcc9f4022c0b1cff9157a4a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 13:54:53 -0500 Subject: [PATCH 04/16] Sanitizer: allow
    tags --- priv/scrubbers/default.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 7b06994de..0893b17e5 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -39,6 +39,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:code, []) Meta.allow_tag_with_these_attributes(:del, []) Meta.allow_tag_with_these_attributes(:em, []) + Meta.allow_tag_with_these_attributes(:hr, []) Meta.allow_tag_with_these_attributes(:i, []) Meta.allow_tag_with_these_attributes(:li, []) Meta.allow_tag_with_these_attributes(:ol, []) From f8c93246d69a193ead81248879ba260e98673b3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 14:27:50 -0500 Subject: [PATCH 05/16] Refactor Earmark code, fix tests --- lib/pleroma/formatter.ex | 4 ++++ .../object_validators/audio_video_validator.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 2 +- priv/scrubbers/default.ex | 2 ++ test/pleroma/web/common_api/utils_test.exs | 10 +++++----- test/pleroma/web/common_api_test.exs | 2 +- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 61906dda6..1be12055f 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -121,6 +121,10 @@ def mentions_escape(text, options \\ []) do end end + def markdown_to_html(text) do + Earmark.as_html!(text) + end + def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index eaf94797a..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -95,7 +95,7 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) when is_binary(content) do content = content - |> Earmark.as_html!() + |> Pleroma.Formatter.markdown_to_html() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b434a069e..be86009af 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,7 +294,7 @@ def format_input(text, "text/html", options) do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!() + |> Formatter.markdown_to_html() |> Formatter.linkify(options) |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 0893b17e5..4694a92a5 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -59,6 +59,8 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"]) Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) + @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index c6abbbe84..ab6392b1f 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -178,13 +178,13 @@ test "Paragraph" do test "raw HTML" do code = ~s[OwO] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    #{code}

    " + assert result == ~s[OwO] end test "rulers" do code = ~s[before\n\n-----\n\nafter] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    before

    after

    " + assert result == "

    before

    after

    " end test "blockquote" do @@ -204,7 +204,7 @@ test "code" do code = ~s[```\nputs "Hello World"\n```] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[
    puts "Hello World"
    ] + assert result == ~s[
    puts "Hello World"
    ] end test "lists" do @@ -227,9 +227,9 @@ test "delegated renderers" do assert result == ~s[

    aaaa~

    ] # strikethrough - code = ~s[aaaa~] + code = ~s[~~aaaa~~~] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[

    aaaa~

    ] + assert result == ~s[

    aaaa~

    ] end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 585b2c174..c1b1af073 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -558,7 +558,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do object = Object.normalize(activity) - assert object.data["content"] == "

    2hu

    alert('xss')" + assert object.data["content"] == "

    2hu

    " assert object.data["source"] == post end From f1c67115d89ddcc7b10b963579dd621fca2094db Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 18:09:49 -0500 Subject: [PATCH 06/16] Upgrade linkify, test URL issues, fixes #2026 #1942 --- test/pleroma/web/common_api/utils_test.exs | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index ab6392b1f..28b05ed91 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -175,6 +175,54 @@ test "Paragraph" do assert result == "

    Hello

    World!

    " end + test "links" do + code = "https://en.wikipedia.org/wiki/Animal_Crossing_(video_game)" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + + code = "https://github.com/pragdave/earmark/" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + end + + test "link with local mention" do + insert(:user, %{nickname: "lain"}) + + code = "https://example.com/@lain" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + end + + test "local mentions" do + mario = insert(:user, %{nickname: "mario"}) + luigi = insert(:user, %{nickname: "luigi"}) + + code = "@mario @luigi yo what's up?" + {result, _, []} = Utils.format_input(code, "text/markdown") + + assert result == + ~s[

    @mario @luigi yo what’s up?

    ] + end + + test "remote mentions" do + mario = insert(:user, %{nickname: "mario@mushroom.kingdom", local: false}) + luigi = insert(:user, %{nickname: "luigi@mushroom.kingdom", local: false}) + + code = "@mario@mushroom.kingdom @luigi@mushroom.kingdom yo what's up?" + {result, _, []} = Utils.format_input(code, "text/markdown") + + assert result == + ~s[

    @mario @luigi yo what’s up?

    ] + end + test "raw HTML" do code = ~s[OwO] {result, [], []} = Utils.format_input(code, "text/markdown") @@ -205,6 +253,10 @@ test "code" do code = ~s[```\nputs "Hello World"\n```] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[
    puts "Hello World"
    ] + + code = ~s[
    \n
    ] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[
    <div>\n</div>
    ] end test "lists" do From 642729b49fca41fb142c6121fedf35c96c03b018 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 19:16:57 -0500 Subject: [PATCH 07/16] Fix AudioVideoValidator markdown --- .../web/activity_pub/object_validators/audio_video_validator.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 9b38aa4c2..fa3e2c026 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,6 +96,7 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) content = content |> Pleroma.Formatter.markdown_to_html() + |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) From 6520599b7deac56780e1496c969cc45ff2e9f5da Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Dec 2020 13:43:40 -0600 Subject: [PATCH 08/16] Update Earmark to 1.4.13, use the new compact_output mode --- lib/pleroma/formatter.ex | 2 +- mix.exs | 2 +- mix.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 1be12055f..2aa236ca9 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -122,7 +122,7 @@ def mentions_escape(text, options \\ []) do end def markdown_to_html(text) do - Earmark.as_html!(text) + Earmark.as_html!(text, %Earmark.Options{compact_output: true}) end def html_escape({text, mentions, hashtags}, type) do diff --git a/mix.exs b/mix.exs index feb7eefa3..06d77edb7 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defp deps do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.10"}, + {:earmark, "1.4.13"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index 29439a438..e4dd32c83 100644 --- a/mix.lock +++ b/mix.lock @@ -27,8 +27,8 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, + "earmark": {:hex, :earmark, "1.4.13", "2c6ce9768fc9fdbf4046f457e207df6360ee6c91ee1ecb8e9a139f96a4289d91", [:mix], [{:earmark_parser, ">= 1.4.12", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "a0cf3ed88ef2b1964df408889b5ecb886d1a048edde53497fc935ccd15af3403"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, From f318d8e56df1e30f41c7ddf2e306b3552034921f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Dec 2020 17:28:00 -0600 Subject: [PATCH 09/16] Use Pleroma.Formatter.markdown_to_html/1 in the tests --- test/pleroma/earmark_renderer_test.exs | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/pleroma/earmark_renderer_test.exs b/test/pleroma/earmark_renderer_test.exs index 220d97d16..3adbefc1e 100644 --- a/test/pleroma/earmark_renderer_test.exs +++ b/test/pleroma/earmark_renderer_test.exs @@ -6,74 +6,74 @@ defmodule Pleroma.EarmarkRendererTest do test "Paragraph" do code = ~s[Hello\n\nWorld!] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    Hello

    World!

    " end test "raw HTML" do code = ~s[OwO] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    #{code}

    " end test "rulers" do code = ~s[before\n\n-----\n\nafter] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    before

    after

    " end test "headings" do code = ~s[# h1\n## h2\n### h3\n] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    h1

    h2

    h3

    ] end test "blockquote" do code = ~s[> whoms't are you quoting?] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    whoms’t are you quoting?

    " end test "code" do code = ~s[`mix`] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    mix

    ] code = ~s[``mix``] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    mix

    ] code = ~s[```\nputs "Hello World"\n```] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[
    puts "Hello World"
    ] end test "lists" do code = ~s[- one\n- two\n- three\n- four] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "
    • one
    • two
    • three
    • four
    " code = ~s[1. one\n2. two\n3. three\n4. four\n] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "
    1. one
    2. two
    3. three
    4. four
    " end test "delegated renderers" do code = ~s[a
    b] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    #{code}

    " code = ~s[*aaaa~*] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] code = ~s[**aaaa~**] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] # strikethrought code = ~s[aaaa~] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] end end From 004bcedb074d50bc42803e4c0a884239bd504b3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:23:11 -0500 Subject: [PATCH 10/16] Upgrade Earmark 1.4.15 --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index 06d77edb7..8ba2d8fbc 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defp deps do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.13"}, + {:earmark, "1.4.15"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index e4dd32c83..06542f18d 100644 --- a/mix.lock +++ b/mix.lock @@ -27,8 +27,8 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.13", "2c6ce9768fc9fdbf4046f457e207df6360ee6c91ee1ecb8e9a139f96a4289d91", [:mix], [{:earmark_parser, ">= 1.4.12", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "a0cf3ed88ef2b1964df408889b5ecb886d1a048edde53497fc935ccd15af3403"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, + "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, From 6727a3659f60c0e09fa6375b6c0843c01f5be3dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:27:06 -0500 Subject: [PATCH 11/16] Remove Pleroma.Formatter.minify/2 --- lib/pleroma/formatter.ex | 11 ----------- .../object_validators/audio_video_validator.ex | 1 - lib/pleroma/web/common_api/utils.ex | 1 - test/pleroma/formatter_test.exs | 7 ------- 4 files changed, 20 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 2aa236ca9..baf652a5a 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -142,17 +142,6 @@ def html_escape(text, "text/plain") do |> Enum.join("") end - def minify({text, mentions, hashtags}, type) do - {minify(text, type), mentions, hashtags} - end - - def minify(text, "text/html") do - text - |> String.replace(">\n", ">") - |> String.replace("> ", ">") - |> String.replace(" <", "<") - end - def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index fa3e2c026..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,7 +96,6 @@ defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) content = content |> Pleroma.Formatter.markdown_to_html() - |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index be86009af..4731e79be 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -296,7 +296,6 @@ def format_input(text, "text/markdown", options) do |> Formatter.mentions_escape(options) |> Formatter.markdown_to_html() |> Formatter.linkify(options) - |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end diff --git a/test/pleroma/formatter_test.exs b/test/pleroma/formatter_test.exs index ceedd1b6d..5781a3f01 100644 --- a/test/pleroma/formatter_test.exs +++ b/test/pleroma/formatter_test.exs @@ -307,11 +307,4 @@ test "it escapes HTML in plain text" do assert Formatter.html_escape(text, "text/plain") == expected end - - test "it minifies html" do - text = "

    \nhello

    \n

    \nworld

    \n" - expected = "

    hello

    world

    " - - assert Formatter.minify(text, "text/html") == expected - end end From 53760d2cda9b9f241355365b3fff9852bcb1a8a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:51:18 -0500 Subject: [PATCH 12/16] Delete obsolete EarmarkRendereTests (moved to UtilsTest) --- test/pleroma/earmark_renderer_test.exs | 79 -------------------------- 1 file changed, 79 deletions(-) delete mode 100644 test/pleroma/earmark_renderer_test.exs diff --git a/test/pleroma/earmark_renderer_test.exs b/test/pleroma/earmark_renderer_test.exs deleted file mode 100644 index 3adbefc1e..000000000 --- a/test/pleroma/earmark_renderer_test.exs +++ /dev/null @@ -1,79 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.EarmarkRendererTest do - use ExUnit.Case - - test "Paragraph" do - code = ~s[Hello\n\nWorld!] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    Hello

    World!

    " - end - - test "raw HTML" do - code = ~s[OwO] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    #{code}

    " - end - - test "rulers" do - code = ~s[before\n\n-----\n\nafter] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    before

    after

    " - end - - test "headings" do - code = ~s[# h1\n## h2\n### h3\n] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    h1

    h2

    h3

    ] - end - - test "blockquote" do - code = ~s[> whoms't are you quoting?] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    whoms’t are you quoting?

    " - end - - test "code" do - code = ~s[`mix`] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    mix

    ] - - code = ~s[``mix``] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    mix

    ] - - code = ~s[```\nputs "Hello World"\n```] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[
    puts "Hello World"
    ] - end - - test "lists" do - code = ~s[- one\n- two\n- three\n- four] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "
    • one
    • two
    • three
    • four
    " - - code = ~s[1. one\n2. two\n3. three\n4. four\n] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "
    1. one
    2. two
    3. three
    4. four
    " - end - - test "delegated renderers" do - code = ~s[a
    b] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    #{code}

    " - - code = ~s[*aaaa~*] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - - code = ~s[**aaaa~**] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - - # strikethrought - code = ~s[aaaa~] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - end -end From a8fa00ef666f574aec8048626aed78a7d62e6915 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:55:43 -0500 Subject: [PATCH 13/16] Fix failing remote mentions test, valid TLDs --- test/pleroma/web/common_api/utils_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 28b05ed91..8c79a9a83 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -209,10 +209,10 @@ test "local mentions" do end test "remote mentions" do - mario = insert(:user, %{nickname: "mario@mushroom.kingdom", local: false}) - luigi = insert(:user, %{nickname: "luigi@mushroom.kingdom", local: false}) + mario = insert(:user, %{nickname: "mario@mushroom.world", local: false}) + luigi = insert(:user, %{nickname: "luigi@mushroom.world", local: false}) - code = "@mario@mushroom.kingdom @luigi@mushroom.kingdom yo what's up?" + code = "@mario@mushroom.world @luigi@mushroom.world yo what's up?" {result, _, []} = Utils.format_input(code, "text/markdown") assert result == From dca87c5e7b4b12918cf59a83a77be389a7e0df01 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 1 May 2021 11:28:06 -0500 Subject: [PATCH 14/16] CHANGELOG: markdown --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0171763..ed6e548dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Applying ConcurrentLimiter settings via AdminAPI - User login failures if their `notification_settings` were in a NULL state. - Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity +- Fixed some Markdown issues, including trailing slash in links. ## [2.3.0] - 2020-03-01 From c80b1aaf514dec6b538a9833d48df027708b6b4d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 May 2021 14:27:03 -0500 Subject: [PATCH 15/16] Don't crash when email settings are invalid Fixes: https://git.pleroma.social/pleroma/pleroma/-/issues/2606 Fixes: https://gitlab.com/soapbox-pub/soapbox/-/issues/4 --- lib/pleroma/application_requirements.ex | 38 ++++++++++--------- .../pleroma/application_requirements_test.exs | 18 ++++----- test/pleroma/user_test.exs | 18 +++++++++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 6ef65b263..c412dec5e 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,15 +34,16 @@ defp handle_result({:error, message}), do: raise(VerifyError, message: message) defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.error(""" - To send welcome email do you need to enable mail. - \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true - """) + Logger.warn(""" + To send welcome emails, you need to enable the mailer. + Welcome emails will NOT be sent with the current config. - {:error, "The mail disabled."} - else - :ok + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end defp check_welcome_message_config!(result), do: result @@ -51,18 +52,21 @@ defp check_welcome_message_config!(result), do: result # def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && - not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do - Logger.error( - "Account activation enabled, but no Mailer settings enabled.\n" <> - "Please set config :pleroma, :instance, account_activation_required: false\n" <> - "Otherwise setup and enable Mailer." - ) + not Pleroma.Emails.Mailer.enabled?() do + Logger.warn(""" + Account activation is required, but the mailer is disabled. + Users will NOT be able to confirm their accounts with this config. + Either disable account activation or enable the mailer. - {:error, - "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} - else - :ok + Disable account activation: + config :pleroma, :instance, account_activation_required: false + + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end def check_confirmation_accounts!(result), do: result diff --git a/test/pleroma/application_requirements_test.exs b/test/pleroma/application_requirements_test.exs index 683ac8c96..a54c37968 100644 --- a/test/pleroma/application_requirements_test.exs +++ b/test/pleroma/application_requirements_test.exs @@ -35,13 +35,13 @@ test "doesn't raise if the pool size is unexpected but the respective flag is se setup do: clear_config([:welcome]) setup do: clear_config([Pleroma.Emails.Mailer]) - test "raises if welcome email enabled but mail disabled" do + test "warns if welcome email enabled but mail disabled" do clear_config([:welcome, :email, :enabled], true) clear_config([Pleroma.Emails.Mailer, :enabled], false) - assert_raise Pleroma.ApplicationRequirements.VerifyError, "The mail disabled.", fn -> - capture_log(&Pleroma.ApplicationRequirements.verify!/0) - end + assert capture_log(fn -> + assert Pleroma.ApplicationRequirements.verify!() == :ok + end) =~ "Welcome emails will NOT be sent" end end @@ -57,15 +57,13 @@ test "raises if welcome email enabled but mail disabled" do setup do: clear_config([:instance, :account_activation_required]) - test "raises if account confirmation is required but mailer isn't enable" do + test "warns if account confirmation is required but mailer isn't enabled" do clear_config([:instance, :account_activation_required], true) clear_config([Pleroma.Emails.Mailer, :enabled], false) - assert_raise Pleroma.ApplicationRequirements.VerifyError, - "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails.", - fn -> - capture_log(&Pleroma.ApplicationRequirements.verify!/0) - end + assert capture_log(fn -> + assert Pleroma.ApplicationRequirements.verify!() == :ok + end) =~ "Users will NOT be able to confirm their accounts" end test "doesn't do anything if account confirmation is disabled" do diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 6f5bcab57..f89ea458a 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -572,6 +572,24 @@ test "it sends a registration confirmed email if no others will be sent" do ) end + test "it fails gracefully with invalid email config" do + cng = User.register_changeset(%User{}, @full_user_data) + + # Disable the mailer but enable all the things that want to send emails + clear_config([Pleroma.Emails.Mailer, :enabled], false) + clear_config([:instance, :account_activation_required], true) + clear_config([:instance, :account_approval_required], true) + clear_config([:welcome, :email, :enabled], true) + clear_config([:welcome, :email, :sender], "lain@lain.com") + + # The user is still created + assert {:ok, %User{nickname: "nick"}} = User.register(cng) + + # No emails are sent + ObanHelpers.perform_all() + refute_email_sent() + end + test "it requires an email, name, nickname and password, bio is optional when account_activation_required is enabled" do clear_config([:instance, :account_activation_required], true) From 90770e0841d3ffea87627b35627bfe38cad52f07 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 May 2021 14:30:21 -0500 Subject: [PATCH 16/16] CHANGELOG: don't crash so hard when email settings are invalid --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0171763..74086a54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. +### Fixed +- Don't crash so hard when email settings are invalid. + ## Unreleased (Patch) ### Fixed