From 5f9fbd7d336d2ffed6cd8f2640d9399f78ed7c2f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 13 Feb 2020 03:39:47 +0100 Subject: [PATCH] Formatting: Do not use \n and prefer
instead It moves bbcode to bbcode_pleroma as the former is owned by kaniini and transfering ownership wasn't done in a timely manner. Closes: https://git.pleroma.social/pleroma/pleroma/issues/1374 Closes: https://git.pleroma.social/pleroma/pleroma/issues/1375 --- CHANGELOG.md | 4 + lib/pleroma/earmark_renderer.ex | 256 ++++++++++++++++++ lib/pleroma/web/common_api/utils.ex | 2 +- mix.exs | 2 +- mix.lock | 6 +- test/earmark_renderer_test.ex | 79 ++++++ test/web/common_api/common_api_utils_test.exs | 28 +- 7 files changed, 357 insertions(+), 20 deletions(-) create mode 100644 lib/pleroma/earmark_renderer.ex create mode 100644 test/earmark_renderer_test.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 100228c6c..4168086e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [unreleased] +### Changed +- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `
` for newlines + ## [2.0.0] - 2019-03-08 ### Security - Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request. diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex new file mode 100644 index 000000000..6211a3b4a --- /dev/null +++ b/lib/pleroma/earmark_renderer.ex @@ -0,0 +1,256 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +# +# This file is derived from Earmark, under the following copyright: +# Copyright © 2014 Dave Thomas, The Pragmatic Programmers +# SPDX-License-Identifier: Apache-2.0 +# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex +defmodule Pleroma.EarmarkRenderer do + @moduledoc false + + alias Earmark.Block + alias Earmark.Context + alias Earmark.HtmlRenderer + alias Earmark.Options + + import Earmark.Inline, only: [convert: 3] + import Earmark.Helpers.HtmlHelpers + import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2] + import Earmark.Context, only: [append: 2, set_value: 2] + import Earmark.Options, only: [get_mapper: 1] + + @doc false + def render(blocks, %Context{options: %Options{}} = context) do + messages = get_messages(context) + + {contexts, html} = + get_mapper(context.options).( + blocks, + &render_block(&1, put_in(context.options.messages, [])) + ) + |> Enum.unzip() + + all_messages = + contexts + |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end) + + {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()} + end + + ############# + # Paragraph # + ############# + defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do + lines = convert(lines, lnb, context) + add_attrs(lines, "

#{lines.value}

", attrs, [], lnb) + end + + ######## + # Html # + ######## + defp render_block(%Block.Html{html: html}, context) do + {context, html} + end + + defp render_block(%Block.HtmlComment{lines: lines}, context) do + {context, lines} + end + + defp render_block(%Block.HtmlOneline{html: html}, context) do + {context, html} + end + + ######### + # Ruler # + ######### + defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do + add_attrs(context, "
", attrs, [], lnb) + end + + ########### + # Heading # + ########### + defp render_block( + %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs}, + context + ) do + converted = convert(content, lnb, context) + html = "#{converted.value}" + add_attrs(converted, html, attrs, [], lnb) + end + + ############## + # Blockquote # + ############## + + defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do + {context1, body} = render(blocks, context) + html = "
#{body}
" + add_attrs(context1, html, attrs, [], lnb) + end + + ######### + # Table # + ######### + + defp render_block( + %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs}, + context + ) do + {context1, html} = add_attrs(context, "", attrs, [], lnb) + context2 = set_value(context1, html) + + context3 = + if header do + append(add_trs(append(context2, ""), [header], "th", aligns, lnb), "") + else + # Maybe an error, needed append(context, html) + context2 + end + + context4 = append(add_trs(append(context3, ""), rows, "td", aligns, lnb), "") + + {context4, [context4.value, "
"]} + end + + ######## + # Code # + ######## + + defp render_block( + %Block.Code{lnb: lnb, language: language, attrs: attrs} = block, + %Context{options: options} = context + ) do + class = + if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: "" + + tag = ~s[
]
+    lines = options.render_code.(block)
+    html = ~s[#{tag}#{lines}
] + add_attrs(context, html, attrs, [], lnb) + end + + ######### + # Lists # + ######### + + defp render_block( + %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start}, + context + ) do + {context1, content} = render(items, context) + html = "<#{type}#{start}>#{content}" + add_attrs(context1, html, attrs, [], lnb) + end + + # format a single paragraph list item, and remove the para tags + defp render_block( + %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs}, + context + ) + when length(blocks) == 1 do + {context1, content} = render(blocks, context) + content = Regex.replace(~r{}, content, "") + html = "
  • #{content}
    • " + add_attrs(context1, html, attrs, [], lnb) + end + + # format a spaced list item + defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do + {context1, content} = render(blocks, context) + html = "
  • #{content}
    • " + add_attrs(context1, html, attrs, [], lnb) + end + + ################## + # Footnote Block # + ################## + + defp render_block(%Block.FnList{blocks: footnotes}, context) do + items = + Enum.map(footnotes, fn note -> + blocks = append_footnote_link(note) + %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks} + end) + + {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context) + {context1, Enum.join([~s[
    ], "
    ", html, "
    "])} + end + + ####################################### + # Isolated IALs are rendered as paras # + ####################################### + + defp render_block(%Block.Ial{verbatim: verbatim}, context) do + {context, "

    {:#{verbatim}}

    "} + end + + #################### + # IDDef is ignored # + #################### + + defp render_block(%Block.IdDef{}, context), do: {context, ""} + + ##################################### + # And here are the inline renderers # + ##################################### + + defdelegate br, to: HtmlRenderer + defdelegate codespan(text), to: HtmlRenderer + defdelegate em(text), to: HtmlRenderer + defdelegate strong(text), to: HtmlRenderer + defdelegate strikethrough(text), to: HtmlRenderer + + defdelegate link(url, text), to: HtmlRenderer + defdelegate link(url, text, title), to: HtmlRenderer + + defdelegate image(path, alt, title), to: HtmlRenderer + + defdelegate footnote_link(ref, backref, number), to: HtmlRenderer + + # Table rows + defp add_trs(context, rows, tag, aligns, lnb) do + numbered_rows = + rows + |> Enum.zip(Stream.iterate(lnb, &(&1 + 1))) + + numbered_rows + |> Enum.reduce(context, fn {row, lnb}, ctx -> + append(add_tds(append(ctx, ""), row, tag, aligns, lnb), "") + end) + end + + defp add_tds(context, row, tag, aligns, lnb) do + Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb)) + end + + defp add_td_fn(row, tag, aligns, lnb) do + fn n, ctx -> + style = + case Enum.at(aligns, n - 1, :default) do + :default -> "" + align -> " style=\"text-align: #{align}\"" + end + + col = Enum.at(row, n - 1) + converted = convert(col, lnb, set_messages(ctx, [])) + append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}") + end + end + + ############################### + # Append Footnote Return Link # + ############################### + + defdelegate append_footnote_link(note), to: HtmlRenderer + defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer + + defdelegate render_code(lines), to: HtmlRenderer + + defp code_classes(language, prefix) do + ["" | String.split(prefix || "")] + |> Enum.map(fn pfx -> "#{pfx}#{language}" end) + |> Enum.join(" ") + end +end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 348fdedf1..635e7cd38 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -331,7 +331,7 @@ def format_input(text, "text/html", options) do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!() + |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer}) |> Formatter.linkify(options) |> Formatter.html_escape("text/html") end diff --git a/mix.exs b/mix.exs index 90aeaf33a..b55a79a77 100644 --- a/mix.exs +++ b/mix.exs @@ -126,7 +126,7 @@ defp deps do {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, {:earmark, "~> 1.3"}, - {:bbcode, "~> 0.1.1"}, + {:bbcode_pleroma, "~> 0.2.0"}, {:ex_machina, "~> 2.3", only: :test}, {:credo, "~> 1.1.0", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.3", only: :test}, diff --git a/mix.lock b/mix.lock index c8b30a6f9..1b4fbc927 100644 --- a/mix.lock +++ b/mix.lock @@ -3,10 +3,11 @@ "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, - "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5a981b98ac7d366a9b6bf40eac389aaf4d6e623c631e6b6f8a6b571efaafd338"}, + "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]}, + "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, - "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"}, + "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"}, "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, @@ -110,4 +111,3 @@ "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "9315c8f37c108835cf3f8e9157d7a9b8f420a34f402d1b1620a31aed5b93ecdf"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, } - diff --git a/test/earmark_renderer_test.ex b/test/earmark_renderer_test.ex new file mode 100644 index 000000000..220d97d16 --- /dev/null +++ b/test/earmark_renderer_test.ex @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.EarmarkRendererTest do + use ExUnit.Case + + test "Paragraph" do + code = ~s[Hello\n\nWorld!] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "

    Hello

    World!

    " + end + + test "raw HTML" do + code = ~s[OwO] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "

    #{code}

    " + end + + test "rulers" do + code = ~s[before\n\n-----\n\nafter] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "

    before

    after

    " + end + + test "headings" do + code = ~s[# h1\n## h2\n### h3\n] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    h1

    h2

    h3

    ] + end + + test "blockquote" do + code = ~s[> whoms't are you quoting?] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "

    whoms’t are you quoting?

    " + end + + test "code" do + code = ~s[`mix`] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    mix

    ] + + code = ~s[``mix``] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    mix

    ] + + code = ~s[```\nputs "Hello World"\n```] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[
    puts "Hello World"
    ] + end + + test "lists" do + code = ~s[- one\n- two\n- three\n- four] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "
    • one
    • two
    • three
    • four
    " + + code = ~s[1. one\n2. two\n3. three\n4. four\n] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "
    1. one
    2. two
    3. three
    4. four
    " + end + + test "delegated renderers" do + code = ~s[a
    b] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == "

    #{code}

    " + + code = ~s[*aaaa~*] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    aaaa~

    ] + + code = ~s[**aaaa~**] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    aaaa~

    ] + + # strikethrought + code = ~s[aaaa~] + result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + assert result == ~s[

    aaaa~

    ] + end +end diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index b380d10d8..45fc94522 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -89,8 +89,8 @@ test "works for bare text/html" do assert output == expected - text = "

    hello world!

    \n\n

    second paragraph

    " - expected = "

    hello world!

    \n\n

    second paragraph

    " + text = "

    hello world!


    \n

    second paragraph

    " + expected = "

    hello world!


    \n

    second paragraph

    " {output, [], []} = Utils.format_input(text, "text/html") @@ -99,14 +99,14 @@ test "works for bare text/html" do test "works for bare text/markdown" do text = "**hello world**" - expected = "

    hello world

    \n" + expected = "

    hello world

    " {output, [], []} = Utils.format_input(text, "text/markdown") assert output == expected text = "**hello world**\n\n*another paragraph*" - expected = "

    hello world

    \n

    another paragraph

    \n" + expected = "

    hello world

    another paragraph

    " {output, [], []} = Utils.format_input(text, "text/markdown") @@ -118,7 +118,7 @@ test "works for bare text/markdown" do by someone """ - expected = "

    cool quote

    \n
    \n

    by someone

    \n" + expected = "

    cool quote

    by someone

    " {output, [], []} = Utils.format_input(text, "text/markdown") @@ -134,7 +134,7 @@ test "works for bare text/bbcode" do assert output == expected text = "[b]hello world![/b]\n\nsecond paragraph!" - expected = "hello world!
    \n
    \nsecond paragraph!" + expected = "hello world!

    second paragraph!" {output, [], []} = Utils.format_input(text, "text/bbcode") @@ -143,7 +143,7 @@ test "works for bare text/bbcode" do text = "[b]hello world![/b]\n\nsecond paragraph!" expected = - "hello world!
    \n
    \n<strong>second paragraph!</strong>" + "hello world!

    <strong>second paragraph!</strong>" {output, [], []} = Utils.format_input(text, "text/bbcode") @@ -156,16 +156,14 @@ test "works for text/markdown with mentions" do text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*" - expected = - ~s(

    hello world

    \n

    another @user__test and @user__test google.com paragraph

    \n) - {output, _, _} = Utils.format_input(text, "text/markdown") - assert output == expected + assert output == + ~s(

    hello world

    another @user__test and @user__test google.com paragraph

    ) end end