diff --git a/CHANGELOG.md b/CHANGELOG.md
index 100228c6c..4168086e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+## [unreleased]
+### Changed
+- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `
` for newlines
+
## [2.0.0] - 2019-03-08
### Security
- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex
new file mode 100644
index 000000000..6211a3b4a
--- /dev/null
+++ b/lib/pleroma/earmark_renderer.ex
@@ -0,0 +1,256 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+#
+# This file is derived from Earmark, under the following copyright:
+# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
+# SPDX-License-Identifier: Apache-2.0
+# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
+defmodule Pleroma.EarmarkRenderer do
+ @moduledoc false
+
+ alias Earmark.Block
+ alias Earmark.Context
+ alias Earmark.HtmlRenderer
+ alias Earmark.Options
+
+ import Earmark.Inline, only: [convert: 3]
+ import Earmark.Helpers.HtmlHelpers
+ import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
+ import Earmark.Context, only: [append: 2, set_value: 2]
+ import Earmark.Options, only: [get_mapper: 1]
+
+ @doc false
+ def render(blocks, %Context{options: %Options{}} = context) do
+ messages = get_messages(context)
+
+ {contexts, html} =
+ get_mapper(context.options).(
+ blocks,
+ &render_block(&1, put_in(context.options.messages, []))
+ )
+ |> Enum.unzip()
+
+ all_messages =
+ contexts
+ |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
+
+ {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
+ end
+
+ #############
+ # Paragraph #
+ #############
+ defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
+ lines = convert(lines, lnb, context)
+ add_attrs(lines, "
#{lines.value}
", attrs, [], lnb)
+ end
+
+ ########
+ # Html #
+ ########
+ defp render_block(%Block.Html{html: html}, context) do
+ {context, html}
+ end
+
+ defp render_block(%Block.HtmlComment{lines: lines}, context) do
+ {context, lines}
+ end
+
+ defp render_block(%Block.HtmlOneline{html: html}, context) do
+ {context, html}
+ end
+
+ #########
+ # Ruler #
+ #########
+ defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
+ add_attrs(context, "
", attrs, [], lnb)
+ end
+
+ ###########
+ # Heading #
+ ###########
+ defp render_block(
+ %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
+ context
+ ) do
+ converted = convert(content, lnb, context)
+ html = "#{converted.value}"
+ add_attrs(converted, html, attrs, [], lnb)
+ end
+
+ ##############
+ # Blockquote #
+ ##############
+
+ defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+ {context1, body} = render(blocks, context)
+ html = "#{body}
"
+ add_attrs(context1, html, attrs, [], lnb)
+ end
+
+ #########
+ # Table #
+ #########
+
+ defp render_block(
+ %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
+ context
+ ) do
+ {context1, html} = add_attrs(context, "", attrs, [], lnb)
+ context2 = set_value(context1, html)
+
+ context3 =
+ if header do
+ append(add_trs(append(context2, ""), [header], "th", aligns, lnb), "")
+ else
+ # Maybe an error, needed append(context, html)
+ context2
+ end
+
+ context4 = append(add_trs(append(context3, ""), rows, "td", aligns, lnb), "")
+
+ {context4, [context4.value, "
"]}
+ end
+
+ ########
+ # Code #
+ ########
+
+ defp render_block(
+ %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
+ %Context{options: options} = context
+ ) do
+ class =
+ if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
+
+ tag = ~s[]
+ lines = options.render_code.(block)
+ html = ~s[#{tag}#{lines}
]
+ add_attrs(context, html, attrs, [], lnb)
+ end
+
+ #########
+ # Lists #
+ #########
+
+ defp render_block(
+ %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
+ context
+ ) do
+ {context1, content} = render(items, context)
+ html = "<#{type}#{start}>#{content}#{type}>"
+ add_attrs(context1, html, attrs, [], lnb)
+ end
+
+ # format a single paragraph list item, and remove the para tags
+ defp render_block(
+ %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
+ context
+ )
+ when length(blocks) == 1 do
+ {context1, content} = render(blocks, context)
+ content = Regex.replace(~r{?p>}, content, "")
+ html = "#{content}"
+ add_attrs(context1, html, attrs, [], lnb)
+ end
+
+ # format a spaced list item
+ defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+ {context1, content} = render(blocks, context)
+ html = "#{content}"
+ add_attrs(context1, html, attrs, [], lnb)
+ end
+
+ ##################
+ # Footnote Block #
+ ##################
+
+ defp render_block(%Block.FnList{blocks: footnotes}, context) do
+ items =
+ Enum.map(footnotes, fn note ->
+ blocks = append_footnote_link(note)
+ %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
+ end)
+
+ {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
+ {context1, Enum.join([~s["])}
+ end
+
+ #######################################
+ # Isolated IALs are rendered as paras #
+ #######################################
+
+ defp render_block(%Block.Ial{verbatim: verbatim}, context) do
+ {context, "{:#{verbatim}}
"}
+ end
+
+ ####################
+ # IDDef is ignored #
+ ####################
+
+ defp render_block(%Block.IdDef{}, context), do: {context, ""}
+
+ #####################################
+ # And here are the inline renderers #
+ #####################################
+
+ defdelegate br, to: HtmlRenderer
+ defdelegate codespan(text), to: HtmlRenderer
+ defdelegate em(text), to: HtmlRenderer
+ defdelegate strong(text), to: HtmlRenderer
+ defdelegate strikethrough(text), to: HtmlRenderer
+
+ defdelegate link(url, text), to: HtmlRenderer
+ defdelegate link(url, text, title), to: HtmlRenderer
+
+ defdelegate image(path, alt, title), to: HtmlRenderer
+
+ defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
+
+ # Table rows
+ defp add_trs(context, rows, tag, aligns, lnb) do
+ numbered_rows =
+ rows
+ |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
+
+ numbered_rows
+ |> Enum.reduce(context, fn {row, lnb}, ctx ->
+ append(add_tds(append(ctx, ""), row, tag, aligns, lnb), "
")
+ end)
+ end
+
+ defp add_tds(context, row, tag, aligns, lnb) do
+ Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
+ end
+
+ defp add_td_fn(row, tag, aligns, lnb) do
+ fn n, ctx ->
+ style =
+ case Enum.at(aligns, n - 1, :default) do
+ :default -> ""
+ align -> " style=\"text-align: #{align}\""
+ end
+
+ col = Enum.at(row, n - 1)
+ converted = convert(col, lnb, set_messages(ctx, []))
+ append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}#{tag}>")
+ end
+ end
+
+ ###############################
+ # Append Footnote Return Link #
+ ###############################
+
+ defdelegate append_footnote_link(note), to: HtmlRenderer
+ defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
+
+ defdelegate render_code(lines), to: HtmlRenderer
+
+ defp code_classes(language, prefix) do
+ ["" | String.split(prefix || "")]
+ |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
+ |> Enum.join(" ")
+ end
+end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 348fdedf1..635e7cd38 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -331,7 +331,7 @@ def format_input(text, "text/html", options) do
def format_input(text, "text/markdown", options) do
text
|> Formatter.mentions_escape(options)
- |> Earmark.as_html!()
+ |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
|> Formatter.linkify(options)
|> Formatter.html_escape("text/html")
end
diff --git a/mix.exs b/mix.exs
index 90aeaf33a..b55a79a77 100644
--- a/mix.exs
+++ b/mix.exs
@@ -126,7 +126,7 @@ defp deps do
{:ex_aws_s3, "~> 2.0"},
{:sweet_xml, "~> 0.6.6"},
{:earmark, "~> 1.3"},
- {:bbcode, "~> 0.1.1"},
+ {:bbcode_pleroma, "~> 0.2.0"},
{:ex_machina, "~> 2.3", only: :test},
{:credo, "~> 1.1.0", only: [:dev, :test], runtime: false},
{:mock, "~> 0.3.3", only: :test},
diff --git a/mix.lock b/mix.lock
index c8b30a6f9..1b4fbc927 100644
--- a/mix.lock
+++ b/mix.lock
@@ -3,10 +3,11 @@
"auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]},
"base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"},
"base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
- "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5a981b98ac7d366a9b6bf40eac389aaf4d6e623c631e6b6f8a6b571efaafd338"},
+ "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]},
+ "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
"benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"},
"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
- "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"},
+ "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
"calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"},
"captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]},
"certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
@@ -110,4 +111,3 @@
"web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "9315c8f37c108835cf3f8e9157d7a9b8f420a34f402d1b1620a31aed5b93ecdf"},
"websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []},
}
-
diff --git a/test/earmark_renderer_test.ex b/test/earmark_renderer_test.ex
new file mode 100644
index 000000000..220d97d16
--- /dev/null
+++ b/test/earmark_renderer_test.ex
@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors
+# SPDX-License-Identifier: AGPL-3.0-only
+defmodule Pleroma.EarmarkRendererTest do
+ use ExUnit.Case
+
+ test "Paragraph" do
+ code = ~s[Hello\n\nWorld!]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "Hello
World!
"
+ end
+
+ test "raw HTML" do
+ code = ~s[OwO]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "#{code}
"
+ end
+
+ test "rulers" do
+ code = ~s[before\n\n-----\n\nafter]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "before
after
"
+ end
+
+ test "headings" do
+ code = ~s[# h1\n## h2\n### h3\n]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[h1
h2
h3
]
+ end
+
+ test "blockquote" do
+ code = ~s[> whoms't are you quoting?]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "whoms’t are you quoting?
"
+ end
+
+ test "code" do
+ code = ~s[`mix`]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[mix
]
+
+ code = ~s[``mix``]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[mix
]
+
+ code = ~s[```\nputs "Hello World"\n```]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[puts "Hello World"
]
+ end
+
+ test "lists" do
+ code = ~s[- one\n- two\n- three\n- four]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ""
+
+ code = ~s[1. one\n2. two\n3. three\n4. four\n]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "- one
- two
- three
- four
"
+ end
+
+ test "delegated renderers" do
+ code = ~s[a
b]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == "#{code}
"
+
+ code = ~s[*aaaa~*]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[aaaa~
]
+
+ code = ~s[**aaaa~**]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[aaaa~
]
+
+ # strikethrought
+ code = ~s[aaaa~]
+ result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer})
+ assert result == ~s[aaaa~
]
+ end
+end
diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs
index b380d10d8..45fc94522 100644
--- a/test/web/common_api/common_api_utils_test.exs
+++ b/test/web/common_api/common_api_utils_test.exs
@@ -89,8 +89,8 @@ test "works for bare text/html" do
assert output == expected
- text = "hello world!
\n\nsecond paragraph
"
- expected = "hello world!
\n\nsecond paragraph
"
+ text = "hello world!
\nsecond paragraph
"
+ expected = "hello world!
\nsecond paragraph
"
{output, [], []} = Utils.format_input(text, "text/html")
@@ -99,14 +99,14 @@ test "works for bare text/html" do
test "works for bare text/markdown" do
text = "**hello world**"
- expected = "hello world
\n"
+ expected = "hello world
"
{output, [], []} = Utils.format_input(text, "text/markdown")
assert output == expected
text = "**hello world**\n\n*another paragraph*"
- expected = "hello world
\nanother paragraph
\n"
+ expected = "hello world
another paragraph
"
{output, [], []} = Utils.format_input(text, "text/markdown")
@@ -118,7 +118,7 @@ test "works for bare text/markdown" do
by someone
"""
- expected = "cool quote
\n
\nby someone
\n"
+ expected = "cool quote
by someone
"
{output, [], []} = Utils.format_input(text, "text/markdown")
@@ -134,7 +134,7 @@ test "works for bare text/bbcode" do
assert output == expected
text = "[b]hello world![/b]\n\nsecond paragraph!"
- expected = "hello world!
\n
\nsecond paragraph!"
+ expected = "hello world!
second paragraph!"
{output, [], []} = Utils.format_input(text, "text/bbcode")
@@ -143,7 +143,7 @@ test "works for bare text/bbcode" do
text = "[b]hello world![/b]\n\nsecond paragraph!"
expected =
- "hello world!
\n
\n<strong>second paragraph!</strong>"
+ "hello world!
<strong>second paragraph!</strong>"
{output, [], []} = Utils.format_input(text, "text/bbcode")
@@ -156,16 +156,14 @@ test "works for text/markdown with mentions" do
text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*"
- expected =
- ~s(hello world
\nanother @user__test and @user__test google.com paragraph
\n)
-
{output, _, _} = Utils.format_input(text, "text/markdown")
- assert output == expected
+ assert output ==
+ ~s(hello world
another @user__test and @user__test google.com paragraph
)
end
end