Merge branch 'issues/1016' into 'develop'

[#1016] add clear formatted text/ removes links with private ip

See merge request pleroma/pleroma!1335

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 ### Security
 - Mastodon API: Fix display names not being sanitized
+- Rich media: Do not crawl private IP ranges
+
 ### Added
 - Add a generic settings store for frontends / clients to use.
 - Explicit addressing option for posting.
@@ -63,6 +65,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `skip_thread_containment` option
 - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details.
 - MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links.
+- Configuration: `ignore_hosts` option
+- Configuration: `ignore_tld` option
 
 ### Changed
 - **Breaking:** bind to 127.0.0.1 instead of 0.0.0.0 by default
@@ -102,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
 - HTTP connection timeout is now set to 10 seconds.
 - Respond with a 404 Not implemented JSON error message when requested API is not implemented
+- Rich Media: crawl only https URLs.
 
 ### Fixed
 - Follow requests don't get 'stuck' anymore.

config/config.exs

@@ -330,7 +330,10 @@
 
 config :pleroma, :mrf_subchain, match_actor: %{}
 
-config :pleroma, :rich_media, enabled: true
+config :pleroma, :rich_media,
+  enabled: true,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
 
 config :pleroma, :media_proxy,
   enabled: false,

config/test.exs

@@ -43,7 +43,11 @@
 config :pbkdf2_elixir, rounds: 1
 
 config :tesla, adapter: Tesla.Mock
-config :pleroma, :rich_media, enabled: false
+
+config :pleroma, :rich_media,
+  enabled: false,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
 
 config :web_push_encryption, :vapid_details,
   subject: "mailto:administrator@example.com",

docs/config.md

@@ -417,6 +417,8 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
 
 ## :rich_media
 * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
+* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"]
 
 ## :fetch_initial_posts
 * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts

lib/pleroma/web/rich_media/helpers.ex

@@ -4,27 +4,53 @@
 
 defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.HTML
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
+  @spec validate_page_url(any()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
 
-    if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do
+    page_url
-      URI.parse(page_url) |> validate_page_url
+    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
-    else
+    |> parse_uri(page_url)
-      :error
+  end
+
+  defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
+       when scheme == "https" and not is_nil(authority) do
+    cond do
+      host in Config.get([:rich_media, :ignore_hosts], []) ->
+        :error
+
+      get_tld(host) in Config.get([:rich_media, :ignore_tld], []) ->
+        :error
+
+      true ->
+        :ok
     end
   end
 
-  defp validate_page_url(%URI{authority: nil}), do: :error
-  defp validate_page_url(%URI{scheme: nil}), do: :error
-  defp validate_page_url(%URI{}), do: :ok
   defp validate_page_url(_), do: :error
 
+  defp parse_uri(true, url) do
+    url
+    |> URI.parse()
+    |> validate_page_url
+  end
+
+  defp parse_uri(_, _), do: :error
+
+  defp get_tld(host) do
+    host
+    |> String.split(".")
+    |> Enum.reverse()
+    |> hd
+  end
+
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
-    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+    with true <- Config.get([:rich_media, :enabled]),
          %Object{} = object <- Object.normalize(activity),
          false <- object.data["sensitive"] || false,
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),

test/support/http_request_mock.ex

@@ -757,6 +757,14 @@ def get("http://example.com/ogp", _, _, _) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
   end
 
+  def get("https://example.com/ogp", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
+  end
+
+  def get("https://pleroma.local/notice/9kCP7V", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
+  end
+
   def get("http://example.com/ogp-missing-data", _, _, _) do
     {:ok,
      %Tesla.Env{
@@ -765,6 +773,14 @@ def get("http://example.com/ogp-missing-data", _, _, _) do
      }}
   end
 
+  def get("https://example.com/ogp-missing-data", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/rich_media/ogp-missing-data.html")
+     }}
+  end
+
   def get("http://example.com/malformed", _, _, _) do
     {:ok,
      %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}}

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -312,7 +312,7 @@ test "posting a status with OGP link preview", %{conn: conn} do
       conn
       |> assign(:user, user)
       |> post("/api/v1/statuses", %{
-        "status" => "http://example.com/ogp"
+        "status" => "https://example.com/ogp"
       })
 
     assert %{"id" => id, "card" => %{"title" => "The Rock"}} = json_response(conn, 200)
@@ -2557,7 +2557,7 @@ test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do
     end
 
     test "returns rich-media card", %{conn: conn, user: user} do
-      {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"})
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "https://example.com/ogp"})
 
       card_data = %{
         "image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -2589,7 +2589,7 @@ test "returns rich-media card", %{conn: conn, user: user} do
 
       # works with private posts
       {:ok, activity} =
-        CommonAPI.post(user, %{"status" => "http://example.com/ogp", "visibility" => "direct"})
+        CommonAPI.post(user, %{"status" => "https://example.com/ogp", "visibility" => "direct"})
 
       response_two =
         conn
@@ -2601,7 +2601,8 @@ test "returns rich-media card", %{conn: conn, user: user} do
     end
 
     test "replaces missing description with an empty string", %{conn: conn, user: user} do
-      {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp-missing-data"})
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "https://example.com/ogp-missing-data"})
 
       response =
         conn

test/web/rich_media/helpers_test.exs

@@ -1,14 +1,19 @@
 defmodule Pleroma.Web.RichMedia.HelpersTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.RichMedia.Helpers
 
   import Pleroma.Factory
   import Tesla.Mock
 
   setup do
     mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    rich_media = Config.get([:rich_media, :enabled])
+    on_exit(fn -> Config.put([:rich_media, :enabled], rich_media) end)
+
     :ok
   end
 
@@ -21,11 +26,9 @@ test "refuses to crawl incomplete URLs" do
         "content_type" => "text/markdown"
       })
 
-    Pleroma.Config.put([:rich_media, :enabled], true)
+    Config.put([:rich_media, :enabled], true)
 
     assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
-    Pleroma.Config.put([:rich_media, :enabled], false)
   end
 
   test "refuses to crawl malformed URLs" do
@@ -37,11 +40,9 @@ test "refuses to crawl malformed URLs" do
         "content_type" => "text/markdown"
       })
 
-    Pleroma.Config.put([:rich_media, :enabled], true)
+    Config.put([:rich_media, :enabled], true)
 
     assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
-    Pleroma.Config.put([:rich_media, :enabled], false)
   end
 
   test "crawls valid, complete URLs" do
@@ -49,16 +50,14 @@ test "crawls valid, complete URLs" do
 
     {:ok, activity} =
       CommonAPI.post(user, %{
-        "status" => "[test](http://example.com/ogp)",
+        "status" => "[test](https://example.com/ogp)",
         "content_type" => "text/markdown"
       })
 
-    Pleroma.Config.put([:rich_media, :enabled], true)
+    Config.put([:rich_media, :enabled], true)
 
-    assert %{page_url: "http://example.com/ogp", rich_media: _} =
+    assert %{page_url: "https://example.com/ogp", rich_media: _} =
              Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
-    Pleroma.Config.put([:rich_media, :enabled], false)
   end
 
   test "refuses to crawl URLs from posts marked sensitive" do
@@ -74,11 +73,9 @@ test "refuses to crawl URLs from posts marked sensitive" do
 
     assert object.data["sensitive"]
 
-    Pleroma.Config.put([:rich_media, :enabled], true)
+    Config.put([:rich_media, :enabled], true)
 
     assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
-    Pleroma.Config.put([:rich_media, :enabled], false)
   end
 
   test "refuses to crawl URLs from posts tagged NSFW" do
@@ -93,10 +90,28 @@ test "refuses to crawl URLs from posts tagged NSFW" do
 
     assert object.data["sensitive"]
 
-    Pleroma.Config.put([:rich_media, :enabled], true)
+    Config.put([:rich_media, :enabled], true)
 
     assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+  end
 
-    Pleroma.Config.put([:rich_media, :enabled], false)
+  test "refuses to crawl URLs of private network from posts" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{"status" => "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
+
+    {:ok, activity2} = CommonAPI.post(user, %{"status" => "https://10.111.10.1/notice/9kCP7V"})
+    {:ok, activity3} = CommonAPI.post(user, %{"status" => "https://172.16.32.40/notice/9kCP7V"})
+    {:ok, activity4} = CommonAPI.post(user, %{"status" => "https://192.168.10.40/notice/9kCP7V"})
+    {:ok, activity5} = CommonAPI.post(user, %{"status" => "https://pleroma.local/notice/9kCP7V"})
+
+    Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Helpers.fetch_data_for_activity(activity)
+    assert %{} = Helpers.fetch_data_for_activity(activity2)
+    assert %{} = Helpers.fetch_data_for_activity(activity3)
+    assert %{} = Helpers.fetch_data_for_activity(activity4)
+    assert %{} = Helpers.fetch_data_for_activity(activity5)
   end
 end