Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 ### Security
 - Mastodon API: Fix display names not being sanitized
+- Rich media: Do not crawl private IP ranges
+
 ### Added
 - Add a generic settings store for frontends / clients to use.
 - Explicit addressing option for posting.
@@ -26,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Federation: Support for `Question` and `Answer` objects
 - Federation: Support for reports
 - Configuration: `poll_limits` option
+- Configuration: `pack_extensions` option
 - Configuration: `safe_dm_mentions` option
 - Configuration: `link_name` option
 - Configuration: `fetch_initial_posts` option
@@ -63,6 +66,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `skip_thread_containment` option
 - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details.
 - MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links.
+- Configuration: `ignore_hosts` option
+- Configuration: `ignore_tld` option
 
 ### Changed
 - **Breaking:** bind to 127.0.0.1 instead of 0.0.0.0 by default
@@ -102,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Posts which are marked sensitive or tagged nsfw no longer have link previews.
 - HTTP connection timeout is now set to 10 seconds.
 - Respond with a 404 Not implemented JSON error message when requested API is not implemented
+- Rich Media: crawl only https URLs.
 
 ### Fixed
 - Follow requests don't get 'stuck' anymore.

config/config.exs

@@ -99,6 +99,7 @@
 
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
+  pack_extensions: [".png", ".gif"],
   groups: [
     # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
     Custom: ["/emoji/*.png", "/emoji/**/*.png"]
@@ -330,7 +331,10 @@
 
 config :pleroma, :mrf_subchain, match_actor: %{}
 
-config :pleroma, :rich_media, enabled: true
+config :pleroma, :rich_media,
+  enabled: true,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
 
 config :pleroma, :media_proxy,
   enabled: false,

config/test.exs

@@ -43,7 +43,11 @@
 config :pbkdf2_elixir, rounds: 1
 
 config :tesla, adapter: Tesla.Mock
-config :pleroma, :rich_media, enabled: false
+
+config :pleroma, :rich_media,
+  enabled: false,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
 
 config :web_push_encryption, :vapid_details,
   subject: "mailto:administrator@example.com",

docs/config.md

@@ -417,6 +417,8 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
 
 ## :rich_media
 * `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
+* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"]
 
 ## :fetch_initial_posts
 * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
@@ -604,6 +606,7 @@ Configure OAuth 2 provider capabilities:
 
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
+* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
 

lib/pleroma/emoji.ex

@@ -148,11 +148,13 @@ defp load_pack(pack_dir, emoji_groups) do
     if File.exists?(emoji_txt) do
       load_from_file(emoji_txt, emoji_groups)
     else
+      extensions = Pleroma.Config.get([:emoji, :pack_extensions])
+
       Logger.info(
-        "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji"
+        "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
       )
 
-      make_shortcode_to_file_map(pack_dir, [".png"])
+      make_shortcode_to_file_map(pack_dir, extensions)
       |> Enum.map(fn {shortcode, rel_file} ->
         filename = Path.join("/emoji/#{pack_name}", rel_file)
 

lib/pleroma/web/common_api/common_api.ex

@@ -247,6 +247,7 @@ def post(user, %{"status" => status} = data) do
 
       res
     else
+      {:private_to_public, true} -> {:error, "The message visibility must be direct"}
       {:error, _} = e -> e
       e -> {:error, e}
     end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -844,7 +844,7 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end

lib/pleroma/web/rich_media/helpers.ex

@@ -4,27 +4,53 @@
 
 defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.HTML
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
+  @spec validate_page_url(any()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
 
-    if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do
+    page_url
-      URI.parse(page_url) |> validate_page_url
+    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
-    else
+    |> parse_uri(page_url)
+  end
+
+  defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
+       when scheme == "https" and not is_nil(authority) do
+    cond do
+      host in Config.get([:rich_media, :ignore_hosts], []) ->
         :error
+
+      get_tld(host) in Config.get([:rich_media, :ignore_tld], []) ->
+        :error
+
+      true ->
+        :ok
     end
   end
 
-  defp validate_page_url(%URI{authority: nil}), do: :error
-  defp validate_page_url(%URI{scheme: nil}), do: :error
-  defp validate_page_url(%URI{}), do: :ok
   defp validate_page_url(_), do: :error
 
+  defp parse_uri(true, url) do
+    url
+    |> URI.parse()
+    |> validate_page_url
+  end
+
+  defp parse_uri(_, _), do: :error
+
+  defp get_tld(host) do
+    host
+    |> String.split(".")
+    |> Enum.reverse()
+    |> hd
+  end
+
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
-    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+    with true <- Config.get([:rich_media, :enabled]),
          %Object{} = object <- Object.normalize(activity),
          false <- object.data["sensitive"] || false,
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),