Merge branch 'develop' into stable

CHANGELOG.md

@@ -4,6 +4,41 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## 2024.03
+
+## Added
+- CLI tasks best-effort checking for past abuse of the recent spoofing exploit
+- new `:mrf_steal_emoji, :download_unknown_size` option; defaults to `false`
+
+## Changed
+- `Pleroma.Upload, :base_url` now MUST be configured explicitly if used;
+  use of the same domain as the instance is **strongly** discouraged
+- `:media_proxy, :base_url` now MUST be configured explicitly if used;
+  use of the same domain as the instance is **strongly** discouraged
+- StealEmoji:
+  - now uses the pack.json format;
+    existing users must migrate with an out-of-band script (check release notes)
+  - only steals shortcodes recognised as valid
+  - URLs of stolen emoji is no longer predictable
+- The `Dedupe` upload filter is now always active;
+  `AnonymizeFilenames` is again opt-in
+- received AP data is sanity checked before we attempt to parse it as a user
+- Uploads, emoji and media proxy now restrict Content-Type headers to a safe subset
+- Akkoma will no longer fetch and parse objects hosted on the same domain
+
+## Fixed
+- Critical security issue allowing Akkoma to be used as a vector for
+  (depending on configuration) impersonation of other users or creation
+  of bogus users and posts on the upload domain
+- Critical security issue letting Akkoma fall for the above impersonation
+  payloads due to lack of strict id checking
+- Critical security issue allowing domains redirect to to pose as the initial domain
+  (e.g. with media proxy's fallback redirects)
+- refetched objects can no longer attribute themselves to third-party actors
+  (this had no externally visible effect since actor info is read from the Create activity)
+- our litepub JSON-LD schema is now served with the correct content type
+- remote APNG attachments are now recognised as images
+
 ## 2024.02
 
 ## Added

SECURITY.md

@@ -1,16 +1,21 @@
-# Pleroma backend security policy
+# Akkoma backend security handling
-
-## Supported versions
-
-Currently, Pleroma offers bugfixes and security patches only for the latest minor release.
-
-| Version | Support 
-|---------| --------
-| 2.2     | Bugfixes and security patches
 
 ## Reporting a vulnerability
 
-Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities.
+Please send an email (preferably encrypted) or
+a DM via our IRC to one of the following people:
+
+| Forgejo nick | IRC nick      | Email         | GPG                                     |
+| ------------ | ------------- | ------------- | --------------------------------------- |
+| floatinghost | FloatingGhost | *see GPG key* | https://coffee-and-dreams.uk/pubkey.asc |
+
 ## Announcements
 
-New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at <https://pleroma.social/announcements/tags/security/feed.xml>. 
+New releases and security issues are announced at
+[meta.akkoma.dev](https://meta.akkoma.dev/c/releases) and
+[@akkoma@ihatebeinga.live](https://ihatebeinga.live/akkoma).
+
+Both also offer RSS feeds
+([meta](https://meta.akkoma.dev/c/releases/7.rss),
+[fedi](https://ihatebeinga.live/users/akkoma.rss))
+so you can keep an eye on it without any accounts.

config/config.exs

@@ -61,11 +61,12 @@
 # Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
-  filters: [Pleroma.Upload.Filter.Dedupe],
+  filters: [],
   link_name: false,
   proxy_remote: false,
   filename_display_max_length: 30,
-  base_url: nil
+  base_url: nil,
+  allowed_mime_types: ["image", "audio", "video"]
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
@@ -148,18 +149,38 @@
   format: "$metadata[$level] $message",
   metadata: [:request_id]
 
+# ———————————————————————————————————————————————————————————————
+#                       W  A  R  N  I  N  G
+# ———————————————————————————————————————————————————————————————
+#
+#  Whenever adding a privileged new custom type for e.g.
+#  ActivityPub objects, ALWAYS map their extension back
+#  to "application/octet-stream".
+#  Else files served by us can automatically end up with
+#  those privileged types causing severe security hazards.
+#  (We need those mappings so Phoenix can assoiate its format
+#   (the "extension") to incoming requests of those MIME types)
+#
+# ———————————————————————————————————————————————————————————————
 config :mime, :types, %{
   "application/xml" => ["xml"],
   "application/xrd+xml" => ["xrd+xml"],
   "application/jrd+json" => ["jrd+json"],
   "application/activity+json" => ["activity+json"],
-  "application/ld+json" => ["activity+json"]
+  "application/ld+json" => ["activity+json"],
+  # Can be removed when bumping MIME past 2.0.5
+  # see https://akkoma.dev/AkkomaGang/akkoma/issues/657
+  "image/apng" => ["apng"]
 }
 
 config :mime, :extensions, %{
-  "activity+json" => "application/activity+json"
+  "xrd+xml" => "text/plain",
+  "jrd+json" => "text/plain",
+  "activity+json" => "text/plain"
 }
 
+# ———————————————————————————————————————————————————————————————
+
 config :tesla, :adapter, {Tesla.Adapter.Finch, name: MyFinch}
 
 # Configures http settings, upstream proxy etc.

config/description.exs

@@ -105,6 +105,19 @@
           "https://cdn-host.com"
         ]
       },
+      %{
+        key: :allowed_mime_types,
+        label: "Allowed MIME types",
+        type: {:list, :string},
+        description:
+          "List of MIME (main) types uploads are allowed to identify themselves with. Other types may still be uploaded, but will identify as a generic binary to clients. WARNING: Loosening this over the defaults can lead to security issues. Removing types is safe, but only add to the list if you are sure you know what you are doing.",
+        suggestions: [
+          "image",
+          "audio",
+          "video",
+          "font"
+        ]
+      },
       %{
         key: :proxy_remote,
         type: :boolean,

docs/docs/administration/CLI_tasks/security.md

@@ -0,0 +1,56 @@
+# Security-related tasks
+
+{! administration/CLI_tasks/general_cli_task_info.include !}
+
+!!! danger
+    Many of these tasks were written in response to a patched exploit.
+    It is recommended to run those very soon after installing its respective security update.
+    Over time with db migrations they might become less accurate or be removed altogether.
+    If you never ran an affected version, there’s no point in running them.
+
+## Spoofed AcitivityPub objects exploit (2024-03, fixed in 3.11.1)
+
+### Search for uploaded spoofing payloads
+
+Scans local uploads for spoofing payloads.
+If the instance is not using the local uploader it was not affected.
+Attachments wil be scanned anyway in case local uploader was used in the past.
+
+!!! note
+    This cannot reliably detect payloads attached to deleted posts.
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl security spoof-uploaded
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.security spoof-uploaded
+    ```
+
+### Search for counterfeit posts in database
+
+Scans all notes in the database for signs of being spoofed.
+
+!!! note
+    Spoofs targeting local accounts can be detected rather reliably
+    (with some restrictions documented in the task’s logs).
+    Counterfeit posts from remote users cannot. A best-effort attempt is made, but
+    a thorough attacker can avoid this and it may yield a small amount of false positives.
+
+    Should you find counterfeit posts of local users, let other admins know so they can delete the too.
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl security spoof-inserted
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.security spoof-inserted
+    ```

docs/docs/configuration/cheatsheet.md

@@ -236,7 +236,9 @@ config :pleroma, :mrf_user_allowlist, %{
 #### :mrf_steal_emoji
 * `hosts`: List of hosts to steal emojis from
 * `rejected_shortcodes`: Regex-list of shortcodes to reject
-* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
+* `size_limit`: File size limit (in bytes), checked before download if possible (and remote server honest),
+   otherwise or again checked before saving emoji to the disk
+* `download_unknown_size`: whether to download an emoji when the remote server doesn’t report its size in advance
 
 #### :mrf_activity_expiration
 
@@ -396,7 +398,8 @@ This section describe PWA manifest instance-specific values. Currently this opti
 ## :media_proxy
 
 * `enabled`: Enables proxying of remote media to the instance’s proxy
-* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
+* `base_url`: The base URL to access a user-uploaded file.
+  Using a (sub)domain distinct from the instance endpoint is **strongly** recommended.
 * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
 * `whitelist`: List of hosts with scheme to bypass the mediaproxy (e.g. `https://example.com`)
 * `invalidation`: options for remove media from cache after delete object:
@@ -597,8 +600,9 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 
 * `uploader`: Which one of the [uploaders](#uploaders) to use.
 * `filters`: List of [upload filters](#upload-filters) to use.
-* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
+* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
-* `base_url`: The base URL to access a user-uploaded file. Useful when you want to host the media files via another domain or are using a 3rd party S3 provider.
+* `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
+  Using a (sub)domain distinct from the instance endpoint is **strongly** recommended.
 * `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
@@ -638,17 +642,18 @@ config :ex_aws, :s3,
 
 ### Upload filters
 
-#### Pleroma.Upload.Filter.AnonymizeFilename
-
-This filter replaces the filename (not the path) of an upload. For complete obfuscation, add
-`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename.
-
-* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
-
 #### Pleroma.Upload.Filter.Dedupe
 
+**Always** active; cannot be turned off.
+Renames files to their hash and prevents duplicate files filling up the disk.
 No specific configuration.
 
+#### Pleroma.Upload.Filter.AnonymizeFilename
+
+This filter replaces the declared filename (not the path) of an upload.
+
+* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
+
 #### Pleroma.Upload.Filter.Exiftool
 
 This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.

docs/docs/configuration/hardening.md

@@ -17,6 +17,16 @@ This sets the Akkoma application server to only listen to the localhost interfac
 
 This sets the `secure` flag on Akkoma’s session cookie. This makes sure, that the cookie is only accepted over encrypted HTTPs connections. This implicitly renames the cookie from `pleroma_key` to `__Host-pleroma-key` which enforces some restrictions. (see [cookie prefixes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#Cookie_prefixes))
 
+### `Pleroma.Upload, :uploader, :base_url`
+
+> Recommended value: *anything on a different domain than the instance endpoint; e.g. https://media.myinstance.net/*
+
+Uploads are user controlled and (unless you’re running a true single-user
+instance) should therefore not be considered trusted. But the domain is used
+as a pivilege boundary e.g. by HTTP content security policy and ActivityPub.
+Having uploads on the same domain enabled several past vulnerabilities
+able to be exploited by malicious users.
+
 ### `:http_security`
 
 > Recommended value: `true`

docs/docs/configuration/howto_mediaproxy.md

@@ -6,7 +6,16 @@ With the `mediaproxy` function you can use nginx to cache this content, so users
 
 ## Activate it
 
-* Edit your nginx config and add the following location: 
+* Edit your nginx config and add the following location to your main server block:
+```
+location /proxy {
+        return 404;
+}
+```
+
+* Set up a subdomain for the proxy with its nginx config on the same machine
+  *(the latter is not strictly required, but for simplicity we’ll assume so)*
+* In this subdomain’s server block add
 ```
 location /proxy {
         proxy_cache akkoma_media_cache;
@@ -26,9 +35,9 @@ config :pleroma, :media_proxy,
       enabled: true,
       proxy_opts: [
             redirect_on_failure: true
-      ]
+      ],
-      #base_url: "https://cache.akkoma.social"
+      base_url: "https://cache.akkoma.social"
 ```
-If you want to use a subdomain to serve the files, uncomment `base_url`, change the url and add a comma after `true` in the previous line.
+You **really** should use a subdomain to serve proxied files; while we will fix bugs resulting from this, serving arbitrary remote content on your main domain namespace is a significant attack surface.
 
 * Restart nginx and Akkoma

installation/nginx/akkoma.nginx

@@ -75,9 +75,48 @@ server {
     proxy_set_header Host $http_host;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
+    location ~ ^/(media|proxy) {
+        return 404;
+    }
+
     location / {
         proxy_pass http://phoenix;
     }
+}
+
+# Upload and MediaProxy Subdomain
+# (see main domain setup for more details)
+server {
+    server_name    media.example.tld;
+
+    listen         80;
+    listen         [::]:80;
+
+    location / {
+      return         301 https://$server_name$request_uri;
+    }
+}
+
+server {
+    server_name media.example.tld;
+
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+
+    ssl_trusted_certificate   /etc/letsencrypt/live/media.example.tld/chain.pem;
+    ssl_certificate           /etc/letsencrypt/live/media.example.tld/fullchain.pem;
+    ssl_certificate_key       /etc/letsencrypt/live/media.example.tld/privkey.pem;
+    # .. copy all other the ssl_* and gzip_* stuff from main domain
+
+    # the nginx default is 1m, not enough for large media uploads
+    client_max_body_size 16m;
+    ignore_invalid_headers off;
+
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
     location ~ ^/(media|proxy) {
         proxy_cache        akkoma_media_cache;
@@ -91,4 +130,8 @@ server {
         chunked_transfer_encoding on;
         proxy_pass         http://phoenix;
     }
+
+    location / {
+        return 404;
+    }
 }

lib/mix/tasks/pleroma/instance.ex

@@ -20,6 +20,7 @@ def run(["gen" | rest]) do
           output: :string,
           output_psql: :string,
           domain: :string,
+          media_url: :string,
           instance_name: :string,
           admin_email: :string,
           notify_email: :string,
@@ -35,8 +36,7 @@ def run(["gen" | rest]) do
           listen_ip: :string,
           listen_port: :string,
           strip_uploads: :string,
-          anonymize_uploads: :string,
+          anonymize_uploads: :string
-          dedupe_uploads: :string
         ],
         aliases: [
           o: :output,
@@ -64,6 +64,14 @@ def run(["gen" | rest]) do
           ":"
         ) ++ [443]
 
+      media_url =
+        get_option(
+          options,
+          :media_url,
+          "What base url will uploads use? (e.g https://media.example.com/media)\n" <>
+            "  Generally this should NOT use the same domain as the instance       "
+        )
+
       name =
         get_option(
           options,
@@ -186,14 +194,6 @@ def run(["gen" | rest]) do
           "n"
         ) === "y"
 
-      dedupe_uploads =
-        get_option(
-          options,
-          :dedupe_uploads,
-          "Do you want to deduplicate uploaded files? (y/n)",
-          "n"
-        ) === "y"
-
       Config.put([:instance, :static_dir], static_dir)
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@@ -207,6 +207,7 @@ def run(["gen" | rest]) do
         EEx.eval_file(
           template_dir <> "/sample_config.eex",
           domain: domain,
+          media_url: media_url,
           port: port,
           email: email,
           notify_email: notify_email,
@@ -230,8 +231,7 @@ def run(["gen" | rest]) do
           upload_filters:
             upload_filters(%{
               strip: strip_uploads,
-              anonymize: anonymize_uploads,
+              anonymize: anonymize_uploads
-              dedupe: dedupe_uploads
             })
         )
 
@@ -319,13 +319,6 @@ defp upload_filters(filters) when is_map(filters) do
         enabled_filters
       end
 
-    enabled_filters =
-      if filters.dedupe do
-        enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
-      else
-        enabled_filters
-      end
-
     enabled_filters
   end
 end

lib/mix/tasks/pleroma/security.ex

@@ -0,0 +1,330 @@
+# Akkoma: Magically expressive social media
+# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Security do
+  use Mix.Task
+  import Ecto.Query
+  import Mix.Pleroma
+
+  alias Pleroma.Config
+
+  require Logger
+
+  @shortdoc """
+  Security-related tasks, like e.g. checking for signs past exploits were abused.
+  """
+
+  # Constants etc
+  defp local_id_prefix(), do: Pleroma.Web.Endpoint.url() <> "/"
+
+  defp local_id_pattern(), do: local_id_prefix() <> "%"
+
+  @activity_exts ["activity+json", "activity%2Bjson"]
+
+  defp activity_ext_url_patterns() do
+    for e <- @activity_exts do
+      for suf <- ["", "?%"] do
+        # Escape literal % for use in SQL patterns
+        ee = String.replace(e, "%", "\\%")
+        "%.#{ee}#{suf}"
+      end
+    end
+    |> List.flatten()
+  end
+
+  # Search for malicious uploads exploiting the lack of Content-Type sanitisation from before 2024-03
+  def run(["spoof-uploaded"]) do
+    Logger.put_process_level(self(), :notice)
+    start_pleroma()
+
+    IO.puts("""
+    +------------------------+
+    |  SPOOF SEARCH UPLOADS  |
+    +------------------------+
+    Checking if any uploads are using privileged types.
+    NOTE if attachment deletion is enabled, payloads used
+         in the past may no longer exist.
+    """)
+
+    do_spoof_uploaded()
+  end
+
+  # Fuzzy search for potentially counterfeit activities in the database resulting from the same exploit
+  def run(["spoof-inserted"]) do
+    Logger.put_process_level(self(), :notice)
+    start_pleroma()
+
+    IO.puts("""
+    +----------------------+
+    |  SPOOF SEARCH NOTES  |
+    +----------------------+
+    Starting fuzzy search for counterfeit activities.
+    NOTE this can not guarantee detecting all counterfeits
+         and may yield a small percentage of false positives.
+    """)
+
+    do_spoof_inserted()
+  end
+
+  # +-----------------------------+
+  # | S P O O F - U P L O A D E D |
+  # +-----------------------------+
+  defp do_spoof_uploaded() do
+    files =
+      case Config.get!([Pleroma.Upload, :uploader]) do
+        Pleroma.Uploaders.Local ->
+          uploads_search_spoofs_local_dir(Config.get!([Pleroma.Uploaders.Local, :uploads]))
+
+        _ ->
+          IO.puts("""
+          NOTE:
+            Not using local uploader; thus not affected by this exploit.
+            It's impossible to check for files, but in case local uploader was used before
+            or to check if anyone futilely attempted a spoof, notes will still be scanned.
+          """)
+
+          []
+      end
+
+    emoji = uploads_search_spoofs_local_dir(Config.get!([:instance, :static_dir]))
+
+    post_attachs = uploads_search_spoofs_notes()
+
+    not_orphaned_urls =
+      post_attachs
+      |> Enum.map(fn {_u, _a, url} -> url end)
+      |> MapSet.new()
+
+    orphaned_attachs = upload_search_orphaned_attachments(not_orphaned_urls)
+
+    IO.puts("\nSearch concluded; here are the results:")
+    pretty_print_list_with_title(emoji, "Emoji")
+    pretty_print_list_with_title(files, "Uploaded Files")
+    pretty_print_list_with_title(post_attachs, "(Not Deleted) Post Attachments")
+    pretty_print_list_with_title(orphaned_attachs, "Orphaned Uploads")
+
+    IO.puts("""
+    In total found
+      #{length(emoji)} emoji
+      #{length(files)} uploads
+      #{length(post_attachs)} not deleted posts
+      #{length(orphaned_attachs)} orphaned attachments
+    """)
+  end
+
+  defp uploads_search_spoofs_local_dir(dir) do
+    local_dir = String.replace_suffix(dir, "/", "")
+
+    IO.puts("Searching for suspicious files in #{local_dir}...")
+
+    glob_ext = "{" <> Enum.join(@activity_exts, ",") <> "}"
+
+    Path.wildcard(local_dir <> "/**/*." <> glob_ext, match_dot: true)
+    |> Enum.map(fn path ->
+      String.replace_prefix(path, local_dir <> "/", "")
+    end)
+    |> Enum.sort()
+  end
+
+  defp uploads_search_spoofs_notes() do
+    IO.puts("Now querying DB for posts with spoofing attachments. This might take a while...")
+
+    patterns = [local_id_pattern() | activity_ext_url_patterns()]
+
+    # if jsonb_array_elemsts in FROM can be used with normal Ecto functions, idk how
+    """
+    SELECT DISTINCT a.data->>'actor', a.id, url->>'href'
+    FROM public.objects AS o JOIN public.activities AS a
+         ON o.data->>'id' = a.data->>'object',
+       jsonb_array_elements(o.data->'attachment') AS attachs,
+       jsonb_array_elements(attachs->'url') AS url
+    WHERE o.data->>'type' = 'Note' AND
+          o.data->>'id' LIKE $1::text AND (
+            url->>'href' LIKE $2::text OR
+            url->>'href' LIKE $3::text OR
+            url->>'href' LIKE $4::text OR
+            url->>'href' LIKE $5::text
+          )
+    ORDER BY a.data->>'actor', a.id, url->>'href';
+    """
+    |> Pleroma.Repo.query!(patterns, timeout: :infinity)
+    |> map_raw_id_apid_tuple()
+  end
+
+  defp upload_search_orphaned_attachments(not_orphaned_urls) do
+    IO.puts("""
+    Now querying DB for orphaned spoofing attachment (i.e. their post was deleted,
+    but if :cleanup_attachments was not enabled traces remain in the database)
+    This might take a bit...
+    """)
+
+    patterns = activity_ext_url_patterns()
+
+    """
+    SELECT DISTINCT attach.id, url->>'href'
+    FROM public.objects AS attach,
+         jsonb_array_elements(attach.data->'url') AS url
+    WHERE (attach.data->>'type' = 'Image' OR
+           attach.data->>'type' = 'Document')
+          AND (
+            url->>'href' LIKE $1::text OR
+            url->>'href' LIKE $2::text OR
+            url->>'href' LIKE $3::text OR
+            url->>'href' LIKE $4::text
+          )
+    ORDER BY attach.id, url->>'href';
+    """
+    |> Pleroma.Repo.query!(patterns, timeout: :infinity)
+    |> then(fn res -> Enum.map(res.rows, fn [id, url] -> {id, url} end) end)
+    |> Enum.filter(fn {_, url} -> !(url in not_orphaned_urls) end)
+  end
+
+  # +-----------------------------+
+  # | S P O O F - I N S E R T E D |
+  # +-----------------------------+
+  defp do_spoof_inserted() do
+    IO.puts("""
+    Searching for local posts whose Create activity has no ActivityPub id...
+      This is a pretty good indicator, but only for spoofs of local actors
+      and only if the spoofing happened after around late 2021.
+    """)
+
+    idless_create =
+      search_local_notes_without_create_id()
+      |> Enum.sort()
+
+    IO.puts("Done.\n")
+
+    IO.puts("""
+    Now trying to weed out other poorly hidden spoofs.
+    This can't detect all and may have some false positives.
+    """)
+
+    likely_spoofed_posts_set = MapSet.new(idless_create)
+
+    sus_pattern_posts =
+      search_sus_notes_by_id_patterns()
+      |> Enum.filter(fn r -> !(r in likely_spoofed_posts_set) end)
+
+    IO.puts("Done.\n")
+
+    IO.puts("""
+    Finally, searching for spoofed, local user accounts.
+    (It's impossible to detect spoofed remote users)
+    """)
+
+    spoofed_users = search_bogus_local_users()
+
+    pretty_print_list_with_title(sus_pattern_posts, "Maybe Spoofed Posts")
+    pretty_print_list_with_title(idless_create, "Likely Spoofed Posts")
+    pretty_print_list_with_title(spoofed_users, "Spoofed local user accounts")
+
+    IO.puts("""
+    In total found:
+      #{length(spoofed_users)} bogus users
+      #{length(idless_create)} likely spoofed posts
+      #{length(sus_pattern_posts)} maybe spoofed posts
+    """)
+  end
+
+  defp search_local_notes_without_create_id() do
+    Pleroma.Object
+    |> where([o], fragment("?->>'id' LIKE ?", o.data, ^local_id_pattern()))
+    |> join(:inner, [o], a in Pleroma.Activity,
+      on: fragment("?->>'object' = ?->>'id'", a.data, o.data)
+    )
+    |> where([o, a], fragment("NOT (? \\? 'id') OR ?->>'id' IS NULL", a.data, a.data))
+    |> select([o, a], {a.id, fragment("?->>'id'", o.data)})
+    |> order_by([o, a], a.id)
+    |> Pleroma.Repo.all()
+  end
+
+  defp search_sus_notes_by_id_patterns() do
+    [ep1, ep2, ep3, ep4] = activity_ext_url_patterns()
+
+    Pleroma.Object
+    |> where(
+      [o],
+      # for local objects we know exactly how a genuine id looks like
+      # (though a thorough attacker can emulate this)
+      # for remote posts, use some best-effort patterns
+      fragment(
+        """
+         (?->>'id' LIKE ? AND ?->>'id' NOT SIMILAR TO
+          ? || 'objects/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}')
+        """,
+        o.data,
+        ^local_id_pattern(),
+        o.data,
+        ^local_id_prefix()
+      ) or
+        fragment("?->>'id' LIKE ?", o.data, "%/emoji/%") or
+        fragment("?->>'id' LIKE ?", o.data, "%/media/%") or
+        fragment("?->>'id' LIKE ?", o.data, "%/proxy/%") or
+        fragment("?->>'id' LIKE ?", o.data, ^ep1) or
+        fragment("?->>'id' LIKE ?", o.data, ^ep2) or
+        fragment("?->>'id' LIKE ?", o.data, ^ep3) or
+        fragment("?->>'id' LIKE ?", o.data, ^ep4)
+    )
+    |> join(:inner, [o], a in Pleroma.Activity,
+      on: fragment("?->>'object' = ?->>'id'", a.data, o.data)
+    )
+    |> select([o, a], {a.id, fragment("?->>'id'", o.data)})
+    |> order_by([o, a], a.id)
+    |> Pleroma.Repo.all()
+  end
+
+  defp search_bogus_local_users() do
+    Pleroma.User.Query.build(%{})
+    |> where([u], u.local == false and like(u.ap_id, ^local_id_pattern()))
+    |> order_by([u], u.ap_id)
+    |> select([u], u.ap_id)
+    |> Pleroma.Repo.all()
+  end
+
+  # +-----------------------------------+
+  # | module-specific utility functions |
+  # +-----------------------------------+
+  defp pretty_print_list_with_title(list, title) do
+    title_len = String.length(title)
+    title_underline = String.duplicate("=", title_len)
+    IO.puts(title)
+    IO.puts(title_underline)
+    pretty_print_list(list)
+  end
+
+  defp pretty_print_list([]), do: IO.puts("")
+
+  defp pretty_print_list([{a, o} | rest])
+       when (is_binary(a) or is_number(a)) and is_binary(o) do
+    IO.puts("  {#{a}, #{o}}")
+    pretty_print_list(rest)
+  end
+
+  defp pretty_print_list([{u, a, o} | rest])
+       when is_binary(a) and is_binary(u) and is_binary(o) do
+    IO.puts("  {#{u}, #{a}, #{o}}")
+    pretty_print_list(rest)
+  end
+
+  defp pretty_print_list([e | rest]) when is_binary(e) do
+    IO.puts("  #{e}")
+    pretty_print_list(rest)
+  end
+
+  defp pretty_print_list([e | rest]), do: pretty_print_list([inspect(e) | rest])
+
+  defp map_raw_id_apid_tuple(res) do
+    user_prefix = local_id_prefix() <> "users/"
+
+    Enum.map(res.rows, fn
+      [uid, aid, oid] ->
+        {
+          String.replace_prefix(uid, user_prefix, ""),
+          FlakeId.to_string(aid),
+          oid
+        }
+    end)
+  end
+end

lib/pleroma/emoji/pack.ex

@@ -26,12 +26,37 @@ defmodule Pleroma.Emoji.Pack do
   alias Pleroma.Emoji.Pack
   alias Pleroma.Utils
 
+  # Invalid/Malicious names are supposed to be filtered out before path joining,
+  # but there are many entrypoints to affected functions so as the code changes
+  # we might accidentally let an unsanitised name slip through.
+  # To make sure, use the below which crash the process otherwise.
+
+  # ALWAYS use this when constructing paths from external name!
+  # (name meaning it must be only a single path component)
+  defp path_join_name_safe(dir, name) do
+    if to_string(name) != Path.basename(name) or name in ["..", ".", ""] do
+      raise "Invalid or malicious pack name: #{name}"
+    else
+      Path.join(dir, name)
+    end
+  end
+
+  # ALWAYS use this to join external paths
+  # (which are allowed to have several components)
+  defp path_join_safe(dir, path) do
+    {:ok, safe_path} = Path.safe_relative(path)
+    Path.join(dir, safe_path)
+  end
+
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
-         dir <- Path.join(emoji_path(), name),
+         dir <- path_join_name_safe(emoji_path(), name),
          :ok <- File.mkdir(dir) do
-      save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
+      save_pack(%__MODULE__{
+        path: dir,
+        pack_file: Path.join(dir, "pack.json")
+      })
     end
   end
 
@@ -65,7 +90,7 @@ def show(opts) do
           {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
   def delete(name) do
     with :ok <- validate_not_empty([name]),
-         pack_path <- Path.join(emoji_path(), name) do
+         pack_path <- path_join_name_safe(emoji_path(), name) do
       File.rm_rf(pack_path)
     end
   end
@@ -89,7 +114,7 @@ defp unpack_zip_emojies(zip_files) do
     end)
   end
 
-  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
+  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t() | binary()) ::
           {:ok, t()}
           | {:error, File.posix() | atom()}
   def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
@@ -107,7 +132,7 @@ def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"}
           Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
             emoji_file = %Plug.Upload{
               filename: item[:filename],
-              path: Path.join(tmp_dir, item[:path])
+              path: path_join_safe(tmp_dir, item[:path])
             }
 
             {:ok, updated_pack} =
@@ -137,6 +162,14 @@ def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"}
   end
 
   def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
+    try_add_file(pack, shortcode, filename, file)
+  end
+
+  def add_file(%Pack{} = pack, shortcode, filename, filedata) when is_binary(filedata) do
+    try_add_file(pack, shortcode, filename, filedata)
+  end
+
+  defp try_add_file(%Pack{} = pack, shortcode, filename, file) do
     with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
          {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
@@ -189,6 +222,7 @@ def import_from_filesystem do
          {:ok, results} <- File.ls(emoji_path) do
       names =
         results
+        # items come from File.ls, thus safe
         |> Enum.map(&Path.join(emoji_path, &1))
         |> Enum.reject(fn path ->
           File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
@@ -287,8 +321,8 @@ def update_metadata(name, data) do
 
   @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
   def load_pack(name) do
-    name = Path.basename(name)
+    pack_dir = path_join_name_safe(emoji_path(), name)
-    pack_file = Path.join([emoji_path(), name, "pack.json"])
+    pack_file = Path.join(pack_dir, "pack.json")
 
     with {:ok, _} <- File.stat(pack_file),
          {:ok, pack_data} <- File.read(pack_file) do
@@ -412,7 +446,13 @@ defp downloadable?(pack) do
   end
 
   defp create_archive_and_cache(pack, hash) do
-    files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
+    files = [
+      ~c"pack.json"
+      | Enum.map(pack.files, fn {_, file} ->
+          {:ok, file} = Path.safe_relative(file)
+          to_charlist(file)
+        end)
+    ]
 
     {:ok, {_, result}} =
       :zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)])
@@ -474,7 +514,7 @@ defp validate_not_empty(list) do
   end
 
   defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
-    file_path = Path.join(pack.path, filename)
+    file_path = path_join_safe(pack.path, filename)
     create_subdirs(file_path)
 
     with {:ok, _} <- File.copy(upload_path, file_path) do
@@ -482,6 +522,12 @@ defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
     end
   end
 
+  defp save_file(file_data, pack, filename) when is_binary(file_data) do
+    file_path = path_join_safe(pack.path, filename)
+    create_subdirs(file_path)
+    File.write(file_path, file_data, [:binary])
+  end
+
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
     %{pack | files: files, files_count: length(Map.keys(files))}
@@ -493,8 +539,8 @@ defp delete_emoji(pack, shortcode) do
   end
 
   defp rename_file(pack, filename, new_filename) do
-    old_path = Path.join(pack.path, filename)
+    old_path = path_join_safe(pack.path, filename)
-    new_path = Path.join(pack.path, new_filename)
+    new_path = path_join_safe(pack.path, new_filename)
     create_subdirs(new_path)
 
     with :ok <- File.rename(old_path, new_path) do
@@ -512,7 +558,7 @@ defp create_subdirs(file_path) do
 
   defp remove_file(pack, shortcode) do
     with {:ok, filename} <- get_filename(pack, shortcode),
-         emoji <- Path.join(pack.path, filename),
+         emoji <- path_join_safe(pack.path, filename),
          :ok <- File.rm(emoji) do
       remove_dir_if_empty(emoji, filename)
     end
@@ -530,7 +576,7 @@ defp remove_dir_if_empty(emoji, filename) do
 
   defp get_filename(pack, shortcode) do
     with %{^shortcode => filename} when is_binary(filename) <- pack.files,
-         file_path <- Path.join(pack.path, filename),
+         file_path <- path_join_safe(pack.path, filename),
          {:ok, _} <- File.stat(file_path) do
       {:ok, filename}
     else
@@ -568,7 +614,7 @@ defp validate_downloadable(pack) do
   end
 
   defp copy_as(remote_pack, local_name) do
-    path = Path.join(emoji_path(), local_name)
+    path = path_join_name_safe(emoji_path(), local_name)
 
     %__MODULE__{
       name: local_name,

lib/pleroma/object/containment.ex

@@ -11,6 +11,9 @@ defmodule Pleroma.Object.Containment do
   Object containment is an important step in validating remote objects to prevent
   spoofing, therefore removal of object containment functions is NOT recommended.
   """
+
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
   def get_actor(%{"actor" => actor}) when is_binary(actor) do
     actor
   end
@@ -47,6 +50,31 @@ def get_object(_) do
   defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
   defp compare_uris(_id_uri, _other_uri), do: :error
 
+  defp compare_uris_exact(uri, uri), do: :ok
+
+  defp compare_uris_exact(%URI{} = id, %URI{} = other),
+    do: compare_uris_exact(URI.to_string(id), URI.to_string(other))
+
+  defp compare_uris_exact(id_uri, other_uri)
+       when is_binary(id_uri) and is_binary(other_uri) do
+    norm_id = String.replace_suffix(id_uri, "/", "")
+    norm_other = String.replace_suffix(other_uri, "/", "")
+    if norm_id == norm_other, do: :ok, else: :error
+  end
+
+  @doc """
+  Checks whether an URL to fetch from is from the local server.
+
+  We never want to fetch from ourselves; if it’s not in the database
+  it can’t be authentic and must be a counterfeit.
+  """
+  def contain_local_fetch(id) do
+    case compare_uris(URI.parse(id), Pleroma.Web.Endpoint.struct_url()) do
+      :ok -> :error
+      _ -> :ok
+    end
+  end
+
   @doc """
   Checks that an imported AP object's actor matches the host it came from.
   """
@@ -62,8 +90,31 @@ def contain_origin(id, %{"actor" => _actor} = params) do
   def contain_origin(id, %{"attributedTo" => actor} = params),
     do: contain_origin(id, Map.put(params, "actor", actor))
 
-  def contain_origin(_id, _data), do: :error
+  def contain_origin(_id, _data), do: :ok
 
+  @doc """
+  Check whether the fetch URL (after redirects) exactly (sans tralining slash) matches either
+  the canonical ActivityPub id or the objects url field (for display URLs from *key and Mastodon)
+
+  Since this is meant to be used for fetches, anonymous or transient objects are not accepted here.
+  """
+  def contain_id_to_fetch(url, %{"id" => id} = data) when is_binary(id) do
+    with {:id, :error} <- {:id, compare_uris_exact(id, url)},
+         # "url" can be a "Link" object and this is checked before full normalisation
+         display_url <- Transmogrifier.fix_url(data)["url"],
+         true <- display_url != nil do
+      compare_uris_exact(display_url, url)
+    else
+      {:id, :ok} -> :ok
+      _ -> :error
+    end
+  end
+
+  def contain_id_to_fetch(_url, _data), do: :error
+
+  @doc """
+  Check whether the object id is from the same host as another id
+  """
   def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(other_id) do
     id_uri = URI.parse(id)
     other_uri = URI.parse(other_id)
@@ -85,4 +136,12 @@ def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
     do: contain_origin(id, object)
 
   def contain_child(_), do: :ok
+
+  @doc "Checks whether two URIs belong to the same domain"
+  def same_origin(id1, id2) do
+    uri1 = URI.parse(id1)
+    uri2 = URI.parse(id2)
+
+    compare_uris(uri1, uri2)
+  end
 end

lib/pleroma/object/fetcher.ex

@@ -18,6 +18,16 @@ defmodule Pleroma.Object.Fetcher do
   require Logger
   require Pleroma.Constants
 
+  @moduledoc """
+  This module deals with correctly fetching Acitivity Pub objects in a safe way.
+
+  The core function is `fetch_and_contain_remote_object_from_id/1` which performs
+  the actual fetch and common safety and authenticity checks. Other `fetch_*`
+  function use the former and perform some additional tasks
+  """
+
+  @mix_env Mix.env()
+
   defp touch_changeset(changeset) do
     updated_at =
       NaiveDateTime.utc_now()
@@ -103,18 +113,26 @@ defp reinject_object(%Object{} = object, new_data) do
     end
   end
 
+  @doc "Assumes object already is in our database and refetches from remote to update (e.g. for polls)"
   def refetch_object(%Object{data: %{"id" => id}} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
          {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
+         {:id, true} <- {:id, new_data["id"] == id},
          {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
       {:local, true} -> {:ok, object}
+      {:id, false} -> {:error, "Object id changed on refetch"}
       e -> {:error, e}
     end
   end
 
-  # Note: will create a Create activity, which we need internally at the moment.
+  @doc """
+    Fetches a new object and puts it through the processing pipeline for inbound objects
+
+    Note: will also insert a fake Create activity, since atm we internally
+    need everything to be traced back to a Create activity.
+  """
   def fetch_object_from_id(id, options \\ []) do
     with %URI{} = uri <- URI.parse(id),
          # let's check the URI is even vaguely valid first
@@ -127,7 +145,6 @@ def fetch_object_from_id(id, options \\ []) do
          {_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
          {_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
          params <- prepare_activity_params(data),
-         {_, :ok} <- {:containment, Containment.contain_origin(id, params)},
          {_, {:ok, activity}} <-
            {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
          {_, _data, %Object{} = object} <-
@@ -140,9 +157,6 @@ def fetch_object_from_id(id, options \\ []) do
       {:scheme, false} ->
         {:error, "URI Scheme Invalid"}
 
-      {:containment, _} ->
-        {:error, "Object containment failed."}
-
       {:transmogrifier, {:error, {:reject, e}}} ->
         {:reject, e}
 
@@ -185,6 +199,7 @@ defp prepare_activity_params(data) do
     |> Maps.put_if_present("bcc", data["bcc"])
   end
 
+  @doc "Identical to `fetch_object_from_id/2` but just directly returns the object or on error `nil`"
   def fetch_object_from_id!(id, options \\ []) do
     with {:ok, object} <- fetch_object_from_id(id, options) do
       object
@@ -235,6 +250,7 @@ defp maybe_date_fetch(headers, date) do
     end
   end
 
+  @doc "Fetches arbitrary remote object and performs basic safety and authenticity checks"
   def fetch_and_contain_remote_object_from_id(id)
 
   def fetch_and_contain_remote_object_from_id(%{"id" => id}),
@@ -244,18 +260,29 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, body} <- get_object(id),
+         {_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
+         {:ok, final_id, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
-         :ok <- Containment.contain_origin_from_id(id, data) do
+         {_, :ok} <- {:strict_id, Containment.contain_id_to_fetch(final_id, data)},
-      unless Instances.reachable?(id) do
+         {_, :ok} <- {:containment, Containment.contain_origin(final_id, data)} do
-        Instances.set_reachable(id)
+      unless Instances.reachable?(final_id) do
+        Instances.set_reachable(final_id)
       end
 
       {:ok, data}
     else
+      {:strict_id, _} ->
+        {:error, "Object's ActivityPub id/url does not match final fetch URL"}
+
       {:scheme, _} ->
         {:error, "Unsupported URI scheme"}
 
+      {:local_fetch, _} ->
+        {:error, "Trying to fetch local resource"}
+
+      {:containment, _} ->
+        {:error, "Object containment failed."}
+
       {:error, e} ->
         {:error, e}
 
@@ -267,6 +294,32 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
   def fetch_and_contain_remote_object_from_id(_id),
     do: {:error, "id must be a string"}
 
+  defp check_crossdomain_redirect(final_host, original_url)
+
+  # HOPEFULLY TEMPORARY
+  # Basically none of our Tesla mocks in tests set the (supposed to
+  # exist for Tesla proper) url parameter for their responses
+  # causing almost every fetch in test to fail otherwise
+  if @mix_env == :test do
+    defp check_crossdomain_redirect(nil, _) do
+      {:cross_domain_redirect, false}
+    end
+  end
+
+  defp check_crossdomain_redirect(final_host, original_url) do
+    {:cross_domain_redirect, final_host != URI.parse(original_url).host}
+  end
+
+  if @mix_env == :test do
+    defp get_final_id(nil, initial_url), do: initial_url
+    defp get_final_id("", initial_url), do: initial_url
+  end
+
+  defp get_final_id(final_url, _intial_url) do
+    final_url
+  end
+
+  @doc "Do NOT use; only public for use in tests"
   def get_object(id) do
     date = Pleroma.Signature.signed_date()
 
@@ -275,37 +328,42 @@ def get_object(id) do
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 
-    case HTTP.get(id, headers) do
+    with {:ok, %{body: body, status: code, headers: headers, url: final_url}}
-      {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
+         when code in 200..299 <-
-        case List.keyfind(headers, "content-type", 0) do
+           HTTP.get(id, headers),
-          {_, content_type} ->
+         remote_host <-
-            case Plug.Conn.Utils.media_type(content_type) do
+           URI.parse(final_url).host,
-              {:ok, "application", "activity+json", _} ->
+         {:cross_domain_redirect, false} <-
-                {:ok, body}
+           check_crossdomain_redirect(remote_host, id),
+         {:has_content_type, {_, content_type}} <-
+           {:has_content_type, List.keyfind(headers, "content-type", 0)},
+         {:parse_content_type, {:ok, "application", subtype, type_params}} <-
+           {:parse_content_type, Plug.Conn.Utils.media_type(content_type)} do
+      final_id = get_final_id(final_url, id)
 
-              {:ok, "application", "ld+json",
+      case {subtype, type_params} do
-               %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
+        {"activity+json", _} ->
-                {:ok, body}
+          {:ok, final_id, body}
 
-              # pixelfed sometimes (and only sometimes) responds with http instead of https
+        {"ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
-              {:ok, "application", "ld+json",
+          {:ok, final_id, body}
-               %{"profile" => "http://www.w3.org/ns/activitystreams"}} ->
-                {:ok, body}
-
-              _ ->
-                {:error, {:content_type, content_type}}
-            end
-
-          _ ->
-            {:error, {:content_type, nil}}
-        end
 
+        _ ->
+          {:error, {:content_type, content_type}}
+      end
+    else
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, {"Object has been deleted", id, code}}
 
       {:error, e} ->
         {:error, e}
 
+      {:has_content_type, _} ->
+        {:error, {:content_type, nil}}
+
+      {:parse_content_type, e} ->
+        {:error, {:content_type, e}}
+
       e ->
         {:error, e}
     end

lib/pleroma/reverse_proxy.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.ReverseProxy do
   @failed_request_ttl :timer.seconds(60)
   @methods ~w(GET HEAD)
 
+  @allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], [])
+
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
 
   def max_read_duration_default, do: @max_read_duration
@@ -253,6 +255,7 @@ defp build_resp_headers(headers, opts) do
     headers
     |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
     |> build_resp_cache_headers(opts)
+    |> sanitise_content_type()
     |> build_resp_content_disposition_header(opts)
     |> build_csp_headers()
     |> Keyword.merge(Keyword.get(opts, :resp_headers, []))
@@ -282,6 +285,21 @@ defp build_resp_cache_headers(headers, _opts) do
     end
   end
 
+  defp sanitise_content_type(headers) do
+    original_ct = get_content_type(headers)
+
+    safe_ct =
+      Pleroma.Web.Plugs.Utils.get_safe_mime_type(
+        %{allowed_mime_types: @allowed_mime_types},
+        original_ct
+      )
+
+    [
+      {"content-type", safe_ct}
+      | Enum.filter(headers, fn {k, _v} -> k != "content-type" end)
+    ]
+  end
+
   defp build_resp_content_disposition_header(headers, opts) do
     opt = Keyword.get(opts, :inline_content_types, @inline_content_types)
 

lib/pleroma/upload.ex

@@ -39,6 +39,8 @@ defmodule Pleroma.Upload do
   alias Pleroma.Web.ActivityPub.Utils
   require Logger
 
+  @mix_env Mix.env()
+
   @type source ::
           Plug.Upload.t()
           | (data_uri_string :: String.t())
@@ -64,7 +66,7 @@ defmodule Pleroma.Upload do
           path: String.t()
         }
 
-  @always_enabled_filters [Pleroma.Upload.Filter.AnonymizeFilename]
+  @always_enabled_filters [Pleroma.Upload.Filter.Dedupe]
 
   defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
 
@@ -228,6 +230,13 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
 
   defp url_from_spec(_upload, _base_url, {:url, url}), do: url
 
+  if @mix_env == :test do
+    defp choose_base_url(prim, sec \\ nil),
+      do: prim || sec || Pleroma.Web.Endpoint.url() <> "/media/"
+  else
+    defp choose_base_url(prim, sec \\ nil), do: prim || sec
+  end
+
   def base_url do
     uploader = Config.get([Pleroma.Upload, :uploader])
     upload_base_url = Config.get([Pleroma.Upload, :base_url])
@@ -235,7 +244,7 @@ def base_url do
 
     case uploader do
       Pleroma.Uploaders.Local ->
-        upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
+        choose_base_url(upload_base_url)
 
       Pleroma.Uploaders.S3 ->
         bucket = Config.get([Pleroma.Uploaders.S3, :bucket])
@@ -261,7 +270,7 @@ def base_url do
         end
 
       _ ->
-        public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
+        choose_base_url(public_endpoint, upload_base_url)
     end
   end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -22,6 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Upload
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
+  alias Pleroma.Web.ActivityPub.ObjectValidators.UserValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Streamer
   alias Pleroma.Web.WebFinger
@@ -1722,6 +1723,7 @@ def user_data_from_user_object(data, additional \\ []) do
 
   def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+         {:valid, {:ok, _, _}} <- {:valid, UserValidator.validate(data, [])},
          {:ok, data} <- user_data_from_user_object(data, additional) do
       {:ok, maybe_update_follow_information(data)}
     else
@@ -1734,6 +1736,10 @@ def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
         Logger.debug("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
 
+      {:valid, reason} ->
+        Logger.debug("Data is not a valid user #{ap_id}: #{inspect(reason)}")
+        {:error, "Not a user"}
+
       {:error, e} ->
         Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
@@ -1834,6 +1840,13 @@ def make_user_from_ap_id(ap_id, additional \\ []) do
       with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
         {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
 
+        user =
+          if data.ap_id != ap_id do
+            User.get_cached_by_ap_id(data.ap_id)
+          else
+            user
+          end
+
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -6,10 +6,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
   require Logger
 
   alias Pleroma.Config
+  alias Pleroma.Emoji.Pack
 
   @moduledoc "Detect new emojis by their shortcode and steals them"
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
+  @pack_name "stolen"
+
+  # Config defaults
+  @size_limit 50_000
+  @download_unknown_size false
+
+  defp create_pack() do
+    with {:ok, pack} = Pack.create(@pack_name) do
+      Pack.save_metadata(
+        %{
+          "description" => "Collection of emoji auto-stolen from other instances",
+          "homepage" => Pleroma.Web.Endpoint.url(),
+          "can-download" => false,
+          "share-files" => false
+        },
+        pack
+      )
+    end
+  end
+
+  defp load_or_create_pack() do
+    case Pack.load_pack(@pack_name) do
+      {:ok, pack} -> {:ok, pack}
+      {:error, :enoent} -> create_pack()
+      e -> e
+    end
+  end
+
+  defp add_emoji(shortcode, extension, filedata) do
+    {:ok, pack} = load_or_create_pack()
+    # Make final path infeasible to predict to thwart certain kinds of attacks
+    # (48 bits is slighty more than 8 base62 chars, thus 9 chars)
+    salt =
+      :crypto.strong_rand_bytes(6)
+      |> :crypto.bytes_to_integer()
+      |> Base62.encode()
+      |> String.pad_leading(9, "0")
+
+    filename = shortcode <> "-" <> salt <> "." <> extension
+
+    Pack.add_file(pack, shortcode, filename, filedata)
+  end
+
   defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
 
   defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do
@@ -20,31 +64,69 @@ defp shortcode_matches?(shortcode, pattern) do
     String.match?(shortcode, pattern)
   end
 
-  defp steal_emoji({shortcode, url}, emoji_dir_path) do
+  defp reject_emoji?({shortcode, _url}, installed_emoji) do
+    valid_shortcode? = String.match?(shortcode, ~r/^[a-zA-Z0-9_-]+$/)
+
+    rejected_shortcode? =
+      [:mrf_steal_emoji, :rejected_shortcodes]
+      |> Config.get([])
+      |> Enum.any?(fn pattern -> shortcode_matches?(shortcode, pattern) end)
+
+    emoji_installed? = Enum.member?(installed_emoji, shortcode)
+
+    !valid_shortcode? or rejected_shortcode? or emoji_installed?
+  end
+
+  defp steal_emoji(%{} = response, {shortcode, extension}) do
+    case add_emoji(shortcode, extension, response.body) do
+      {:ok, _} ->
+        shortcode
+
+      e ->
+        Logger.warning(
+          "MRF.StealEmojiPolicy: Failed to add #{shortcode} as #{extension}: #{inspect(e)}"
+        )
+
+        nil
+    end
+  end
+
+  defp get_extension_if_safe(response) do
+    content_type =
+      :proplists.get_value("content-type", response.headers, MIME.from_path(response.url))
+
+    case content_type do
+      "image/" <> _ -> List.first(MIME.extensions(content_type))
+      _ -> nil
+    end
+  end
+
+  defp is_remote_size_within_limit?(url) do
+    with {:ok, %{status: status, headers: headers} = _response} when status in 200..299 <-
+           Pleroma.HTTP.request(:head, url, nil, [], []) do
+      content_length = :proplists.get_value("content-length", headers, nil)
+      size_limit = Config.get([:mrf_steal_emoji, :size_limit], @size_limit)
+
+      accept_unknown =
+        Config.get([:mrf_steal_emoji, :download_unknown_size], @download_unknown_size)
+
+      content_length <= size_limit or
+        (content_length == nil and accept_unknown)
+    else
+      _ -> false
+    end
+  end
+
+  defp maybe_steal_emoji({shortcode, url}) do
     url = Pleroma.Web.MediaProxy.url(url)
 
-    with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do
+    with {:remote_size, true} <- {:remote_size, is_remote_size_within_limit?(url)},
-      size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
+         {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do
+      size_limit = Config.get([:mrf_steal_emoji, :size_limit], @size_limit)
+      extension = get_extension_if_safe(response)
 
-      if byte_size(response.body) <= size_limit do
+      if byte_size(response.body) <= size_limit and extension do
-        extension =
+        steal_emoji(response, {shortcode, extension})
-          url
-          |> URI.parse()
-          |> Map.get(:path)
-          |> Path.basename()
-          |> Path.extname()
-
-        shortcode = Path.basename(shortcode)
-        file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png"))
-
-        case File.write(file_path, response.body) do
-          :ok ->
-            shortcode
-
-          e ->
-            Logger.warning("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
-            nil
-        end
       else
         Logger.debug(
           "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{size_limit} B)"
@@ -66,29 +148,10 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa
     if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do
       installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
 
-      emoji_dir_path =
-        Config.get(
-          [:mrf_steal_emoji, :path],
-          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
-        )
-
-      File.mkdir_p(emoji_dir_path)
-
       new_emojis =
         foreign_emojis
-        |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end)
+        |> Enum.reject(&reject_emoji?(&1, installed_emoji))
-        |> Enum.reject(fn {shortcode, _url} ->
+        |> Enum.map(&maybe_steal_emoji(&1))
-          String.contains?(shortcode, ["/", "\\", ".", ":"])
-        end)
-        |> Enum.filter(fn {shortcode, _url} ->
-          reject_emoji? =
-            [:mrf_steal_emoji, :rejected_shortcodes]
-            |> Config.get([])
-            |> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end)
-
-          !reject_emoji?
-        end)
-        |> Enum.map(&steal_emoji(&1, emoji_dir_path))
         |> Enum.filter(& &1)
 
       if !Enum.empty?(new_emojis) do

lib/pleroma/web/activity_pub/object_validators/user_validator.ex

@@ -0,0 +1,92 @@
+# Akkoma: Magically expressive social media
+# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.UserValidator do
+  @moduledoc """
+  Checks whether ActivityPub data represents a valid user
+
+  Users don't go through the same ingest pipeline like activities or other objects.
+  To ensure this can only match a user and no users match in the other pipeline,
+  this is a separate from the generic ObjectValidator.
+  """
+
+  @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating
+
+  alias Pleroma.Object.Containment
+  alias Pleroma.Signature
+
+  @impl true
+  def validate(object, meta)
+
+  def validate(%{"type" => type, "id" => _id} = data, meta)
+      when type in ["Person", "Organization", "Group", "Application"] do
+    with :ok <- validate_pubkey(data),
+         :ok <- validate_inbox(data),
+         :ok <- contain_collection_origin(data) do
+      {:ok, data, meta}
+    else
+      {:error, e} -> {:error, e}
+      e -> {:error, e}
+    end
+  end
+
+  def validate(_, _), do: {:error, "Not a user object"}
+
+  defp mabye_validate_owner(nil, _actor), do: :ok
+  defp mabye_validate_owner(actor, actor), do: :ok
+  defp mabye_validate_owner(_owner, _actor), do: :error
+
+  defp validate_pubkey(
+         %{"id" => id, "publicKey" => %{"id" => pk_id, "publicKeyPem" => _key}} = data
+       )
+       when id != nil do
+    with {_, {:ok, kactor}} <- {:key, Signature.key_id_to_actor_id(pk_id)},
+         true <- id == kactor,
+         :ok <- mabye_validate_owner(Map.get(data, "owner"), id) do
+      :ok
+    else
+      {:key, _} ->
+        {:error, "Unable to determine actor id from key id"}
+
+      false ->
+        {:error, "Key id does not relate to user id"}
+
+      _ ->
+        {:error, "Actor does not own its public key"}
+    end
+  end
+
+  # pubkey is optional atm
+  defp validate_pubkey(_data), do: :ok
+
+  defp validate_inbox(%{"id" => id, "inbox" => inbox}) do
+    case Containment.same_origin(id, inbox) do
+      :ok -> :ok
+      :error -> {:error, "Inbox on different doamin"}
+    end
+  end
+
+  defp validate_inbox(_), do: {:error, "No inbox"}
+
+  defp check_field_value(%{"id" => id} = _data, value) do
+    Containment.same_origin(id, value)
+  end
+
+  defp maybe_check_field(data, field) do
+    with val when val != nil <- data[field],
+         :ok <- check_field_value(data, val) do
+      :ok
+    else
+      nil -> :ok
+      _ -> {:error, "#{field} on different domain"}
+    end
+  end
+
+  defp contain_collection_origin(data) do
+    Enum.reduce(["followers", "following", "featured"], :ok, fn
+      field, :ok -> maybe_check_field(data, field)
+      _, error -> error
+    end)
+  end
+end

lib/pleroma/web/endpoint.ex

@@ -98,6 +98,10 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only: Pleroma.Web.static_paths(),
+    # JSON-LD is accepted by some servers for AP objects and activities,
+    # thus only enable it here instead of a global extension mapping
+    # (it's our only *.jsonld file anyway)
+    content_types: %{"litepub-0.1.jsonld" => "application/ld+json"},
     # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
     gzip: true,
     cache_control_for_etags: @static_cache_control,

lib/pleroma/web/media_proxy.ex

@@ -14,6 +14,8 @@ defmodule Pleroma.Web.MediaProxy do
 
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
 
+  @mix_env Mix.env()
+
   def cache_table, do: @cache_table
 
   @spec in_banned_urls(String.t()) :: boolean()
@@ -144,8 +146,14 @@ def filename(url_or_path) do
     if path = URI.parse(url_or_path).path, do: Path.basename(path)
   end
 
-  def base_url do
+  if @mix_env == :test do
-    Config.get([:media_proxy, :base_url], Endpoint.url())
+    def base_url do
+      Config.get([:media_proxy, :base_url], Endpoint.url())
+    end
+  else
+    def base_url do
+      Config.get!([:media_proxy, :base_url])
+    end
   end
 
   defp proxy_url(path, sig_base64, url_base64, filename) do

lib/pleroma/web/plugs/instance_static.ex

@@ -3,8 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.InstanceStatic do
+  import Plug.Conn
+
   require Pleroma.Constants
 
+  alias Pleroma.Web.Plugs.Utils
+
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.
 
@@ -43,11 +47,25 @@ def call(conn, _) do
     conn
   end
 
-  defp call_static(conn, opts, from) do
+  defp set_static_content_type(conn, "/emoji/" <> _ = request_path) do
+    real_mime = MIME.from_path(request_path)
+    safe_mime = Utils.get_safe_mime_type(%{allowed_mime_types: ["image"]}, real_mime)
+
+    put_resp_header(conn, "content-type", safe_mime)
+  end
+
+  defp set_static_content_type(conn, request_path) do
+    put_resp_header(conn, "content-type", MIME.from_path(request_path))
+  end
+
+  defp call_static(%{request_path: request_path} = conn, opts, from) do
     opts =
       opts
       |> Map.put(:from, from)
+      |> Map.put(:set_content_type, false)
 
-    Plug.Static.call(conn, opts)
+    conn
+    |> set_static_content_type(request_path)
+    |> Pleroma.Web.Plugs.StaticNoCT.call(opts)
   end
 end

lib/pleroma/web/plugs/static_no_content_type.ex

@@ -0,0 +1,469 @@
+# This is almost identical to Plug.Static from Plug 1.15.3 (2024-01-16)
+# It being copied is a temporary measure to fix an urgent bug without
+# needing to wait for merge of a suitable patch upstream
+# The differences are:
+#  - this leading comment
+#  - renaming of the module from 'Plug.Static' to 'Pleroma.Web.Plugs.StaticNoCT'
+#  - additon of set_content_type option
+
+defmodule Pleroma.Web.Plugs.StaticNoCT do
+  @moduledoc """
+  A plug for serving static assets.
+
+  It requires two options:
+
+    * `:at` - the request path to reach for static assets.
+      It must be a string.
+
+    * `:from` - the file system path to read static assets from.
+      It can be either: a string containing a file system path, an
+      atom representing the application name (where assets will
+      be served from `priv/static`), a tuple containing the
+      application name and the directory to serve assets from (besides
+      `priv/static`), or an MFA tuple.
+
+  The preferred form is to use `:from` with an atom or tuple, since
+  it will make your application independent from the starting directory.
+  For example, if you pass:
+
+      plug Plug.Static, from: "priv/app/path"
+
+  Plug.Static will be unable to serve assets if you build releases
+  or if you change the current directory. Instead do:
+
+      plug Plug.Static, from: {:app_name, "priv/app/path"}
+
+  If a static asset cannot be found, `Plug.Static` simply forwards
+  the connection to the rest of the pipeline.
+
+  ## Cache mechanisms
+
+  `Plug.Static` uses etags for HTTP caching. This means browsers/clients
+  should cache assets on the first request and validate the cache on
+  following requests, not downloading the static asset once again if it
+  has not changed. The cache-control for etags is specified by the
+  `cache_control_for_etags` option and defaults to `"public"`.
+
+  However, `Plug.Static` also supports direct cache control by using
+  versioned query strings. If the request query string starts with
+  "?vsn=", `Plug.Static` assumes the application is versioning assets
+  and does not set the `ETag` header, meaning the cache behaviour will
+  be specified solely by the `cache_control_for_vsn_requests` config,
+  which defaults to `"public, max-age=31536000"`.
+
+  ## Options
+
+    * `:encodings` - list of 2-ary tuples where first value is value of
+      the `Accept-Encoding` header and second is extension of the file to
+      be served if given encoding is accepted by client. Entries will be tested
+      in order in list, so entries higher in list will be preferred. Defaults
+      to: `[]`.
+
+      In addition to setting this value directly it supports 2 additional
+      options for compatibility reasons:
+
+        + `:brotli` - will append `{"br", ".br"}` to the encodings list.
+        + `:gzip` - will append `{"gzip", ".gz"}` to the encodings list.
+
+      Additional options will be added in the above order (Brotli takes
+      preference over Gzip) to reflect older behaviour which was set due
+      to fact that Brotli in general provides better compression ratio than
+      Gzip.
+
+    * `:cache_control_for_etags` - sets the cache header for requests
+      that use etags. Defaults to `"public"`.
+
+    * `:etag_generation` - specify a `{module, function, args}` to be used
+      to generate   an etag. The `path` of the resource will be passed to
+      the function, as well as the `args`. If this option is not supplied,
+      etags will be generated based off of file size and modification time.
+      Note it is [recommended for the etag value to be quoted](https://tools.ietf.org/html/rfc7232#section-2.3),
+      which Plug won't do automatically.
+
+    * `:cache_control_for_vsn_requests` - sets the cache header for
+      requests starting with "?vsn=" in the query string. Defaults to
+      `"public, max-age=31536000"`.
+
+    * `:only` - filters which requests to serve. This is useful to avoid
+      file system access on every request when this plug is mounted
+      at `"/"`. For example, if `only: ["images", "favicon.ico"]` is
+      specified, only files in the "images" directory and the
+      "favicon.ico" file will be served by `Plug.Static`.
+      Note that `Plug.Static` matches these filters against request
+      uri and not against the filesystem. When requesting
+      a file with name containing non-ascii or special characters,
+      you should use urlencoded form. For example, you should write
+      `only: ["file%20name"]` instead of `only: ["file name"]`.
+      Defaults to `nil` (no filtering).
+
+    * `:only_matching` - a relaxed version of `:only` that will
+      serve any request as long as one of the given values matches the
+      given path. For example, `only_matching: ["images", "favicon"]`
+      will match any request that starts at "images" or "favicon",
+      be it "/images/foo.png", "/images-high/foo.png", "/favicon.ico"
+      or "/favicon-high.ico". Such matches are useful when serving
+      digested files at the root. Defaults to `nil` (no filtering).
+
+    * `:headers` - other headers to be set when serving static assets. Specify either
+      an enum of key-value pairs or a `{module, function, args}` to return an enum. The
+      `conn` will be passed to the function, as well as the `args`.
+
+    * `:content_types` - custom MIME type mapping. As a map with filename as key
+      and content type as value. For example:
+      `content_types: %{"apple-app-site-association" => "application/json"}`.
+
+    * `:set_content_type` - by default Plug.Static (re)sets the content type header
+      using auto-detection and the `:content_types` map. But when set to `false`
+      no content-type header will be inserted instead retaining the original
+      value or lack thereof. This can be useful when custom logic for appropiate
+      content types is needed which cannot be reasonably expressed as a static
+      filename map.
+
+  ## Examples
+
+  This plug can be mounted in a `Plug.Builder` pipeline as follows:
+
+      defmodule MyPlug do
+        use Plug.Builder
+
+        plug Plug.Static,
+          at: "/public",
+          from: :my_app,
+          only: ~w(images robots.txt)
+        plug :not_found
+
+        def not_found(conn, _) do
+          send_resp(conn, 404, "not found")
+        end
+      end
+
+  """
+
+  @behaviour Plug
+  @allowed_methods ~w(GET HEAD)
+
+  import Plug.Conn
+  alias Plug.Conn
+
+  # In this module, the `:prim_file` Erlang module along with the `:file_info`
+  # record are used instead of the more common and Elixir-y `File` module and
+  # `File.Stat` struct, respectively. The reason behind this is performance: all
+  # the `File` operations pass through a single process in order to support node
+  # operations that we simply don't need when serving assets.
+
+  require Record
+  Record.defrecordp(:file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl"))
+
+  defmodule InvalidPathError do
+    defexception message: "invalid path for static asset", plug_status: 400
+  end
+
+  @impl true
+  def init(opts) do
+    from =
+      case Keyword.fetch!(opts, :from) do
+        {_, _} = from -> from
+        {_, _, _} = from -> from
+        from when is_atom(from) -> {from, "priv/static"}
+        from when is_binary(from) -> from
+        _ -> raise ArgumentError, ":from must be an atom, a binary or a tuple"
+      end
+
+    encodings =
+      opts
+      |> Keyword.get(:encodings, [])
+      |> maybe_add("br", ".br", Keyword.get(opts, :brotli, false))
+      |> maybe_add("gzip", ".gz", Keyword.get(opts, :gzip, false))
+
+    %{
+      encodings: encodings,
+      only_rules: {Keyword.get(opts, :only, []), Keyword.get(opts, :only_matching, [])},
+      qs_cache: Keyword.get(opts, :cache_control_for_vsn_requests, "public, max-age=31536000"),
+      et_cache: Keyword.get(opts, :cache_control_for_etags, "public"),
+      et_generation: Keyword.get(opts, :etag_generation, nil),
+      headers: Keyword.get(opts, :headers, %{}),
+      content_types: Keyword.get(opts, :content_types, %{}),
+      set_content_type: Keyword.get(opts, :set_content_type, true),
+      from: from,
+      at: opts |> Keyword.fetch!(:at) |> Plug.Router.Utils.split()
+    }
+  end
+
+  @impl true
+  def call(
+        conn = %Conn{method: meth},
+        %{at: at, only_rules: only_rules, from: from, encodings: encodings} = options
+      )
+      when meth in @allowed_methods do
+    segments = subset(at, conn.path_info)
+
+    if allowed?(only_rules, segments) do
+      segments = Enum.map(segments, &uri_decode/1)
+
+      if invalid_path?(segments) do
+        raise InvalidPathError, "invalid path for static asset: #{conn.request_path}"
+      end
+
+      path = path(from, segments)
+      range = get_req_header(conn, "range")
+      encoding = file_encoding(conn, path, range, encodings)
+      serve_static(encoding, conn, segments, range, options)
+    else
+      conn
+    end
+  end
+
+  def call(conn, _options) do
+    conn
+  end
+
+  defp uri_decode(path) do
+    # TODO: Remove rescue as this can't fail from Elixir v1.13
+    try do
+      URI.decode(path)
+    rescue
+      ArgumentError ->
+        raise InvalidPathError
+    end
+  end
+
+  defp allowed?(_only_rules, []), do: false
+  defp allowed?({[], []}, _list), do: true
+
+  defp allowed?({full, prefix}, [h | _]) do
+    h in full or (prefix != [] and match?({0, _}, :binary.match(h, prefix)))
+  end
+
+  defp maybe_put_content_type(conn, false, _, _), do: conn
+
+  defp maybe_put_content_type(conn, _, types, filename) do
+    content_type = Map.get(types, filename) || MIME.from_path(filename)
+
+    conn
+    |> put_resp_header("content-type", content_type)
+  end
+
+  defp serve_static({content_encoding, file_info, path}, conn, segments, range, options) do
+    %{
+      qs_cache: qs_cache,
+      et_cache: et_cache,
+      et_generation: et_generation,
+      headers: headers,
+      content_types: types,
+      set_content_type: set_content_type
+    } = options
+
+    case put_cache_header(conn, qs_cache, et_cache, et_generation, file_info, path) do
+      {:stale, conn} ->
+        filename = List.last(segments)
+
+        conn
+        |> maybe_put_content_type(set_content_type, types, filename)
+        |> put_resp_header("accept-ranges", "bytes")
+        |> maybe_add_encoding(content_encoding)
+        |> merge_headers(headers)
+        |> serve_range(file_info, path, range, options)
+
+      {:fresh, conn} ->
+        conn
+        |> maybe_add_vary(options)
+        |> send_resp(304, "")
+        |> halt()
+    end
+  end
+
+  defp serve_static(:error, conn, _segments, _range, _options) do
+    conn
+  end
+
+  defp serve_range(conn, file_info, path, [range], options) do
+    file_info(size: file_size) = file_info
+
+    with %{"bytes" => bytes} <- Plug.Conn.Utils.params(range),
+         {range_start, range_end} <- start_and_end(bytes, file_size) do
+      send_range(conn, path, range_start, range_end, file_size, options)
+    else
+      _ -> send_entire_file(conn, path, options)
+    end
+  end
+
+  defp serve_range(conn, _file_info, path, _range, options) do
+    send_entire_file(conn, path, options)
+  end
+
+  defp start_and_end("-" <> rest, file_size) do
+    case Integer.parse(rest) do
+      {last, ""} when last > 0 and last <= file_size -> {file_size - last, file_size - 1}
+      _ -> :error
+    end
+  end
+
+  defp start_and_end(range, file_size) do
+    case Integer.parse(range) do
+      {first, "-"} when first >= 0 ->
+        {first, file_size - 1}
+
+      {first, "-" <> rest} when first >= 0 ->
+        case Integer.parse(rest) do
+          {last, ""} when last >= first -> {first, min(last, file_size - 1)}
+          _ -> :error
+        end
+
+      _ ->
+        :error
+    end
+  end
+
+  defp send_range(conn, path, 0, range_end, file_size, options) when range_end == file_size - 1 do
+    send_entire_file(conn, path, options)
+  end
+
+  defp send_range(conn, path, range_start, range_end, file_size, _options) do
+    length = range_end - range_start + 1
+
+    conn
+    |> put_resp_header("content-range", "bytes #{range_start}-#{range_end}/#{file_size}")
+    |> send_file(206, path, range_start, length)
+    |> halt()
+  end
+
+  defp send_entire_file(conn, path, options) do
+    conn
+    |> maybe_add_vary(options)
+    |> send_file(200, path)
+    |> halt()
+  end
+
+  defp maybe_add_encoding(conn, nil), do: conn
+  defp maybe_add_encoding(conn, ce), do: put_resp_header(conn, "content-encoding", ce)
+
+  defp maybe_add_vary(conn, %{encodings: encodings}) do
+    # If we serve gzip or brotli at any moment, we need to set the proper vary
+    # header regardless of whether we are serving gzip content right now.
+    # See: http://www.fastly.com/blog/best-practices-for-using-the-vary-header/
+    if encodings != [] do
+      update_in(conn.resp_headers, &[{"vary", "Accept-Encoding"} | &1])
+    else
+      conn
+    end
+  end
+
+  defp put_cache_header(
+         %Conn{query_string: "vsn=" <> _} = conn,
+         qs_cache,
+         _et_cache,
+         _et_generation,
+         _file_info,
+         _path
+       )
+       when is_binary(qs_cache) do
+    {:stale, put_resp_header(conn, "cache-control", qs_cache)}
+  end
+
+  defp put_cache_header(conn, _qs_cache, et_cache, et_generation, file_info, path)
+       when is_binary(et_cache) do
+    etag = etag_for_path(file_info, et_generation, path)
+
+    conn =
+      conn
+      |> put_resp_header("cache-control", et_cache)
+      |> put_resp_header("etag", etag)
+
+    if etag in get_req_header(conn, "if-none-match") do
+      {:fresh, conn}
+    else
+      {:stale, conn}
+    end
+  end
+
+  defp put_cache_header(conn, _, _, _, _, _) do
+    {:stale, conn}
+  end
+
+  defp etag_for_path(file_info, et_generation, path) do
+    case et_generation do
+      {module, function, args} ->
+        apply(module, function, [path | args])
+
+      nil ->
+        file_info(size: size, mtime: mtime) = file_info
+        <<?", {size, mtime} |> :erlang.phash2() |> Integer.to_string(16)::binary, ?">>
+    end
+  end
+
+  defp file_encoding(conn, path, [_range], _encodings) do
+    # We do not support compression for range queries.
+    file_encoding(conn, path, nil, [])
+  end
+
+  defp file_encoding(conn, path, _range, encodings) do
+    encoded =
+      Enum.find_value(encodings, fn {encoding, ext} ->
+        if file_info = accept_encoding?(conn, encoding) && regular_file_info(path <> ext) do
+          {encoding, file_info, path <> ext}
+        end
+      end)
+
+    cond do
+      not is_nil(encoded) ->
+        encoded
+
+      file_info = regular_file_info(path) ->
+        {nil, file_info, path}
+
+      true ->
+        :error
+    end
+  end
+
+  defp regular_file_info(path) do
+    case :prim_file.read_file_info(path) do
+      {:ok, file_info(type: :regular) = file_info} ->
+        file_info
+
+      _ ->
+        nil
+    end
+  end
+
+  defp accept_encoding?(conn, encoding) do
+    encoding? = &String.contains?(&1, [encoding, "*"])
+
+    Enum.any?(get_req_header(conn, "accept-encoding"), fn accept ->
+      accept |> Plug.Conn.Utils.list() |> Enum.any?(encoding?)
+    end)
+  end
+
+  defp maybe_add(list, key, value, true), do: list ++ [{key, value}]
+  defp maybe_add(list, _key, _value, false), do: list
+
+  defp path({module, function, arguments}, segments)
+       when is_atom(module) and is_atom(function) and is_list(arguments),
+       do: Enum.join([apply(module, function, arguments) | segments], "/")
+
+  defp path({app, from}, segments) when is_atom(app) and is_binary(from),
+    do: Enum.join([Application.app_dir(app), from | segments], "/")
+
+  defp path(from, segments),
+    do: Enum.join([from | segments], "/")
+
+  defp subset([h | expected], [h | actual]), do: subset(expected, actual)
+  defp subset([], actual), do: actual
+  defp subset(_, _), do: []
+
+  defp invalid_path?(list) do
+    invalid_path?(list, :binary.compile_pattern(["/", "\\", ":", "\0"]))
+  end
+
+  defp invalid_path?([h | _], _match) when h in [".", "..", ""], do: true
+  defp invalid_path?([h | t], match), do: String.contains?(h, match) or invalid_path?(t)
+  defp invalid_path?([], _match), do: false
+
+  defp merge_headers(conn, {module, function, args}) do
+    merge_headers(conn, apply(module, function, [conn | args]))
+  end
+
+  defp merge_headers(conn, headers) do
+    merge_resp_headers(conn, headers)
+  end
+end

lib/pleroma/web/plugs/uploaded_media.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do
   require Logger
 
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.Plugs.Utils
 
   @behaviour Plug
   # no slashes
@@ -28,10 +29,21 @@ def init(_opts) do
       |> Keyword.put(:at, "/__unconfigured_media_plug")
       |> Plug.Static.init()
 
-    %{static_plug_opts: static_plug_opts}
+    config = Pleroma.Config.get(Pleroma.Upload)
+    allowed_mime_types = Keyword.fetch!(config, :allowed_mime_types)
+    uploader = Keyword.fetch!(config, :uploader)
+
+    %{
+      static_plug_opts: static_plug_opts,
+      allowed_mime_types: allowed_mime_types,
+      uploader: uploader
+    }
   end
 
-  def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
+  def call(
+        %{request_path: <<"/", @path, "/", file::binary>>} = conn,
+        %{uploader: uploader} = opts
+      ) do
     conn =
       case fetch_query_params(conn) do
         %{query_params: %{"name" => name}} = conn ->
@@ -44,10 +56,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
       end
       |> merge_resp_headers([{"content-security-policy", "sandbox"}])
 
-    config = Pleroma.Config.get(Pleroma.Upload)
+    with {:ok, get_method} <- uploader.get_file(file),
-
-    with uploader <- Keyword.fetch!(config, :uploader),
-         {:ok, get_method} <- uploader.get_file(file),
          false <- media_is_banned(conn, get_method) do
       get_media(conn, get_method, opts)
     else
@@ -68,13 +77,23 @@ defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
 
   defp media_is_banned(_, _), do: false
 
+  defp set_content_type(conn, opts, filepath) do
+    real_mime = MIME.from_path(filepath)
+    clean_mime = Utils.get_safe_mime_type(opts, real_mime)
+    put_resp_header(conn, "content-type", clean_mime)
+  end
+
   defp get_media(conn, {:static_dir, directory}, opts) do
     static_opts =
       Map.get(opts, :static_plug_opts)
       |> Map.put(:at, [@path])
       |> Map.put(:from, directory)
+      |> Map.put(:set_content_type, false)
 
-    conn = Plug.Static.call(conn, static_opts)
+    conn =
+      conn
+      |> set_content_type(opts, conn.request_path)
+      |> Pleroma.Web.Plugs.StaticNoCT.call(static_opts)
 
     if conn.halted do
       conn

lib/pleroma/web/plugs/utils.ex

@@ -0,0 +1,14 @@
+# Akkoma: Magically expressive social media
+# Copyright © 2024 Akkoma Authors <https://akkoma.dev>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.Utils do
+  @moduledoc """
+  Some helper functions shared across several plugs
+  """
+
+  def get_safe_mime_type(%{allowed_mime_types: allowed_mime_types} = _opts, mime) do
+    [maintype | _] = String.split(mime, "/", parts: 2)
+    if maintype in allowed_mime_types, do: mime, else: "application/octet-stream"
+  end
+end

mix.exs

@@ -21,13 +21,13 @@ def project do
       source_url: "https://akkoma.dev/AkkomaGang/akkoma",
       docs: [
         source_url_pattern: "https://akkoma.dev/AkkomaGang/akkoma/blob/develop/%{path}#L%{line}",
-        logo: "priv/static/images/logo.png",
+        logo: "priv/static/logo-512.png",
-        extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/**/*.md"),
+        extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/docs/**/*.md"),
         groups_for_extras: [
-          "Installation manuals": Path.wildcard("docs/installation/*.md"),
+          "Installation manuals": Path.wildcard("docs/docs/installation/*.md"),
-          Configuration: Path.wildcard("docs/config/*.md"),
+          Configuration: Path.wildcard("docs/docs/config/*.md"),
-          Administration: Path.wildcard("docs/admin/*.md"),
+          Administration: Path.wildcard("docs/docs/admin/*.md"),
-          "Pleroma's APIs and Mastodon API extensions": Path.wildcard("docs/api/*.md")
+          "Pleroma's APIs and Mastodon API extensions": Path.wildcard("docs/docs/api/*.md")
         ],
         main: "readme",
         output: "priv/static/doc"

priv/templates/sample_config.eex

@@ -78,6 +78,8 @@ config :joken, default_signer: "<%= jwt_secret %>"
 
 config :pleroma, configurable_from_database: <%= db_configurable? %>
 
+config :pleroma, Pleroma.Upload,
 <%= if Kernel.length(upload_filters) > 0 do
-"config :pleroma, Pleroma.Upload, filters: #{inspect(upload_filters)}"
+"  filters: #{inspect(upload_filters)},"
 end %>
+  base_url: "<%= media_url %>"

test/fixtures/bridgy/actor.json

@@ -70,7 +70,7 @@
   "preferredUsername": "jk.nipponalba.scot",
   "summary": "",
   "publicKey": {
-    "id": "jk.nipponalba.scot",
+    "id": "https://fed.brid.gy/jk.nipponalba.scot#key",
     "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdarxwzxnNbJ2hneWOYHkYJowk\npyigQtxlUd0VjgSQHwxU9kWqfbrHBVADyTtcqi/4dAzQd3UnCI1TPNnn4LPZY9PW\noiWd3Zl1/EfLFxO7LU9GS7fcSLQkyj5JNhSlN3I8QPudZbybrgRDVZYooDe1D+52\n5KLGqC2ajrIVOiDRTQIDAQAB\n-----END PUBLIC KEY-----"
   },
   "inbox": "https://fed.brid.gy/jk.nipponalba.scot/inbox",

test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json

@@ -37,7 +37,7 @@
         "sharedInbox": "https://osada.macgirvin.com/inbox"
     },
     "publicKey": {
-        "id": "https://osada.macgirvin.com/channel/mike/public_key_pem",
+        "id": "https://osada.macgirvin.com/channel/mike",
         "owner": "https://osada.macgirvin.com/channel/mike",
         "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAskSyK2VwBNKbzZl9XNJk\nvxU5AAilmRArMmmKSzphdHaVBHakeafUfixvqNrQ/oX2srJvJKcghNmEMrJ6MJ7r\npeEndVOo7pcP4PwVjtnC06p3J711q5tBehqM25BfCLCrB2YqWF6c8zk3CPN3Na21\n8k5s4cO95N/rGN+Po0XFAX/HjKjlpgNpKRDrpxmXxTU8NZfAqeQGJ5oiMBZI9vVB\n+eU7t1L6F5/XWuUCeP4OMrG8oZX822AREba8rknS6DpkWGES0Rx2eNOyYTf6ue75\nI6Ek6rlO+da5wMWr+3BvYMq4JMIwTHzAO+ZqqJPFpzKSiVuAWb2DOX/MDFecVWJE\ntF/R60lONxe4e/00MPCoDdqkLKdwROsk1yGL7z4Zk6jOWFEhIcWy/d2Ya5CpPvS3\nu4wNN4jkYAjra+8TiloRELhV4gpcEk8nkyNwLXOhYm7zQ5sIc5rfXoIrFzALB86W\nG05Nnqg+77zZIaTZpD9qekYlaEt+0OVtt9TTIeTiudQ983l6mfKwZYymrzymH1dL\nVgxBRYo+Z53QOSLiSKELfTBZxEoP1pBw6RiOHXydmJ/39hGgc2YAY/5ADwW2F2yb\nJ7+gxG6bPJ3ikDLYcD4CB5iJQdnTcDsFt3jyHAT6wOCzFAYPbHUqtzHfUM30dZBn\nnJhQF8udPLcXLaj6GW75JacCAwEAAQ==\n-----END PUBLIC KEY-----\n"
     },

test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json

@@ -3,7 +3,7 @@
         "attributedTo": "http://mastodon.example.org/users/admin",
         "attachment": [],
         "content": "<p>this post was not actually written by Haelwenn</p>",
-        "id": "https://info.pleroma.site/activity2.json",
+        "id": "https://info.pleroma.site/activity3.json",
         "published": "2018-09-01T22:15:00Z",
         "tag": [],
         "to": [

test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json

@@ -1 +1 @@
-{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1","https://hubzilla.example.org/apschema/v1.2"],"type":"Person","id":"https://hubzilla.example.org/channel/kaniini","preferredUsername":"kaniini","name":"kaniini","icon":{"type":"Image","mediaType":"image/jpeg","url":"https://hubzilla.example.org/photo/profile/l/281","height":300,"width":300},"url":{"type":"Link","mediaType":"text/html","href":"https://hubzilla.example.org/channel/kaniini"},"inbox":"https://hubzilla.example.org/inbox/kaniini","outbox":"https://hubzilla.example.org/outbox/kaniini","followers":"https://hubzilla.example.org/followers/kaniini","following":"https://hubzilla.example.org/following/kaniini","endpoints":{"sharedInbox":"https://hubzilla.example.org/inbox"},"publicKey":{"id":"https://hubzilla.example.org/channel/kaniini/public_key_pem","owner":"https://hubzilla.example.org/channel/kaniini","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXCDkQPw+1N8B2CUd5s2\nbYvjHt+t7soMNfUiRy0qGbgW46S45k5lCq1KpbFIX3sgGZ4OWjnXVbvjCJi4kl5M\nfm5DBXzpuu05AmjVl8hqk4GejajiE/1Nq0uWHPiOSFWispUjCzzCu65V+IsiE5JU\nvcL6WEf/pYNRq7gYqyT693F7+cO5/rVv9OScx5UOxbIuU1VXYhdHCqAMDJWadC89\nhePrcD3HOQKl06W2tDxHcWk6QjrdsUQGbNOgK/QIN9gSxA+rCFEvH5O0HAhI0aXq\ncOB+vysJUFLeQOAqmAKvKS5V6RqE1GqqT0pDWHack4EmQi0gkgVzo+45xoP6wfDl\nWwG88w21LNxGvGHuN4I8mg6cEoApqKQBSOj086UtfDfSlPC1B+PRD2phE5etucHd\nF/RIWN3SxVzU9BKIiaDm2gwOpvI8QuorQb6HDtZFO5NsSN3PnMnSywPe7kXl/469\nuQRYXrseqyOVIi6WjhvXkyWVKVE5CBz+S8wXHfKph+9YOyUcJeAVMijp9wrjBlMc\noSzOGu79oM7tpMSq/Xo6ePJ/glNOwZR+OKrg92Qp9BGTKDNwGrxuxP/9KwWtGLNf\nOMTtIkxtC3ubhxL3lBxOd7l+Bmum0UJV2f8ogkCgvTpIz05jMoyU8qWl6kkWNQlY\nDropXWaOfy7Lac+G4qlfSgsCAwEAAQ==\n-----END PUBLIC KEY-----\n"},"nomadicLocations":[{"id":"https://hubzilla.example.org/locs/kaniini","type":"nomadicLocation","locationAddress":"acct:kaniini@hubzilla.example.org","locationPrimary":true,"locationDeleted":false}],"signature":{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"],"type":"RsaSignature2017","nonce":"6b981a2f3bdcffc20252e3b131d4a4569fd2dea9fac543e5196136302f492694","creator":"https://hubzilla.example.org/channel/kaniini/public_key_pem","created":"2018-05-19T08:19:13Z","signatureValue":"ezpT4iCIUzJSeJa/Jsf4EkgbX9enWZG/0eliLXZcvkeCX9mZabaX9LMQRViP2GSlAJBHJu+UqK5LWaoWw9pYkQQHUL+43w2DeBxQicEcPqpT46j6pHuWptfwB8YHTC2/Pb56Y/jseU37j+FW8xVmcGZk4cPqJRLQNojwJlQiFOpBEd4Cel6081W12Pep578+6xBL+h92RJsWznA1gE/NV9dkCqoAoNdiORJg68sVTm0yYxPit2D/DLwXUFeBhC47EZtY3DtAOf7rADGwbquXKug/wtEI47R4p9dJvMWERSVW9O2FmDk8deUjRR3qO1iYGce8O+uMnnBHmuTcToRUHH7mxfMdqjfbcZ9DGBjKtLPSOyVPT9rENeyX8fsksmX0XhfHsNSWkmeDaU5/Au3IY75gDewiGzmzLOpRc6GUnHHro7lMpyMuo3lLZKjNVsFZbx+sXCYwORz5GAMuwIt/iCUdrsQsF5aycqfUAZrFBPguH6DVjbMUqyLvS78sDKiWqgWVhq9VDKse+WuQaJLGBDJNF9APoA6NDMjjIBZfmkGf2mV7ubIYihoOncUjahFqxU5306cNxAcdj2uNcwkgX4BCnBe/L2YsvMHhZrupzDewWWy4fxhktyoZ7VhLSl1I7fMPytjOpb9EIvng4DHGX2t+hKfon2rCGfECPavwiTM="}}
+{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1","https://hubzilla.example.org/apschema/v1.2"],"type":"Person","id":"https://hubzilla.example.org/channel/kaniini","preferredUsername":"kaniini","name":"kaniini","icon":{"type":"Image","mediaType":"image/jpeg","url":"https://hubzilla.example.org/photo/profile/l/281","height":300,"width":300},"url":{"type":"Link","mediaType":"text/html","href":"https://hubzilla.example.org/channel/kaniini"},"inbox":"https://hubzilla.example.org/inbox/kaniini","outbox":"https://hubzilla.example.org/outbox/kaniini","followers":"https://hubzilla.example.org/followers/kaniini","following":"https://hubzilla.example.org/following/kaniini","endpoints":{"sharedInbox":"https://hubzilla.example.org/inbox"},"publicKey":{"id":"https://hubzilla.example.org/channel/kaniini","owner":"https://hubzilla.example.org/channel/kaniini","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXCDkQPw+1N8B2CUd5s2\nbYvjHt+t7soMNfUiRy0qGbgW46S45k5lCq1KpbFIX3sgGZ4OWjnXVbvjCJi4kl5M\nfm5DBXzpuu05AmjVl8hqk4GejajiE/1Nq0uWHPiOSFWispUjCzzCu65V+IsiE5JU\nvcL6WEf/pYNRq7gYqyT693F7+cO5/rVv9OScx5UOxbIuU1VXYhdHCqAMDJWadC89\nhePrcD3HOQKl06W2tDxHcWk6QjrdsUQGbNOgK/QIN9gSxA+rCFEvH5O0HAhI0aXq\ncOB+vysJUFLeQOAqmAKvKS5V6RqE1GqqT0pDWHack4EmQi0gkgVzo+45xoP6wfDl\nWwG88w21LNxGvGHuN4I8mg6cEoApqKQBSOj086UtfDfSlPC1B+PRD2phE5etucHd\nF/RIWN3SxVzU9BKIiaDm2gwOpvI8QuorQb6HDtZFO5NsSN3PnMnSywPe7kXl/469\nuQRYXrseqyOVIi6WjhvXkyWVKVE5CBz+S8wXHfKph+9YOyUcJeAVMijp9wrjBlMc\noSzOGu79oM7tpMSq/Xo6ePJ/glNOwZR+OKrg92Qp9BGTKDNwGrxuxP/9KwWtGLNf\nOMTtIkxtC3ubhxL3lBxOd7l+Bmum0UJV2f8ogkCgvTpIz05jMoyU8qWl6kkWNQlY\nDropXWaOfy7Lac+G4qlfSgsCAwEAAQ==\n-----END PUBLIC KEY-----\n"},"nomadicLocations":[{"id":"https://hubzilla.example.org/locs/kaniini","type":"nomadicLocation","locationAddress":"acct:kaniini@hubzilla.example.org","locationPrimary":true,"locationDeleted":false}],"signature":{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"],"type":"RsaSignature2017","nonce":"6b981a2f3bdcffc20252e3b131d4a4569fd2dea9fac543e5196136302f492694","creator":"https://hubzilla.example.org/channel","created":"2018-05-19T08:19:13Z","signatureValue":"ezpT4iCIUzJSeJa/Jsf4EkgbX9enWZG/0eliLXZcvkeCX9mZabaX9LMQRViP2GSlAJBHJu+UqK5LWaoWw9pYkQQHUL+43w2DeBxQicEcPqpT46j6pHuWptfwB8YHTC2/Pb56Y/jseU37j+FW8xVmcGZk4cPqJRLQNojwJlQiFOpBEd4Cel6081W12Pep578+6xBL+h92RJsWznA1gE/NV9dkCqoAoNdiORJg68sVTm0yYxPit2D/DLwXUFeBhC47EZtY3DtAOf7rADGwbquXKug/wtEI47R4p9dJvMWERSVW9O2FmDk8deUjRR3qO1iYGce8O+uMnnBHmuTcToRUHH7mxfMdqjfbcZ9DGBjKtLPSOyVPT9rENeyX8fsksmX0XhfHsNSWkmeDaU5/Au3IY75gDewiGzmzLOpRc6GUnHHro7lMpyMuo3lLZKjNVsFZbx+sXCYwORz5GAMuwIt/iCUdrsQsF5aycqfUAZrFBPguH6DVjbMUqyLvS78sDKiWqgWVhq9VDKse+WuQaJLGBDJNF9APoA6NDMjjIBZfmkGf2mV7ubIYihoOncUjahFqxU5306cNxAcdj2uNcwkgX4BCnBe/L2YsvMHhZrupzDewWWy4fxhktyoZ7VhLSl1I7fMPytjOpb9EIvng4DHGX2t+hKfon2rCGfECPavwiTM="}}

test/fixtures/tesla_mock/relay@mastdon.example.org.json

@@ -11,7 +11,7 @@
     "toot": "http://joinmastodon.org/ns#",
     "Emoji": "toot:Emoji"
   }],
-  "id": "http://mastodon.example.org/users/admin",
+  "id": "http://mastodon.example.org/users/relay",
   "type": "Application",
   "invisible": true,
   "following": "http://mastodon.example.org/users/admin/following",
@@ -24,8 +24,8 @@
   "url": "http://mastodon.example.org/@admin",
   "manuallyApprovesFollowers": false,
   "publicKey": {
-    "id": "http://mastodon.example.org/users/admin#main-key",
+    "id": "http://mastodon.example.org/users/relay#main-key",
-    "owner": "http://mastodon.example.org/users/admin",
+    "owner": "http://mastodon.example.org/users/relay",
     "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"
   },
   "attachment": [{

test/fixtures/users_mock/friendica_followers.json

@@ -13,7 +13,7 @@
       "directMessage": "litepub:directMessage"
     }
   ],
-  "id": "http://localhost:8080/followers/fuser3",
+  "id": "http://remote.org/followers/fuser3",
   "type": "OrderedCollection",
   "totalItems": 296
 }

test/fixtures/users_mock/friendica_following.json

@@ -13,7 +13,7 @@
       "directMessage": "litepub:directMessage"
     }
   ],
-  "id": "http://localhost:8080/following/fuser3",
+  "id": "http://remote.org/following/fuser3",
   "type": "OrderedCollection",
   "totalItems": 32
 }

test/fixtures/users_mock/masto_closed_followers.json

@@ -1,7 +1,7 @@
 {
   "@context": "https://www.w3.org/ns/activitystreams",
-  "id": "http://localhost:4001/users/masto_closed/followers",
+  "id": "http://remote.org/users/masto_closed/followers",
   "type": "OrderedCollection",
   "totalItems": 437,
-  "first": "http://localhost:4001/users/masto_closed/followers?page=1"
+  "first": "http://remote.org/users/masto_closed/followers?page=1"
 }

test/fixtures/users_mock/masto_closed_followers_page.json

@@ -1 +1 @@
-{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://localhost:4001/users/masto_closed/followers?page=2","partOf":"http://localhost:4001/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}
+{"@context":"https://www.w3.org/ns/activitystreams","id":"http://remote.org/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://remote.org/users/masto_closed/followers?page=2","partOf":"http://remote.org/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}

test/fixtures/users_mock/masto_closed_following.json

@@ -1,7 +1,7 @@
 {
   "@context": "https://www.w3.org/ns/activitystreams",
-  "id": "http://localhost:4001/users/masto_closed/following",
+  "id": "http://remote.org/users/masto_closed/following",
   "type": "OrderedCollection",
   "totalItems": 152,
-  "first": "http://localhost:4001/users/masto_closed/following?page=1"
+  "first": "http://remote.org/users/masto_closed/following?page=1"
 }

test/fixtures/users_mock/masto_closed_following_page.json

@@ -1 +1 @@
-{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://localhost:4001/users/masto_closed/following?page=2","partOf":"http://localhost:4001/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}
+{"@context":"https://www.w3.org/ns/activitystreams","id":"http://remote.org/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://remote.org/users/masto_closed/following?page=2","partOf":"http://remote.org/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}

test/fixtures/users_mock/pleroma_followers.json

@@ -1,14 +1,14 @@
 {
   "type": "OrderedCollection",
   "totalItems": 527,
-  "id": "http://localhost:4001/users/fuser2/followers",
+  "id": "http://remote.org/users/fuser2/followers",
   "first": {
     "type": "OrderedCollectionPage",
     "totalItems": 527,
-    "partOf": "http://localhost:4001/users/fuser2/followers",
+    "partOf": "http://remote.org/users/fuser2/followers",
     "orderedItems": [],
-    "next": "http://localhost:4001/users/fuser2/followers?page=2",
+    "next": "http://remote.org/users/fuser2/followers?page=2",
-    "id": "http://localhost:4001/users/fuser2/followers?page=1"
+    "id": "http://remote.org/users/fuser2/followers?page=1"
   },
   "@context": [
     "https://www.w3.org/ns/activitystreams",

test/fixtures/users_mock/pleroma_following.json

@@ -1,14 +1,14 @@
 {
   "type": "OrderedCollection",
   "totalItems": 267,
-  "id": "http://localhost:4001/users/fuser2/following",
+  "id": "http://remote.org/users/fuser2/following",
   "first": {
     "type": "OrderedCollectionPage",
     "totalItems": 267,
-    "partOf": "http://localhost:4001/users/fuser2/following",
+    "partOf": "http://remote.org/users/fuser2/following",
     "orderedItems": [],
-    "next": "http://localhost:4001/users/fuser2/following?page=2",
+    "next": "http://remote.org/users/fuser2/following?page=2",
-    "id": "http://localhost:4001/users/fuser2/following?page=1"
+    "id": "http://remote.org/users/fuser2/following?page=1"
   },
   "@context": [
     "https://www.w3.org/ns/activitystreams",

test/mix/tasks/pleroma/instance_test.exs

@@ -39,6 +39,8 @@ test "running gen" do
         tmp_path() <> "setup.psql",
         "--domain",
         "test.pleroma.social",
+        "--media-url",
+        "https://media.pleroma.social/media",
         "--instance-name",
         "Pleroma",
         "--admin-email",
@@ -69,8 +71,6 @@ test "running gen" do
         "./test/../test/instance/static/",
         "--strip-uploads",
         "y",
-        "--dedupe-uploads",
-        "n",
         "--anonymize-uploads",
         "n"
       ])
@@ -92,6 +92,7 @@ test "running gen" do
     assert generated_config =~ "configurable_from_database: true"
     assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
     assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]"
+    assert generated_config =~ "base_url: \"https://media.pleroma.social/media\""
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
   end

test/mix/tasks/pleroma/uploads_test.exs

@@ -16,7 +16,6 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do
       Mix.shell(Mix.Shell.IO)
     end)
 
-    File.mkdir_p!("test/uploads")
     :ok
   end
 

test/pleroma/collections/collections_fetcher_test.exs

@@ -12,11 +12,14 @@ defmodule Akkoma.Collections.FetcherTest do
   end
 
   test "it should extract items from an embedded array in a Collection" do
+    ap_id = "https://example.com/collection/ordered_array"
+
     unordered_collection =
       "test/fixtures/collections/unordered_array.json"
       |> File.read!()
-
+      |> Jason.decode!()
-    ap_id = "https://example.com/collection/ordered_array"
+      |> Map.put("id", ap_id)
+      |> Jason.encode!(pretty: true)
 
     Tesla.Mock.mock(fn
       %{

test/pleroma/emoji/pack_test.exs

@@ -93,7 +93,9 @@ test "add emoji file", %{pack: pack} do
     assert updated_pack.files_count == 1
   end
 
-  test "load_pack/1 ignores path traversal in a forged pack name", %{pack: pack} do
+  test "load_pack/1 panics on path traversal in a forged pack name" do
-    assert {:ok, ^pack} = Pack.load_pack("../../../../../dump_pack")
+    assert_raise(RuntimeError, "Invalid or malicious pack name: ../../../../../dump_pack", fn ->
+      Pack.load_pack("../../../../../dump_pack")
+    end)
   end
 end

test/pleroma/object/containment_test.exs

@@ -17,16 +17,58 @@ defmodule Pleroma.Object.ContainmentTest do
   end
 
   describe "general origin containment" do
-    test "works for completely actorless posts" do
+    test "handles completly actorless objects gracefully" do
-      assert :error ==
+      assert :ok ==
-               Containment.contain_origin("https://glaceon.social/users/monorail", %{
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
                  "deleted" => "2019-10-30T05:48:50.249606Z",
                  "formerType" => "Note",
-                 "id" => "https://glaceon.social/users/monorail/statuses/103049757364029187",
+                 "id" => "https://glaceon.social/statuses/123",
                  "type" => "Tombstone"
                })
     end
 
+    test "errors for spoofed actors" do
+      assert :error ==
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
+                 "actor" => "https://otp.akkoma.dev/users/you",
+                 "id" => "https://glaceon.social/statuses/123",
+                 "type" => "Note"
+               })
+    end
+
+    test "errors for spoofed attributedTo" do
+      assert :error ==
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
+                 "attributedTo" => "https://otp.akkoma.dev/users/you",
+                 "id" => "https://glaceon.social/statuses/123",
+                 "type" => "Note"
+               })
+    end
+
+    test "accepts valid actors" do
+      assert :ok ==
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
+                 "actor" => "https://glaceon.social/users/monorail",
+                 "attributedTo" => "https://glaceon.social/users/monorail",
+                 "id" => "https://glaceon.social/statuses/123",
+                 "type" => "Note"
+               })
+
+      assert :ok ==
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
+                 "actor" => "https://glaceon.social/users/monorail",
+                 "id" => "https://glaceon.social/statuses/123",
+                 "type" => "Note"
+               })
+
+      assert :ok ==
+               Containment.contain_origin("https://glaceon.social/statuses/123", %{
+                 "attributedTo" => "https://glaceon.social/users/monorail",
+                 "id" => "https://glaceon.social/statuses/123",
+                 "type" => "Note"
+               })
+    end
+
     test "contain_origin_from_id() catches obvious spoofing attempts" do
       data = %{
         "id" => "http://example.com/~alyssa/activities/1234.json"
@@ -63,6 +105,56 @@ test "contain_origin_from_id() allows matching IDs" do
         )
     end
 
+    test "contain_id_to_fetch() refuses alternate IDs within the same origin domain" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json",
+        "url" => "http://example.com/@alyssa/status/1234"
+      }
+
+      :error =
+        Containment.contain_id_to_fetch(
+          "http://example.com/~alyssa/activities/1234",
+          data
+        )
+    end
+
+    test "contain_id_to_fetch() allows matching IDs" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json/"
+      }
+
+      :ok =
+        Containment.contain_id_to_fetch(
+          "http://example.com/~alyssa/activities/1234.json/",
+          data
+        )
+
+      :ok =
+        Containment.contain_id_to_fetch(
+          "http://example.com/~alyssa/activities/1234.json",
+          data
+        )
+    end
+
+    test "contain_id_to_fetch() allows display URLs" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json",
+        "url" => "http://example.com/@alyssa/status/1234"
+      }
+
+      :ok =
+        Containment.contain_id_to_fetch(
+          "http://example.com/@alyssa/status/1234",
+          data
+        )
+
+      :ok =
+        Containment.contain_id_to_fetch(
+          "http://example.com/@alyssa/status/1234/",
+          data
+        )
+    end
+
     test "users cannot be collided through fake direction spoofing attempts" do
       _user =
         insert(:user, %{

test/pleroma/object/fetcher_test.exs

@@ -14,6 +14,17 @@ defmodule Pleroma.Object.FetcherTest do
   import Mock
   import Tesla.Mock
 
+  defp spoofed_object_with_ids(
+         id \\ "https://patch.cx/objects/spoof",
+         actor_id \\ "https://patch.cx/users/rin"
+       ) do
+    File.read!("test/fixtures/spoofed-object.json")
+    |> Jason.decode!()
+    |> Map.put("id", id)
+    |> Map.put("actor", actor_id)
+    |> Jason.encode!()
+  end
+
   setup do
     mock(fn
       %{method: :get, url: "https://mastodon.example.org/users/userisgone"} ->
@@ -22,6 +33,32 @@ defmodule Pleroma.Object.FetcherTest do
       %{method: :get, url: "https://mastodon.example.org/users/userisgone404"} ->
         %Tesla.Env{status: 404}
 
+      # Spoof: wrong Content-Type
+      %{
+        method: :get,
+        url: "https://patch.cx/objects/spoof_content_type.json"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/objects/spoof_content_type.json",
+          headers: [{"content-type", "application/json"}],
+          body: spoofed_object_with_ids("https://patch.cx/objects/spoof_content_type.json")
+        }
+
+      # Spoof: no Content-Type
+      %{
+        method: :get,
+        url: "https://patch.cx/objects/spoof_content_type"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/objects/spoof_content_type",
+          headers: [],
+          body: spoofed_object_with_ids("https://patch.cx/objects/spoof_content_type")
+        }
+
+      # Spoof: mismatching ids
+      # Variant 1: Non-exisitng fake id
       %{
         method: :get,
         url:
@@ -29,8 +66,75 @@ defmodule Pleroma.Object.FetcherTest do
       } ->
         %Tesla.Env{
           status: 200,
-          headers: [{"content-type", "application/json"}],
+          url:
-          body: File.read!("test/fixtures/spoofed-object.json")
+            "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids()
+        }
+
+      %{method: :get, url: "https://patch.cx/objects/spoof"} ->
+        %Tesla.Env{
+          status: 404,
+          url: "https://patch.cx/objects/spoof",
+          headers: [],
+          body: "Not found"
+        }
+
+      # Varaint 2: two-stage payload
+      %{method: :get, url: "https://patch.cx/media/spoof_stage1.json"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/media/spoof_stage1.json",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids("https://patch.cx/media/spoof_stage2.json")
+        }
+
+      %{method: :get, url: "https://patch.cx/media/spoof_stage2.json"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/media/spoof_stage2.json",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids("https://patch.cx/media/unpredictable.json")
+        }
+
+      # Spoof: cross-domain redirect with original domain id
+      %{method: :get, url: "https://patch.cx/objects/spoof_media_redirect1"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://media.patch.cx/objects/spoof",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids("https://patch.cx/objects/spoof_media_redirect1")
+        }
+
+      # Spoof: cross-domain redirect with final domain id
+      %{method: :get, url: "https://patch.cx/objects/spoof_media_redirect2"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://media.patch.cx/objects/spoof_media_redirect2",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids("https://media.patch.cx/objects/spoof_media_redirect2")
+        }
+
+      # No-Spoof: same domain redirect
+      %{method: :get, url: "https://patch.cx/objects/spoof_redirect"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/objects/spoof_redirect",
+          headers: [{"content-type", "application/activity+json"}],
+          body: spoofed_object_with_ids("https://patch.cx/objects/spoof_redirect")
+        }
+
+      # Spoof: Actor from another domain
+      %{method: :get, url: "https://patch.cx/objects/spoof_foreign_actor"} ->
+        %Tesla.Env{
+          status: 200,
+          url: "https://patch.cx/objects/spoof_foreign_actor",
+          headers: [{"content-type", "application/activity+json"}],
+          body:
+            spoofed_object_with_ids(
+              "https://patch.cx/objects/spoof_foreign_actor",
+              "https://not.patch.cx/users/rin"
+            )
         }
 
       env ->
@@ -46,6 +150,7 @@ defmodule Pleroma.Object.FetcherTest do
         %{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} ->
           %Tesla.Env{
             status: 200,
+            url: "https://social.sakamoto.gq/objects/f20f2497-66d9-4a52-a2e1-1be2a39c32c1",
             body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"),
             headers: HttpRequestMock.activitypub_object_headers()
           }
@@ -129,6 +234,71 @@ test "it rejects objects when attributedTo is wrong (variant 2)" do
     end
   end
 
+  describe "fetcher security and auth checks" do
+    test "it does not fetch a spoofed object without content type" do
+      assert {:error, {:content_type, nil}} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_content_type"
+               )
+    end
+
+    test "it does not fetch a spoofed object with wrong content type" do
+      assert {:error, {:content_type, _}} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_content_type.json"
+               )
+    end
+
+    test "it does not fetch a spoofed object with id different from URL" do
+      assert {:error, "Object's ActivityPub id/url does not match final fetch URL"} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
+               )
+
+      assert {:error, "Object's ActivityPub id/url does not match final fetch URL"} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/media/spoof_stage1.json"
+               )
+    end
+
+    test "it does not fetch an object via cross-domain redirects (initial id)" do
+      assert {:error, {:cross_domain_redirect, true}} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_media_redirect1"
+               )
+    end
+
+    test "it does not fetch an object via cross-domain redirects (final id)" do
+      assert {:error, {:cross_domain_redirect, true}} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_media_redirect2"
+               )
+    end
+
+    test "it accepts same-domain redirects" do
+      assert {:ok, %{"id" => id} = _object} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_redirect"
+               )
+
+      assert id == "https://patch.cx/objects/spoof_redirect"
+    end
+
+    test "it does not fetch a spoofed object with a foreign actor" do
+      assert {:error, "Object containment failed."} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://patch.cx/objects/spoof_foreign_actor"
+               )
+    end
+
+    test "it does not fetch from localhost" do
+      assert {:error, "Trying to fetch local resource"} =
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 Pleroma.Web.Endpoint.url() <> "/spoof_local"
+               )
+    end
+  end
+
   describe "fetching an object" do
     test "it fetches an object" do
       {:ok, object} =
@@ -155,13 +325,6 @@ test "Return MRF reason when fetched status is rejected by one" do
                )
     end
 
-    test "it does not fetch a spoofed object uploaded on an instance as an attachment" do
-      assert {:error, _} =
-               Fetcher.fetch_object_from_id(
-                 "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
-               )
-    end
-
     test "does not fetch anything from a rejected instance" do
       clear_config([:mrf_simple, :reject], [{"evil.example.org", "i said so"}])
 
@@ -583,12 +746,13 @@ test "should return ok if the content type is application/activity+json" do
         } ->
           %Tesla.Env{
             status: 200,
+            url: "https://mastodon.social/2",
             headers: [{"content-type", "application/activity+json"}],
             body: "{}"
           }
       end)
 
-      assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2")
+      assert {:ok, _, "{}"} = Fetcher.get_object("https://mastodon.social/2")
     end
 
     test "should return ok if the content type is application/ld+json with a profile" do
@@ -599,6 +763,7 @@ test "should return ok if the content type is application/ld+json with a profile
         } ->
           %Tesla.Env{
             status: 200,
+            url: "https://mastodon.social/2",
             headers: [
               {"content-type",
                "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""}
@@ -607,24 +772,7 @@ test "should return ok if the content type is application/ld+json with a profile
           }
       end)
 
-      assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2")
+      assert {:ok, _, "{}"} = Fetcher.get_object("https://mastodon.social/2")
-
-      Tesla.Mock.mock(fn
-        %{
-          method: :get,
-          url: "https://mastodon.social/2"
-        } ->
-          %Tesla.Env{
-            status: 200,
-            headers: [
-              {"content-type",
-               "application/ld+json; profile=\"http://www.w3.org/ns/activitystreams\""}
-            ],
-            body: "{}"
-          }
-      end)
-
-      assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2")
     end
 
     test "should not return ok with other content types" do
@@ -635,6 +783,7 @@ test "should not return ok with other content types" do
         } ->
           %Tesla.Env{
             status: 200,
+            url: "https://mastodon.social/2",
             headers: [{"content-type", "application/json"}],
             body: "{}"
           }
@@ -643,5 +792,23 @@ test "should not return ok with other content types" do
       assert {:error, {:content_type, "application/json"}} =
                Fetcher.get_object("https://mastodon.social/2")
     end
+
+    test "returns the url after redirects" do
+      Tesla.Mock.mock(fn
+        %{
+          method: :get,
+          url: "https://mastodon.social/5"
+        } ->
+          %Tesla.Env{
+            status: 200,
+            url: "https://mastodon.social/7",
+            headers: [{"content-type", "application/activity+json"}],
+            body: "{}"
+          }
+      end)
+
+      assert {:ok, "https://mastodon.social/7", "{}"} =
+               Fetcher.get_object("https://mastodon.social/5")
+    end
   end
 end

test/pleroma/object_test.exs

@@ -22,6 +22,13 @@ defmodule Pleroma.ObjectTest do
     :ok
   end
 
+  # Only works for a single attachment but that's all we need here
+  defp get_attachment_filepath(note, uploads_dir) do
+    %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = note
+    filename = href |> Path.basename()
+    "#{uploads_dir}/#{filename}"
+  end
+
   test "returns an object by it's AP id" do
     object = insert(:note)
     found_object = Object.get_by_ap_id(object.data["id"])
@@ -95,14 +102,13 @@ test "Disabled via config" do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+      note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
-        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -111,7 +117,7 @@ test "Disabled via config" do
       assert Object.get_by_id(note.id).data["deleted"]
       refute Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
     end
 
     test "in subdirectories" do
@@ -129,14 +135,13 @@ test "in subdirectories" do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+      note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
-        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -145,7 +150,7 @@ test "in subdirectories" do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
 
     test "with dedupe enabled" do
@@ -168,13 +173,11 @@ test "with dedupe enabled" do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+      note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
-        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
-      filename = Path.basename(href)
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, files} = File.ls(uploads_dir)
+      assert File.exists?("#{path}")
-      assert filename in files
 
       Object.delete(note)
 
@@ -182,8 +185,8 @@ test "with dedupe enabled" do
 
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
-      assert {:ok, files} = File.ls(uploads_dir)
+      # what if another test runs concurrently using the same image file?
-      refute filename in files
+      refute File.exists?("#{path}")
     end
 
     test "with objects that have legacy data.url attribute" do
@@ -203,14 +206,13 @@ test "with objects that have legacy data.url attribute" do
 
       {:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+      note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
-        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -219,7 +221,7 @@ test "with objects that have legacy data.url attribute" do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
 
     test "With custom base_url" do
@@ -238,14 +240,13 @@ test "With custom base_url" do
       {:ok, %Object{} = attachment} =
         Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
 
-      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+      note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
-        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
-      path = href |> Path.dirname() |> Path.basename()
+      path = get_attachment_filepath(note, uploads_dir)
 
-      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+      assert File.exists?("#{path}")
 
       Object.delete(note)
 
@@ -254,7 +255,7 @@ test "With custom base_url" do
       assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
-      assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+      refute File.exists?("#{path}")
     end
   end
 

test/pleroma/reverse_proxy_test.exs

@@ -75,13 +75,16 @@ test "common", %{conn: conn} do
       Tesla.Mock.mock(fn %{method: :head, url: "/head"} ->
         %Tesla.Env{
           status: 200,
-          headers: [{"content-type", "text/html; charset=utf-8"}],
+          headers: [{"content-type", "image/png"}],
           body: ""
         }
       end)
 
       conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head")
-      assert html_response(conn, 200) == ""
+
+      assert conn.status == 200
+      assert Conn.get_resp_header(conn, "content-type") == ["image/png"]
+      assert conn.resp_body == ""
     end
   end
 
@@ -252,4 +255,38 @@ test "with content-disposition header", %{conn: conn} do
       assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers
     end
   end
+
+  describe "content-type sanitisation" do
+    test "preserves video type", %{conn: conn} do
+      Tesla.Mock.mock(fn %{method: :get, url: "/content"} ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-type", "video/mp4"}],
+          body: "test"
+        }
+      end)
+
+      conn = ReverseProxy.call(Map.put(conn, :method, "GET"), "/content")
+
+      assert conn.status == 200
+      assert Conn.get_resp_header(conn, "content-type") == ["video/mp4"]
+      assert conn.resp_body == "test"
+    end
+
+    test "replaces application type", %{conn: conn} do
+      Tesla.Mock.mock(fn %{method: :get, url: "/content"} ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-type", "application/activity+json"}],
+          body: "test"
+        }
+      end)
+
+      conn = ReverseProxy.call(Map.put(conn, :method, "GET"), "/content")
+
+      assert conn.status == 200
+      assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
+      assert conn.resp_body == "test"
+    end
+  end
 end

test/pleroma/upload_test.exs

@@ -188,7 +188,7 @@ test "copies the file to the configured folder with anonymizing filename" do
       refute data["name"] == "an [image.jpg"
     end
 
-    test "escapes invalid characters in url" do
+    test "mangles the filename" do
       File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 
       file = %Plug.Upload{
@@ -200,23 +200,8 @@ test "escapes invalid characters in url" do
       {:ok, data} = Upload.store(file)
       [attachment_url | _] = data["url"]
 
-      assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg"
+      refute Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg"
-    end
+      refute Path.basename(attachment_url["href"]) == "an… image.jpg"
-
-    test "escapes reserved uri characters" do
-      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
-
-      file = %Plug.Upload{
-        content_type: "image/jpeg",
-        path: Path.absname("test/fixtures/image_tmp.jpg"),
-        filename: ":?#[]@!$&\\'()*+,;=.jpg"
-      }
-
-      {:ok, data} = Upload.store(file)
-      [attachment_url | _] = data["url"]
-
-      assert Path.basename(attachment_url["href"]) ==
-               "%3A%3F%23%5B%5D%40%21%24%26%5C%27%28%29%2A%2B%2C%3B%3D.jpg"
     end
   end
 

test/pleroma/user_test.exs

@@ -326,9 +326,9 @@ test "unfollow with synchronizes external user" do
         insert(:user, %{
           local: false,
           nickname: "fuser2",
-          ap_id: "http://localhost:4001/users/fuser2",
+          ap_id: "http://remote.org/users/fuser2",
-          follower_address: "http://localhost:4001/users/fuser2/followers",
+          follower_address: "http://remote.org/users/fuser2/followers",
-          following_address: "http://localhost:4001/users/fuser2/following"
+          following_address: "http://remote.org/users/fuser2/following"
         })
 
       {:ok, user, followed} = User.follow(user, followed, :follow_accept)
@@ -2177,8 +2177,8 @@ test "it returns a list of AP ids for a given set of nicknames" do
 
   describe "sync followers count" do
     setup do
-      user1 = insert(:user, local: false, ap_id: "http://localhost:4001/users/masto_closed")
+      user1 = insert(:user, local: false, ap_id: "http://remote.org/users/masto_closed")
-      user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2")
+      user2 = insert(:user, local: false, ap_id: "http://remote.org/users/fuser2")
       insert(:user, local: true)
       insert(:user, local: false, is_active: false)
       {:ok, user1: user1, user2: user2}
@@ -2272,8 +2272,8 @@ test "updates the counters normally on following/getting a follow when disabled"
       other_user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_closed/followers",
+          follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://localhost:4001/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following",
           ap_enabled: true
         )
 
@@ -2294,8 +2294,8 @@ test "synchronizes the counters with the remote instance for the followed when e
       other_user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_closed/followers",
+          follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://localhost:4001/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following",
           ap_enabled: true
         )
 
@@ -2316,8 +2316,8 @@ test "synchronizes the counters with the remote instance for the follower when e
       other_user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_closed/followers",
+          follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://localhost:4001/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following",
           ap_enabled: true
         )
 

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -312,7 +312,7 @@ test "fetches user featured collection" do
   end
 
   test "fetches user featured collection using the first property" do
-    featured_url = "https://friendica.example.com/raha/collections/featured"
+    featured_url = "https://friendica.example.com/featured/raha"
     first_url = "https://friendica.example.com/featured/raha?page=1"
 
     featured_data =
@@ -350,7 +350,7 @@ test "fetches user featured collection using the first property" do
   end
 
   test "fetches user featured when it has string IDs" do
-    featured_url = "https://example.com/alisaie/collections/featured"
+    featured_url = "https://example.com/users/alisaie/collections/featured"
     dead_url = "https://example.com/users/alisaie/statuses/108311386746229284"
 
     featured_data =
@@ -1304,14 +1304,6 @@ test "returns reblogs for users for whom reblogs have not been muted" do
       %{test_file: test_file}
     end
 
-    test "strips / from filename", %{test_file: file} do
-      file = %Plug.Upload{file | filename: "../../../../../nested/bad.jpg"}
-      {:ok, %Object{} = object} = ActivityPub.upload(file)
-      [%{"href" => href}] = object.data["url"]
-      assert Regex.match?(~r"/bad.jpg$", href)
-      refute Regex.match?(~r"/nested/", href)
-    end
-
     test "sets a description if given", %{test_file: file} do
       {:ok, %Object{} = object} = ActivityPub.upload(file, description: "a cool file")
       assert object.data["name"] == "a cool file"
@@ -1651,8 +1643,8 @@ test "synchronizes following/followers counters" do
       user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/fuser2/followers",
+          follower_address: "http://remote.org/users/fuser2/followers",
-          following_address: "http://localhost:4001/users/fuser2/following"
+          following_address: "http://remote.org/users/fuser2/following"
         )
 
       {:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
@@ -1663,7 +1655,7 @@ test "synchronizes following/followers counters" do
     test "detects hidden followers" do
       mock(fn env ->
         case env.url do
-          "http://localhost:4001/users/masto_closed/followers?page=1" ->
+          "http://remote.org/users/masto_closed/followers?page=1" ->
             %Tesla.Env{status: 403, body: ""}
 
           _ ->
@@ -1674,8 +1666,8 @@ test "detects hidden followers" do
       user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_closed/followers",
+          follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://localhost:4001/users/masto_closed/following"
+          following_address: "http://remote.org/users/masto_closed/following"
         )
 
       {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
@@ -1686,7 +1678,7 @@ test "detects hidden followers" do
     test "detects hidden follows" do
       mock(fn env ->
         case env.url do
-          "http://localhost:4001/users/masto_closed/following?page=1" ->
+          "http://remote.org/users/masto_closed/following?page=1" ->
             %Tesla.Env{status: 403, body: ""}
 
           _ ->
@@ -1697,8 +1689,8 @@ test "detects hidden follows" do
       user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_closed/followers",
+          follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://localhost:4001/users/masto_closed/following"
+          following_address: "http://remote.org/users/masto_closed/following"
         )
 
       {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
@@ -1710,8 +1702,8 @@ test "detects hidden follows/followers for friendica" do
       user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:8080/followers/fuser3",
+          follower_address: "http://remote.org/followers/fuser3",
-          following_address: "http://localhost:8080/following/fuser3"
+          following_address: "http://remote.org/following/fuser3"
         )
 
       {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
@@ -1724,28 +1716,28 @@ test "detects hidden follows/followers for friendica" do
     test "doesn't crash when follower and following counters are hidden" do
       mock(fn env ->
         case env.url do
-          "http://localhost:4001/users/masto_hidden_counters/following" ->
+          "http://remote.org/users/masto_hidden_counters/following" ->
             json(
               %{
                 "@context" => "https://www.w3.org/ns/activitystreams",
-                "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
+                "id" => "http://remote.org/users/masto_hidden_counters/following"
               },
               headers: HttpRequestMock.activitypub_object_headers()
             )
 
-          "http://localhost:4001/users/masto_hidden_counters/following?page=1" ->
+          "http://remote.org/users/masto_hidden_counters/following?page=1" ->
             %Tesla.Env{status: 403, body: ""}
 
-          "http://localhost:4001/users/masto_hidden_counters/followers" ->
+          "http://remote.org/users/masto_hidden_counters/followers" ->
             json(
               %{
                 "@context" => "https://www.w3.org/ns/activitystreams",
-                "id" => "http://localhost:4001/users/masto_hidden_counters/following"
+                "id" => "http://remote.org/users/masto_hidden_counters/followers"
               },
               headers: HttpRequestMock.activitypub_object_headers()
             )
 
-          "http://localhost:4001/users/masto_hidden_counters/followers?page=1" ->
+          "http://remote.org/users/masto_hidden_counters/followers?page=1" ->
             %Tesla.Env{status: 403, body: ""}
         end
       end)
@@ -1753,8 +1745,8 @@ test "doesn't crash when follower and following counters are hidden" do
       user =
         insert(:user,
           local: false,
-          follower_address: "http://localhost:4001/users/masto_hidden_counters/followers",
+          follower_address: "http://remote.org/users/masto_hidden_counters/followers",
-          following_address: "http://localhost:4001/users/masto_hidden_counters/following"
+          following_address: "http://remote.org/users/masto_hidden_counters/following"
         )
 
       {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)

test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs

@@ -7,9 +7,57 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
 
   alias Pleroma.Config
   alias Pleroma.Emoji
+  alias Pleroma.Emoji.Pack
   alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
 
+  defp has_pack?() do
+    case Pack.load_pack("stolen") do
+      {:ok, _pack} -> true
+      {:error, :enoent} -> false
+    end
+  end
+
+  defp has_emoji?(shortcode) do
+    case Pack.load_pack("stolen") do
+      {:ok, pack} -> Map.has_key?(pack.files, shortcode)
+      {:error, :enoent} -> false
+    end
+  end
+
+  defmacro mock_tesla(
+             url \\ "https://example.org/emoji/firedfox.png",
+             status \\ 200,
+             headers \\ [],
+             get_body \\ File.read!("test/fixtures/image.jpg")
+           ) do
+    quote do
+      Tesla.Mock.mock(fn
+        %{method: :head, url: unquote(url)} ->
+          %Tesla.Env{
+            status: unquote(status),
+            body: nil,
+            url: unquote(url),
+            headers: unquote(headers)
+          }
+
+        %{method: :get, url: unquote(url)} ->
+          %Tesla.Env{
+            status: unquote(status),
+            body: unquote(get_body),
+            url: unquote(url),
+            headers: unquote(headers)
+          }
+      end)
+    end
+  end
+
   setup do
+    clear_config(:mrf_steal_emoji,
+      hosts: ["example.org"],
+      size_limit: 284_468,
+      download_unknown_size: true
+    )
+
     emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen")
 
     Emoji.reload()
@@ -26,41 +74,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
       File.rm_rf!(emoji_path)
     end)
 
-    [message: message, path: emoji_path]
+    [message: message]
   end
 
   test "does nothing by default", %{message: message} do
     refute "firedfox" in installed()
 
+    clear_config(:mrf_steal_emoji, [])
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
     refute "firedfox" in installed()
   end
 
   test "Steals emoji on unknown shortcode from allowed remote host", %{
-    message: message,
+    message: message
-    path: path
   } do
     refute "firedfox" in installed()
-    refute File.exists?(path)
+    refute has_pack?()
 
-    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} ->
+    mock_tesla()
-      %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
-    end)
-
-    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
 
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
     assert "firedfox" in installed()
-    assert File.exists?(path)
+    assert has_pack?()
 
-    assert path
+    assert has_emoji?("firedfox")
-           |> Path.join("firedfox.png")
-           |> File.exists?()
   end
 
-  test "rejects invalid shortcodes", %{path: path} do
+  test "rejects invalid shortcodes" do
     message = %{
       "type" => "Create",
       "object" => %{
@@ -69,31 +111,38 @@ test "rejects invalid shortcodes", %{path: path} do
       }
     }
 
-    fullpath = Path.join(path, "fired/fox.png")
+    mock_tesla()
-
-    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox"} ->
-      %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
-    end)
-
-    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
 
     refute "firedfox" in installed()
-    refute File.exists?(path)
+    refute has_pack?()
 
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
     refute "fired/fox" in installed()
-    refute File.exists?(fullpath)
+    refute has_emoji?("fired/fox")
+  end
+
+  test "prefers content-type header for extension" do
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.fud"}],
+        "actor" => "https://example.org/users/admin"
+      }
+    }
+
+    mock_tesla("https://example.org/emoji/firedfox.fud", 200, [{"content-type", "image/gif"}])
+
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
+
+    assert "firedfox" in installed()
+    assert has_emoji?("firedfox")
   end
 
   test "reject regex shortcode", %{message: message} do
     refute "firedfox" in installed()
 
-    clear_config(:mrf_steal_emoji,
+    clear_config([:mrf_steal_emoji, :rejected_shortcodes], [~r/firedfox/])
-      hosts: ["example.org"],
-      size_limit: 284_468,
-      rejected_shortcodes: [~r/firedfox/]
-    )
 
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
@@ -103,11 +152,7 @@ test "reject regex shortcode", %{message: message} do
   test "reject string shortcode", %{message: message} do
     refute "firedfox" in installed()
 
-    clear_config(:mrf_steal_emoji,
+    clear_config([:mrf_steal_emoji, :rejected_shortcodes], ["firedfox"])
-      hosts: ["example.org"],
-      size_limit: 284_468,
-      rejected_shortcodes: ["firedfox"]
-    )
 
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
@@ -117,11 +162,9 @@ test "reject string shortcode", %{message: message} do
   test "reject if size is above the limit", %{message: message} do
     refute "firedfox" in installed()
 
-    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} ->
+    mock_tesla()
-      %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}
-    end)
 
-    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000)
+    clear_config([:mrf_steal_emoji, :size_limit], 50_000)
 
     assert {:ok, _message} = StealEmojiPolicy.filter(message)
 
@@ -131,11 +174,7 @@ test "reject if size is above the limit", %{message: message} do
   test "reject if host returns error", %{message: message} do
     refute "firedfox" in installed()
 
-    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} ->
+    mock_tesla("https://example.org/emoji/firedfox.png", 404, [], "Not found")
-      {:ok, %Tesla.Env{status: 404, body: "Not found"}}
-    end)
-
-    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
 
     ExUnit.CaptureLog.capture_log(fn ->
       assert {:ok, _message} = StealEmojiPolicy.filter(message)
@@ -144,5 +183,44 @@ test "reject if host returns error", %{message: message} do
     refute "firedfox" in installed()
   end
 
+  test "reject unknown size", %{message: message} do
+    clear_config([:mrf_steal_emoji, :download_unknown_size], false)
+    mock_tesla()
+
+    refute "firedfox" in installed()
+
+    ExUnit.CaptureLog.capture_log(fn ->
+      assert {:ok, _message} = StealEmojiPolicy.filter(message)
+    end) =~
+      "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png: {:remote_size, false}"
+
+    refute "firedfox" in installed()
+  end
+
+  test "reject too large content-size before download", %{message: message} do
+    clear_config([:mrf_steal_emoji, :download_unknown_size], false)
+    mock_tesla("https://example.org/emoji/firedfox.png", 200, [{"content-length", 2 ** 30}])
+
+    refute "firedfox" in installed()
+
+    ExUnit.CaptureLog.capture_log(fn ->
+      assert {:ok, _message} = StealEmojiPolicy.filter(message)
+    end) =~
+      "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png: {:remote_size, false}"
+
+    refute "firedfox" in installed()
+  end
+
+  test "accepts content-size below limit", %{message: message} do
+    clear_config([:mrf_steal_emoji, :download_unknown_size], false)
+    mock_tesla("https://example.org/emoji/firedfox.png", 200, [{"content-length", 2}])
+
+    refute "firedfox" in installed()
+
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
+
+    assert "firedfox" in installed()
+  end
+
   defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
 end

test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs

@@ -11,6 +11,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do
   import Pleroma.Factory
 
   describe "attachments" do
+    test "works with apng" do
+      attachment =
+        %{
+          "mediaType" => "image/apng",
+          "name" => "",
+          "type" => "Document",
+          "url" =>
+            "https://media.misskeyusercontent.com/io/2859c26e-cd43-4550-848b-b6243bc3fe28.apng"
+        }
+
+      assert {:ok, attachment} =
+               AttachmentValidator.cast_and_validate(attachment)
+               |> Ecto.Changeset.apply_action(:insert)
+
+      assert attachment.mediaType == "image/apng"
+    end
+
     test "works with honkerific attachments" do
       attachment = %{
         "mediaType" => "",

test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs

@@ -64,6 +64,10 @@ test "mascot retrieving" do
 
     assert json_response_and_validate_schema(ret_conn, 200)
 
+    %{"url" => uploaded_url} = Jason.decode!(ret_conn.resp_body)
+
+    assert uploaded_url != nil and is_binary(uploaded_url)
+
     user = User.get_cached_by_id(user.id)
 
     conn =
@@ -72,6 +76,6 @@ test "mascot retrieving" do
       |> get("/api/v1/pleroma/mascot")
 
     assert %{"url" => url, "type" => "image"} = json_response_and_validate_schema(conn, 200)
-    assert url =~ "an_image"
+    assert url == uploaded_url
   end
 end

test/support/http_request_mock.ex

@@ -572,6 +572,7 @@ def get("https://social.stopwatchingus-heidelberg.de/.well-known/host-meta", _,
      }}
   end
 
+  # Mastodon status via display URL
   def get(
         "http://mastodon.example.org/@admin/99541947525187367",
         _,
@@ -581,6 +582,23 @@ def get(
     {:ok,
      %Tesla.Env{
        status: 200,
+       url: "http://mastodon.example.org/@admin/99541947525187367",
+       body: File.read!("test/fixtures/mastodon-note-object.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  # same status via its canonical ActivityPub id
+  def get(
+        "http://mastodon.example.org/users/admin/statuses/99541947525187367",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       url: "http://mastodon.example.org/users/admin/statuses/99541947525187367",
        body: File.read!("test/fixtures/mastodon-note-object.json"),
        headers: activitypub_object_headers()
      }}
@@ -964,7 +982,7 @@ def get("https://pleroma.local/notice/9kCP7V", _, _, _) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
   end
 
-  def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do
+  def get("http://remote.org/users/masto_closed/followers", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -973,7 +991,7 @@ def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do
      }}
   end
 
-  def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do
+  def get("http://remote.org/users/masto_closed/followers?page=1", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -982,7 +1000,7 @@ def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do
      }}
   end
 
-  def get("http://localhost:4001/users/masto_closed/following", _, _, _) do
+  def get("http://remote.org/users/masto_closed/following", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -991,7 +1009,7 @@ def get("http://localhost:4001/users/masto_closed/following", _, _, _) do
      }}
   end
 
-  def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do
+  def get("http://remote.org/users/masto_closed/following?page=1", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1000,7 +1018,7 @@ def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do
      }}
   end
 
-  def get("http://localhost:8080/followers/fuser3", _, _, _) do
+  def get("http://remote.org/followers/fuser3", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1009,7 +1027,7 @@ def get("http://localhost:8080/followers/fuser3", _, _, _) do
      }}
   end
 
-  def get("http://localhost:8080/following/fuser3", _, _, _) do
+  def get("http://remote.org/following/fuser3", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1018,7 +1036,7 @@ def get("http://localhost:8080/following/fuser3", _, _, _) do
      }}
   end
 
-  def get("http://localhost:4001/users/fuser2/followers", _, _, _) do
+  def get("http://remote.org/users/fuser2/followers", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1027,7 +1045,7 @@ def get("http://localhost:4001/users/fuser2/followers", _, _, _) do
      }}
   end
 
-  def get("http://localhost:4001/users/fuser2/following", _, _, _) do
+  def get("http://remote.org/users/fuser2/following", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,

test/test_helper.exs

@@ -9,6 +9,10 @@
 
 {:ok, _} = Application.ensure_all_started(:ex_machina)
 
+# Prepare and later automatically cleanup upload dir
+uploads_dir = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads")
+File.mkdir_p!(uploads_dir)
+
 ExUnit.after_suite(fn _results ->
   uploads = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads")
   File.rm_rf!(uploads)