diff --git a/CHANGELOG.md b/CHANGELOG.md index 84bb53820..162d25cdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,41 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 2024.03 + +## Added +- CLI tasks best-effort checking for past abuse of the recent spoofing exploit +- new `:mrf_steal_emoji, :download_unknown_size` option; defaults to `false` + +## Changed +- `Pleroma.Upload, :base_url` now MUST be configured explicitly if used; + use of the same domain as the instance is **strongly** discouraged +- `:media_proxy, :base_url` now MUST be configured explicitly if used; + use of the same domain as the instance is **strongly** discouraged +- StealEmoji: + - now uses the pack.json format; + existing users must migrate with an out-of-band script (check release notes) + - only steals shortcodes recognised as valid + - URLs of stolen emoji is no longer predictable +- The `Dedupe` upload filter is now always active; + `AnonymizeFilenames` is again opt-in +- received AP data is sanity checked before we attempt to parse it as a user +- Uploads, emoji and media proxy now restrict Content-Type headers to a safe subset +- Akkoma will no longer fetch and parse objects hosted on the same domain + +## Fixed +- Critical security issue allowing Akkoma to be used as a vector for + (depending on configuration) impersonation of other users or creation + of bogus users and posts on the upload domain +- Critical security issue letting Akkoma fall for the above impersonation + payloads due to lack of strict id checking +- Critical security issue allowing domains redirect to to pose as the initial domain + (e.g. with media proxy's fallback redirects) +- refetched objects can no longer attribute themselves to third-party actors + (this had no externally visible effect since actor info is read from the Create activity) +- our litepub JSON-LD schema is now served with the correct content type +- remote APNG attachments are now recognised as images + ## 2024.02 ## Added diff --git a/SECURITY.md b/SECURITY.md index c009d21d9..d37a8c9ca 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,16 +1,21 @@ -# Pleroma backend security policy - -## Supported versions - -Currently, Pleroma offers bugfixes and security patches only for the latest minor release. - -| Version | Support -|---------| -------- -| 2.2 | Bugfixes and security patches +# Akkoma backend security handling ## Reporting a vulnerability -Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities. +Please send an email (preferably encrypted) or +a DM via our IRC to one of the following people: + +| Forgejo nick | IRC nick | Email | GPG | +| ------------ | ------------- | ------------- | --------------------------------------- | +| floatinghost | FloatingGhost | *see GPG key* | https://coffee-and-dreams.uk/pubkey.asc | + ## Announcements -New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at . +New releases and security issues are announced at +[meta.akkoma.dev](https://meta.akkoma.dev/c/releases) and +[@akkoma@ihatebeinga.live](https://ihatebeinga.live/akkoma). + +Both also offer RSS feeds +([meta](https://meta.akkoma.dev/c/releases/7.rss), +[fedi](https://ihatebeinga.live/users/akkoma.rss)) +so you can keep an eye on it without any accounts. diff --git a/config/config.exs b/config/config.exs index 1c531344c..e0a5eccb1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -61,11 +61,12 @@ config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank. # Upload configuration config :pleroma, Pleroma.Upload, uploader: Pleroma.Uploaders.Local, - filters: [Pleroma.Upload.Filter.Dedupe], + filters: [], link_name: false, proxy_remote: false, filename_display_max_length: 30, - base_url: nil + base_url: nil, + allowed_mime_types: ["image", "audio", "video"] config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads" @@ -148,18 +149,38 @@ config :logger, :ex_syslogger, format: "$metadata[$level] $message", metadata: [:request_id] +# ——————————————————————————————————————————————————————————————— +# W A R N I N G +# ——————————————————————————————————————————————————————————————— +# +# Whenever adding a privileged new custom type for e.g. +# ActivityPub objects, ALWAYS map their extension back +# to "application/octet-stream". +# Else files served by us can automatically end up with +# those privileged types causing severe security hazards. +# (We need those mappings so Phoenix can assoiate its format +# (the "extension") to incoming requests of those MIME types) +# +# ——————————————————————————————————————————————————————————————— config :mime, :types, %{ "application/xml" => ["xml"], "application/xrd+xml" => ["xrd+xml"], "application/jrd+json" => ["jrd+json"], "application/activity+json" => ["activity+json"], - "application/ld+json" => ["activity+json"] + "application/ld+json" => ["activity+json"], + # Can be removed when bumping MIME past 2.0.5 + # see https://akkoma.dev/AkkomaGang/akkoma/issues/657 + "image/apng" => ["apng"] } config :mime, :extensions, %{ - "activity+json" => "application/activity+json" + "xrd+xml" => "text/plain", + "jrd+json" => "text/plain", + "activity+json" => "text/plain" } +# ——————————————————————————————————————————————————————————————— + config :tesla, :adapter, {Tesla.Adapter.Finch, name: MyFinch} # Configures http settings, upstream proxy etc. diff --git a/config/description.exs b/config/description.exs index e108aaae8..3ddd874f8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -105,6 +105,19 @@ config :pleroma, :config_description, [ "https://cdn-host.com" ] }, + %{ + key: :allowed_mime_types, + label: "Allowed MIME types", + type: {:list, :string}, + description: + "List of MIME (main) types uploads are allowed to identify themselves with. Other types may still be uploaded, but will identify as a generic binary to clients. WARNING: Loosening this over the defaults can lead to security issues. Removing types is safe, but only add to the list if you are sure you know what you are doing.", + suggestions: [ + "image", + "audio", + "video", + "font" + ] + }, %{ key: :proxy_remote, type: :boolean, diff --git a/docs/docs/administration/CLI_tasks/security.md b/docs/docs/administration/CLI_tasks/security.md new file mode 100644 index 000000000..a0208c4e5 --- /dev/null +++ b/docs/docs/administration/CLI_tasks/security.md @@ -0,0 +1,56 @@ +# Security-related tasks + +{! administration/CLI_tasks/general_cli_task_info.include !} + +!!! danger + Many of these tasks were written in response to a patched exploit. + It is recommended to run those very soon after installing its respective security update. + Over time with db migrations they might become less accurate or be removed altogether. + If you never ran an affected version, there’s no point in running them. + +## Spoofed AcitivityPub objects exploit (2024-03, fixed in 3.11.1) + +### Search for uploaded spoofing payloads + +Scans local uploads for spoofing payloads. +If the instance is not using the local uploader it was not affected. +Attachments wil be scanned anyway in case local uploader was used in the past. + +!!! note + This cannot reliably detect payloads attached to deleted posts. + +=== "OTP" + + ```sh + ./bin/pleroma_ctl security spoof-uploaded + ``` + +=== "From Source" + + ```sh + mix pleroma.security spoof-uploaded + ``` + +### Search for counterfeit posts in database + +Scans all notes in the database for signs of being spoofed. + +!!! note + Spoofs targeting local accounts can be detected rather reliably + (with some restrictions documented in the task’s logs). + Counterfeit posts from remote users cannot. A best-effort attempt is made, but + a thorough attacker can avoid this and it may yield a small amount of false positives. + + Should you find counterfeit posts of local users, let other admins know so they can delete the too. + +=== "OTP" + + ```sh + ./bin/pleroma_ctl security spoof-inserted + ``` + +=== "From Source" + + ```sh + mix pleroma.security spoof-inserted + ``` diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 3c9113f88..c4259c6cf 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -236,7 +236,9 @@ config :pleroma, :mrf_user_allowlist, %{ #### :mrf_steal_emoji * `hosts`: List of hosts to steal emojis from * `rejected_shortcodes`: Regex-list of shortcodes to reject -* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk +* `size_limit`: File size limit (in bytes), checked before download if possible (and remote server honest), + otherwise or again checked before saving emoji to the disk +* `download_unknown_size`: whether to download an emoji when the remote server doesn’t report its size in advance #### :mrf_activity_expiration @@ -396,7 +398,8 @@ This section describe PWA manifest instance-specific values. Currently this opti ## :media_proxy * `enabled`: Enables proxying of remote media to the instance’s proxy -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. +* `base_url`: The base URL to access a user-uploaded file. + Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. * `whitelist`: List of hosts with scheme to bypass the mediaproxy (e.g. `https://example.com`) * `invalidation`: options for remove media from cache after delete object: @@ -597,8 +600,9 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th * `uploader`: Which one of the [uploaders](#uploaders) to use. * `filters`: List of [upload filters](#upload-filters) to use. -* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to host the media files via another domain or are using a 3rd party S3 provider. +* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers +* `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly. + Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. * `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it. * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30. @@ -638,17 +642,18 @@ config :ex_aws, :s3, ### Upload filters -#### Pleroma.Upload.Filter.AnonymizeFilename - -This filter replaces the filename (not the path) of an upload. For complete obfuscation, add -`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. - -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. - #### Pleroma.Upload.Filter.Dedupe +**Always** active; cannot be turned off. +Renames files to their hash and prevents duplicate files filling up the disk. No specific configuration. +#### Pleroma.Upload.Filter.AnonymizeFilename + +This filter replaces the declared filename (not the path) of an upload. + +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. + #### Pleroma.Upload.Filter.Exiftool This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact. diff --git a/docs/docs/configuration/hardening.md b/docs/docs/configuration/hardening.md index 521183f7d..f8ba048dd 100644 --- a/docs/docs/configuration/hardening.md +++ b/docs/docs/configuration/hardening.md @@ -17,6 +17,16 @@ This sets the Akkoma application server to only listen to the localhost interfac This sets the `secure` flag on Akkoma’s session cookie. This makes sure, that the cookie is only accepted over encrypted HTTPs connections. This implicitly renames the cookie from `pleroma_key` to `__Host-pleroma-key` which enforces some restrictions. (see [cookie prefixes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#Cookie_prefixes)) +### `Pleroma.Upload, :uploader, :base_url` + +> Recommended value: *anything on a different domain than the instance endpoint; e.g. https://media.myinstance.net/* + +Uploads are user controlled and (unless you’re running a true single-user +instance) should therefore not be considered trusted. But the domain is used +as a pivilege boundary e.g. by HTTP content security policy and ActivityPub. +Having uploads on the same domain enabled several past vulnerabilities +able to be exploited by malicious users. + ### `:http_security` > Recommended value: `true` diff --git a/docs/docs/configuration/howto_mediaproxy.md b/docs/docs/configuration/howto_mediaproxy.md index 8ad81bdfb..223ad7eed 100644 --- a/docs/docs/configuration/howto_mediaproxy.md +++ b/docs/docs/configuration/howto_mediaproxy.md @@ -6,7 +6,16 @@ With the `mediaproxy` function you can use nginx to cache this content, so users ## Activate it -* Edit your nginx config and add the following location: +* Edit your nginx config and add the following location to your main server block: +``` +location /proxy { + return 404; +} +``` + +* Set up a subdomain for the proxy with its nginx config on the same machine + *(the latter is not strictly required, but for simplicity we’ll assume so)* +* In this subdomain’s server block add ``` location /proxy { proxy_cache akkoma_media_cache; @@ -26,9 +35,9 @@ config :pleroma, :media_proxy, enabled: true, proxy_opts: [ redirect_on_failure: true - ] - #base_url: "https://cache.akkoma.social" + ], + base_url: "https://cache.akkoma.social" ``` -If you want to use a subdomain to serve the files, uncomment `base_url`, change the url and add a comma after `true` in the previous line. +You **really** should use a subdomain to serve proxied files; while we will fix bugs resulting from this, serving arbitrary remote content on your main domain namespace is a significant attack surface. * Restart nginx and Akkoma diff --git a/installation/nginx/akkoma.nginx b/installation/nginx/akkoma.nginx index 18d92f30f..1d91ce22f 100644 --- a/installation/nginx/akkoma.nginx +++ b/installation/nginx/akkoma.nginx @@ -75,9 +75,48 @@ server { proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + location ~ ^/(media|proxy) { + return 404; + } + location / { proxy_pass http://phoenix; } +} + +# Upload and MediaProxy Subdomain +# (see main domain setup for more details) +server { + server_name media.example.tld; + + listen 80; + listen [::]:80; + + location / { + return 301 https://$server_name$request_uri; + } +} + +server { + server_name media.example.tld; + + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_trusted_certificate /etc/letsencrypt/live/media.example.tld/chain.pem; + ssl_certificate /etc/letsencrypt/live/media.example.tld/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/media.example.tld/privkey.pem; + # .. copy all other the ssl_* and gzip_* stuff from main domain + + # the nginx default is 1m, not enough for large media uploads + client_max_body_size 16m; + ignore_invalid_headers off; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; location ~ ^/(media|proxy) { proxy_cache akkoma_media_cache; @@ -91,4 +130,8 @@ server { chunked_transfer_encoding on; proxy_pass http://phoenix; } + + location / { + return 404; + } } diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 6a7d4f0d3..44f6b6e70 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -20,6 +20,7 @@ defmodule Mix.Tasks.Pleroma.Instance do output: :string, output_psql: :string, domain: :string, + media_url: :string, instance_name: :string, admin_email: :string, notify_email: :string, @@ -35,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Instance do listen_ip: :string, listen_port: :string, strip_uploads: :string, - anonymize_uploads: :string, - dedupe_uploads: :string + anonymize_uploads: :string ], aliases: [ o: :output, @@ -64,6 +64,14 @@ defmodule Mix.Tasks.Pleroma.Instance do ":" ) ++ [443] + media_url = + get_option( + options, + :media_url, + "What base url will uploads use? (e.g https://media.example.com/media)\n" <> + " Generally this should NOT use the same domain as the instance " + ) + name = get_option( options, @@ -186,14 +194,6 @@ defmodule Mix.Tasks.Pleroma.Instance do "n" ) === "y" - dedupe_uploads = - get_option( - options, - :dedupe_uploads, - "Do you want to deduplicate uploaded files? (y/n)", - "n" - ) === "y" - Config.put([:instance, :static_dir], static_dir) secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64) @@ -207,6 +207,7 @@ defmodule Mix.Tasks.Pleroma.Instance do EEx.eval_file( template_dir <> "/sample_config.eex", domain: domain, + media_url: media_url, port: port, email: email, notify_email: notify_email, @@ -230,8 +231,7 @@ defmodule Mix.Tasks.Pleroma.Instance do upload_filters: upload_filters(%{ strip: strip_uploads, - anonymize: anonymize_uploads, - dedupe: dedupe_uploads + anonymize: anonymize_uploads }) ) @@ -319,13 +319,6 @@ defmodule Mix.Tasks.Pleroma.Instance do enabled_filters end - enabled_filters = - if filters.dedupe do - enabled_filters ++ [Pleroma.Upload.Filter.Dedupe] - else - enabled_filters - end - enabled_filters end end diff --git a/lib/mix/tasks/pleroma/security.ex b/lib/mix/tasks/pleroma/security.ex new file mode 100644 index 000000000..f039e0980 --- /dev/null +++ b/lib/mix/tasks/pleroma/security.ex @@ -0,0 +1,330 @@ +# Akkoma: Magically expressive social media +# Copyright © 2024 Akkoma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Security do + use Mix.Task + import Ecto.Query + import Mix.Pleroma + + alias Pleroma.Config + + require Logger + + @shortdoc """ + Security-related tasks, like e.g. checking for signs past exploits were abused. + """ + + # Constants etc + defp local_id_prefix(), do: Pleroma.Web.Endpoint.url() <> "/" + + defp local_id_pattern(), do: local_id_prefix() <> "%" + + @activity_exts ["activity+json", "activity%2Bjson"] + + defp activity_ext_url_patterns() do + for e <- @activity_exts do + for suf <- ["", "?%"] do + # Escape literal % for use in SQL patterns + ee = String.replace(e, "%", "\\%") + "%.#{ee}#{suf}" + end + end + |> List.flatten() + end + + # Search for malicious uploads exploiting the lack of Content-Type sanitisation from before 2024-03 + def run(["spoof-uploaded"]) do + Logger.put_process_level(self(), :notice) + start_pleroma() + + IO.puts(""" + +------------------------+ + | SPOOF SEARCH UPLOADS | + +------------------------+ + Checking if any uploads are using privileged types. + NOTE if attachment deletion is enabled, payloads used + in the past may no longer exist. + """) + + do_spoof_uploaded() + end + + # Fuzzy search for potentially counterfeit activities in the database resulting from the same exploit + def run(["spoof-inserted"]) do + Logger.put_process_level(self(), :notice) + start_pleroma() + + IO.puts(""" + +----------------------+ + | SPOOF SEARCH NOTES | + +----------------------+ + Starting fuzzy search for counterfeit activities. + NOTE this can not guarantee detecting all counterfeits + and may yield a small percentage of false positives. + """) + + do_spoof_inserted() + end + + # +-----------------------------+ + # | S P O O F - U P L O A D E D | + # +-----------------------------+ + defp do_spoof_uploaded() do + files = + case Config.get!([Pleroma.Upload, :uploader]) do + Pleroma.Uploaders.Local -> + uploads_search_spoofs_local_dir(Config.get!([Pleroma.Uploaders.Local, :uploads])) + + _ -> + IO.puts(""" + NOTE: + Not using local uploader; thus not affected by this exploit. + It's impossible to check for files, but in case local uploader was used before + or to check if anyone futilely attempted a spoof, notes will still be scanned. + """) + + [] + end + + emoji = uploads_search_spoofs_local_dir(Config.get!([:instance, :static_dir])) + + post_attachs = uploads_search_spoofs_notes() + + not_orphaned_urls = + post_attachs + |> Enum.map(fn {_u, _a, url} -> url end) + |> MapSet.new() + + orphaned_attachs = upload_search_orphaned_attachments(not_orphaned_urls) + + IO.puts("\nSearch concluded; here are the results:") + pretty_print_list_with_title(emoji, "Emoji") + pretty_print_list_with_title(files, "Uploaded Files") + pretty_print_list_with_title(post_attachs, "(Not Deleted) Post Attachments") + pretty_print_list_with_title(orphaned_attachs, "Orphaned Uploads") + + IO.puts(""" + In total found + #{length(emoji)} emoji + #{length(files)} uploads + #{length(post_attachs)} not deleted posts + #{length(orphaned_attachs)} orphaned attachments + """) + end + + defp uploads_search_spoofs_local_dir(dir) do + local_dir = String.replace_suffix(dir, "/", "") + + IO.puts("Searching for suspicious files in #{local_dir}...") + + glob_ext = "{" <> Enum.join(@activity_exts, ",") <> "}" + + Path.wildcard(local_dir <> "/**/*." <> glob_ext, match_dot: true) + |> Enum.map(fn path -> + String.replace_prefix(path, local_dir <> "/", "") + end) + |> Enum.sort() + end + + defp uploads_search_spoofs_notes() do + IO.puts("Now querying DB for posts with spoofing attachments. This might take a while...") + + patterns = [local_id_pattern() | activity_ext_url_patterns()] + + # if jsonb_array_elemsts in FROM can be used with normal Ecto functions, idk how + """ + SELECT DISTINCT a.data->>'actor', a.id, url->>'href' + FROM public.objects AS o JOIN public.activities AS a + ON o.data->>'id' = a.data->>'object', + jsonb_array_elements(o.data->'attachment') AS attachs, + jsonb_array_elements(attachs->'url') AS url + WHERE o.data->>'type' = 'Note' AND + o.data->>'id' LIKE $1::text AND ( + url->>'href' LIKE $2::text OR + url->>'href' LIKE $3::text OR + url->>'href' LIKE $4::text OR + url->>'href' LIKE $5::text + ) + ORDER BY a.data->>'actor', a.id, url->>'href'; + """ + |> Pleroma.Repo.query!(patterns, timeout: :infinity) + |> map_raw_id_apid_tuple() + end + + defp upload_search_orphaned_attachments(not_orphaned_urls) do + IO.puts(""" + Now querying DB for orphaned spoofing attachment (i.e. their post was deleted, + but if :cleanup_attachments was not enabled traces remain in the database) + This might take a bit... + """) + + patterns = activity_ext_url_patterns() + + """ + SELECT DISTINCT attach.id, url->>'href' + FROM public.objects AS attach, + jsonb_array_elements(attach.data->'url') AS url + WHERE (attach.data->>'type' = 'Image' OR + attach.data->>'type' = 'Document') + AND ( + url->>'href' LIKE $1::text OR + url->>'href' LIKE $2::text OR + url->>'href' LIKE $3::text OR + url->>'href' LIKE $4::text + ) + ORDER BY attach.id, url->>'href'; + """ + |> Pleroma.Repo.query!(patterns, timeout: :infinity) + |> then(fn res -> Enum.map(res.rows, fn [id, url] -> {id, url} end) end) + |> Enum.filter(fn {_, url} -> !(url in not_orphaned_urls) end) + end + + # +-----------------------------+ + # | S P O O F - I N S E R T E D | + # +-----------------------------+ + defp do_spoof_inserted() do + IO.puts(""" + Searching for local posts whose Create activity has no ActivityPub id... + This is a pretty good indicator, but only for spoofs of local actors + and only if the spoofing happened after around late 2021. + """) + + idless_create = + search_local_notes_without_create_id() + |> Enum.sort() + + IO.puts("Done.\n") + + IO.puts(""" + Now trying to weed out other poorly hidden spoofs. + This can't detect all and may have some false positives. + """) + + likely_spoofed_posts_set = MapSet.new(idless_create) + + sus_pattern_posts = + search_sus_notes_by_id_patterns() + |> Enum.filter(fn r -> !(r in likely_spoofed_posts_set) end) + + IO.puts("Done.\n") + + IO.puts(""" + Finally, searching for spoofed, local user accounts. + (It's impossible to detect spoofed remote users) + """) + + spoofed_users = search_bogus_local_users() + + pretty_print_list_with_title(sus_pattern_posts, "Maybe Spoofed Posts") + pretty_print_list_with_title(idless_create, "Likely Spoofed Posts") + pretty_print_list_with_title(spoofed_users, "Spoofed local user accounts") + + IO.puts(""" + In total found: + #{length(spoofed_users)} bogus users + #{length(idless_create)} likely spoofed posts + #{length(sus_pattern_posts)} maybe spoofed posts + """) + end + + defp search_local_notes_without_create_id() do + Pleroma.Object + |> where([o], fragment("?->>'id' LIKE ?", o.data, ^local_id_pattern())) + |> join(:inner, [o], a in Pleroma.Activity, + on: fragment("?->>'object' = ?->>'id'", a.data, o.data) + ) + |> where([o, a], fragment("NOT (? \\? 'id') OR ?->>'id' IS NULL", a.data, a.data)) + |> select([o, a], {a.id, fragment("?->>'id'", o.data)}) + |> order_by([o, a], a.id) + |> Pleroma.Repo.all(timeout: :infinity) + end + + defp search_sus_notes_by_id_patterns() do + [ep1, ep2, ep3, ep4] = activity_ext_url_patterns() + + Pleroma.Object + |> where( + [o], + # for local objects we know exactly how a genuine id looks like + # (though a thorough attacker can emulate this) + # for remote posts, use some best-effort patterns + fragment( + """ + (?->>'id' LIKE ? AND ?->>'id' NOT SIMILAR TO + ? || 'objects/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}') + """, + o.data, + ^local_id_pattern(), + o.data, + ^local_id_prefix() + ) or + fragment("?->>'id' LIKE ?", o.data, "%/emoji/%") or + fragment("?->>'id' LIKE ?", o.data, "%/media/%") or + fragment("?->>'id' LIKE ?", o.data, "%/proxy/%") or + fragment("?->>'id' LIKE ?", o.data, ^ep1) or + fragment("?->>'id' LIKE ?", o.data, ^ep2) or + fragment("?->>'id' LIKE ?", o.data, ^ep3) or + fragment("?->>'id' LIKE ?", o.data, ^ep4) + ) + |> join(:inner, [o], a in Pleroma.Activity, + on: fragment("?->>'object' = ?->>'id'", a.data, o.data) + ) + |> select([o, a], {a.id, fragment("?->>'id'", o.data)}) + |> order_by([o, a], a.id) + |> Pleroma.Repo.all(timeout: :infinity) + end + + defp search_bogus_local_users() do + Pleroma.User.Query.build(%{}) + |> where([u], u.local == false and like(u.ap_id, ^local_id_pattern())) + |> order_by([u], u.ap_id) + |> select([u], u.ap_id) + |> Pleroma.Repo.all(timeout: :infinity) + end + + # +-----------------------------------+ + # | module-specific utility functions | + # +-----------------------------------+ + defp pretty_print_list_with_title(list, title) do + title_len = String.length(title) + title_underline = String.duplicate("=", title_len) + IO.puts(title) + IO.puts(title_underline) + pretty_print_list(list) + end + + defp pretty_print_list([]), do: IO.puts("") + + defp pretty_print_list([{a, o} | rest]) + when (is_binary(a) or is_number(a)) and is_binary(o) do + IO.puts(" {#{a}, #{o}}") + pretty_print_list(rest) + end + + defp pretty_print_list([{u, a, o} | rest]) + when is_binary(a) and is_binary(u) and is_binary(o) do + IO.puts(" {#{u}, #{a}, #{o}}") + pretty_print_list(rest) + end + + defp pretty_print_list([e | rest]) when is_binary(e) do + IO.puts(" #{e}") + pretty_print_list(rest) + end + + defp pretty_print_list([e | rest]), do: pretty_print_list([inspect(e) | rest]) + + defp map_raw_id_apid_tuple(res) do + user_prefix = local_id_prefix() <> "users/" + + Enum.map(res.rows, fn + [uid, aid, oid] -> + { + String.replace_prefix(uid, user_prefix, ""), + FlakeId.to_string(aid), + oid + } + end) + end +end diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 9049d9097..142208854 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -26,12 +26,37 @@ defmodule Pleroma.Emoji.Pack do alias Pleroma.Emoji.Pack alias Pleroma.Utils + # Invalid/Malicious names are supposed to be filtered out before path joining, + # but there are many entrypoints to affected functions so as the code changes + # we might accidentally let an unsanitised name slip through. + # To make sure, use the below which crash the process otherwise. + + # ALWAYS use this when constructing paths from external name! + # (name meaning it must be only a single path component) + defp path_join_name_safe(dir, name) do + if to_string(name) != Path.basename(name) or name in ["..", ".", ""] do + raise "Invalid or malicious pack name: #{name}" + else + Path.join(dir, name) + end + end + + # ALWAYS use this to join external paths + # (which are allowed to have several components) + defp path_join_safe(dir, path) do + {:ok, safe_path} = Path.safe_relative(path) + Path.join(dir, safe_path) + end + @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values} def create(name) do with :ok <- validate_not_empty([name]), - dir <- Path.join(emoji_path(), name), + dir <- path_join_name_safe(emoji_path(), name), :ok <- File.mkdir(dir) do - save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")}) + save_pack(%__MODULE__{ + path: dir, + pack_file: Path.join(dir, "pack.json") + }) end end @@ -65,7 +90,7 @@ defmodule Pleroma.Emoji.Pack do {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values} def delete(name) do with :ok <- validate_not_empty([name]), - pack_path <- Path.join(emoji_path(), name) do + pack_path <- path_join_name_safe(emoji_path(), name) do File.rm_rf(pack_path) end end @@ -89,7 +114,7 @@ defmodule Pleroma.Emoji.Pack do end) end - @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) :: + @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t() | binary()) :: {:ok, t()} | {:error, File.posix() | atom()} def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do @@ -107,7 +132,7 @@ defmodule Pleroma.Emoji.Pack do Enum.map_reduce(emojies, pack, fn item, emoji_pack -> emoji_file = %Plug.Upload{ filename: item[:filename], - path: Path.join(tmp_dir, item[:path]) + path: path_join_safe(tmp_dir, item[:path]) } {:ok, updated_pack} = @@ -137,6 +162,14 @@ defmodule Pleroma.Emoji.Pack do end def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do + try_add_file(pack, shortcode, filename, file) + end + + def add_file(%Pack{} = pack, shortcode, filename, filedata) when is_binary(filedata) do + try_add_file(pack, shortcode, filename, filedata) + end + + defp try_add_file(%Pack{} = pack, shortcode, filename, file) do with :ok <- validate_not_empty([shortcode, filename]), :ok <- validate_emoji_not_exists(shortcode), {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do @@ -189,6 +222,7 @@ defmodule Pleroma.Emoji.Pack do {:ok, results} <- File.ls(emoji_path) do names = results + # items come from File.ls, thus safe |> Enum.map(&Path.join(emoji_path, &1)) |> Enum.reject(fn path -> File.dir?(path) and File.exists?(Path.join(path, "pack.json")) @@ -287,8 +321,8 @@ defmodule Pleroma.Emoji.Pack do @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()} def load_pack(name) do - name = Path.basename(name) - pack_file = Path.join([emoji_path(), name, "pack.json"]) + pack_dir = path_join_name_safe(emoji_path(), name) + pack_file = Path.join(pack_dir, "pack.json") with {:ok, _} <- File.stat(pack_file), {:ok, pack_data} <- File.read(pack_file) do @@ -412,7 +446,13 @@ defmodule Pleroma.Emoji.Pack do end defp create_archive_and_cache(pack, hash) do - files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)] + files = [ + ~c"pack.json" + | Enum.map(pack.files, fn {_, file} -> + {:ok, file} = Path.safe_relative(file) + to_charlist(file) + end) + ] {:ok, {_, result}} = :zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)]) @@ -474,7 +514,7 @@ defmodule Pleroma.Emoji.Pack do end defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do - file_path = Path.join(pack.path, filename) + file_path = path_join_safe(pack.path, filename) create_subdirs(file_path) with {:ok, _} <- File.copy(upload_path, file_path) do @@ -482,6 +522,12 @@ defmodule Pleroma.Emoji.Pack do end end + defp save_file(file_data, pack, filename) when is_binary(file_data) do + file_path = path_join_safe(pack.path, filename) + create_subdirs(file_path) + File.write(file_path, file_data, [:binary]) + end + defp put_emoji(pack, shortcode, filename) do files = Map.put(pack.files, shortcode, filename) %{pack | files: files, files_count: length(Map.keys(files))} @@ -493,8 +539,8 @@ defmodule Pleroma.Emoji.Pack do end defp rename_file(pack, filename, new_filename) do - old_path = Path.join(pack.path, filename) - new_path = Path.join(pack.path, new_filename) + old_path = path_join_safe(pack.path, filename) + new_path = path_join_safe(pack.path, new_filename) create_subdirs(new_path) with :ok <- File.rename(old_path, new_path) do @@ -512,7 +558,7 @@ defmodule Pleroma.Emoji.Pack do defp remove_file(pack, shortcode) do with {:ok, filename} <- get_filename(pack, shortcode), - emoji <- Path.join(pack.path, filename), + emoji <- path_join_safe(pack.path, filename), :ok <- File.rm(emoji) do remove_dir_if_empty(emoji, filename) end @@ -530,7 +576,7 @@ defmodule Pleroma.Emoji.Pack do defp get_filename(pack, shortcode) do with %{^shortcode => filename} when is_binary(filename) <- pack.files, - file_path <- Path.join(pack.path, filename), + file_path <- path_join_safe(pack.path, filename), {:ok, _} <- File.stat(file_path) do {:ok, filename} else @@ -568,7 +614,7 @@ defmodule Pleroma.Emoji.Pack do end defp copy_as(remote_pack, local_name) do - path = Path.join(emoji_path(), local_name) + path = path_join_name_safe(emoji_path(), local_name) %__MODULE__{ name: local_name, diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 040537acf..37bc20e4d 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -11,6 +11,9 @@ defmodule Pleroma.Object.Containment do Object containment is an important step in validating remote objects to prevent spoofing, therefore removal of object containment functions is NOT recommended. """ + + alias Pleroma.Web.ActivityPub.Transmogrifier + def get_actor(%{"actor" => actor}) when is_binary(actor) do actor end @@ -47,6 +50,31 @@ defmodule Pleroma.Object.Containment do defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok defp compare_uris(_id_uri, _other_uri), do: :error + defp compare_uris_exact(uri, uri), do: :ok + + defp compare_uris_exact(%URI{} = id, %URI{} = other), + do: compare_uris_exact(URI.to_string(id), URI.to_string(other)) + + defp compare_uris_exact(id_uri, other_uri) + when is_binary(id_uri) and is_binary(other_uri) do + norm_id = String.replace_suffix(id_uri, "/", "") + norm_other = String.replace_suffix(other_uri, "/", "") + if norm_id == norm_other, do: :ok, else: :error + end + + @doc """ + Checks whether an URL to fetch from is from the local server. + + We never want to fetch from ourselves; if it’s not in the database + it can’t be authentic and must be a counterfeit. + """ + def contain_local_fetch(id) do + case compare_uris(URI.parse(id), Pleroma.Web.Endpoint.struct_url()) do + :ok -> :error + _ -> :ok + end + end + @doc """ Checks that an imported AP object's actor matches the host it came from. """ @@ -62,8 +90,31 @@ defmodule Pleroma.Object.Containment do def contain_origin(id, %{"attributedTo" => actor} = params), do: contain_origin(id, Map.put(params, "actor", actor)) - def contain_origin(_id, _data), do: :error + def contain_origin(_id, _data), do: :ok + @doc """ + Check whether the fetch URL (after redirects) exactly (sans tralining slash) matches either + the canonical ActivityPub id or the objects url field (for display URLs from *key and Mastodon) + + Since this is meant to be used for fetches, anonymous or transient objects are not accepted here. + """ + def contain_id_to_fetch(url, %{"id" => id} = data) when is_binary(id) do + with {:id, :error} <- {:id, compare_uris_exact(id, url)}, + # "url" can be a "Link" object and this is checked before full normalisation + display_url <- Transmogrifier.fix_url(data)["url"], + true <- display_url != nil do + compare_uris_exact(display_url, url) + else + {:id, :ok} -> :ok + _ -> :error + end + end + + def contain_id_to_fetch(_url, _data), do: :error + + @doc """ + Check whether the object id is from the same host as another id + """ def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(other_id) do id_uri = URI.parse(id) other_uri = URI.parse(other_id) @@ -85,4 +136,12 @@ defmodule Pleroma.Object.Containment do do: contain_origin(id, object) def contain_child(_), do: :ok + + @doc "Checks whether two URIs belong to the same domain" + def same_origin(id1, id2) do + uri1 = URI.parse(id1) + uri2 = URI.parse(id2) + + compare_uris(uri1, uri2) + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9e62ca69f..6609b8c1a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -18,6 +18,16 @@ defmodule Pleroma.Object.Fetcher do require Logger require Pleroma.Constants + @moduledoc """ + This module deals with correctly fetching Acitivity Pub objects in a safe way. + + The core function is `fetch_and_contain_remote_object_from_id/1` which performs + the actual fetch and common safety and authenticity checks. Other `fetch_*` + function use the former and perform some additional tasks + """ + + @mix_env Mix.env() + defp touch_changeset(changeset) do updated_at = NaiveDateTime.utc_now() @@ -103,18 +113,26 @@ defmodule Pleroma.Object.Fetcher do end end + @doc "Assumes object already is in our database and refetches from remote to update (e.g. for polls)" def refetch_object(%Object{data: %{"id" => id}} = object) do with {:local, false} <- {:local, Object.local?(object)}, {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id), + {:id, true} <- {:id, new_data["id"] == id}, {:ok, object} <- reinject_object(object, new_data) do {:ok, object} else {:local, true} -> {:ok, object} + {:id, false} -> {:error, "Object id changed on refetch"} e -> {:error, e} end end - # Note: will create a Create activity, which we need internally at the moment. + @doc """ + Fetches a new object and puts it through the processing pipeline for inbound objects + + Note: will also insert a fake Create activity, since atm we internally + need everything to be traced back to a Create activity. + """ def fetch_object_from_id(id, options \\ []) do with %URI{} = uri <- URI.parse(id), # let's check the URI is even vaguely valid first @@ -127,7 +145,6 @@ defmodule Pleroma.Object.Fetcher do {_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)}, {_, nil} <- {:normalize, Object.normalize(data, fetch: false)}, params <- prepare_activity_params(data), - {_, :ok} <- {:containment, Containment.contain_origin(id, params)}, {_, {:ok, activity}} <- {:transmogrifier, Transmogrifier.handle_incoming(params, options)}, {_, _data, %Object{} = object} <- @@ -140,9 +157,6 @@ defmodule Pleroma.Object.Fetcher do {:scheme, false} -> {:error, "URI Scheme Invalid"} - {:containment, _} -> - {:error, "Object containment failed."} - {:transmogrifier, {:error, {:reject, e}}} -> {:reject, e} @@ -185,6 +199,7 @@ defmodule Pleroma.Object.Fetcher do |> Maps.put_if_present("bcc", data["bcc"]) end + @doc "Identical to `fetch_object_from_id/2` but just directly returns the object or on error `nil`" def fetch_object_from_id!(id, options \\ []) do with {:ok, object} <- fetch_object_from_id(id, options) do object @@ -235,6 +250,7 @@ defmodule Pleroma.Object.Fetcher do end end + @doc "Fetches arbitrary remote object and performs basic safety and authenticity checks" def fetch_and_contain_remote_object_from_id(id) def fetch_and_contain_remote_object_from_id(%{"id" => id}), @@ -244,18 +260,29 @@ defmodule Pleroma.Object.Fetcher do Logger.debug("Fetching object #{id} via AP") with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")}, - {:ok, body} <- get_object(id), + {_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)}, + {:ok, final_id, body} <- get_object(id), {:ok, data} <- safe_json_decode(body), - :ok <- Containment.contain_origin_from_id(id, data) do - unless Instances.reachable?(id) do - Instances.set_reachable(id) + {_, :ok} <- {:strict_id, Containment.contain_id_to_fetch(final_id, data)}, + {_, :ok} <- {:containment, Containment.contain_origin(final_id, data)} do + unless Instances.reachable?(final_id) do + Instances.set_reachable(final_id) end {:ok, data} else + {:strict_id, _} -> + {:error, "Object's ActivityPub id/url does not match final fetch URL"} + {:scheme, _} -> {:error, "Unsupported URI scheme"} + {:local_fetch, _} -> + {:error, "Trying to fetch local resource"} + + {:containment, _} -> + {:error, "Object containment failed."} + {:error, e} -> {:error, e} @@ -267,6 +294,32 @@ defmodule Pleroma.Object.Fetcher do def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"} + defp check_crossdomain_redirect(final_host, original_url) + + # HOPEFULLY TEMPORARY + # Basically none of our Tesla mocks in tests set the (supposed to + # exist for Tesla proper) url parameter for their responses + # causing almost every fetch in test to fail otherwise + if @mix_env == :test do + defp check_crossdomain_redirect(nil, _) do + {:cross_domain_redirect, false} + end + end + + defp check_crossdomain_redirect(final_host, original_url) do + {:cross_domain_redirect, final_host != URI.parse(original_url).host} + end + + if @mix_env == :test do + defp get_final_id(nil, initial_url), do: initial_url + defp get_final_id("", initial_url), do: initial_url + end + + defp get_final_id(final_url, _intial_url) do + final_url + end + + @doc "Do NOT use; only public for use in tests" def get_object(id) do date = Pleroma.Signature.signed_date() @@ -275,37 +328,42 @@ defmodule Pleroma.Object.Fetcher do |> maybe_date_fetch(date) |> sign_fetch(id, date) - case HTTP.get(id, headers) do - {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 -> - case List.keyfind(headers, "content-type", 0) do - {_, content_type} -> - case Plug.Conn.Utils.media_type(content_type) do - {:ok, "application", "activity+json", _} -> - {:ok, body} + with {:ok, %{body: body, status: code, headers: headers, url: final_url}} + when code in 200..299 <- + HTTP.get(id, headers), + remote_host <- + URI.parse(final_url).host, + {:cross_domain_redirect, false} <- + check_crossdomain_redirect(remote_host, id), + {:has_content_type, {_, content_type}} <- + {:has_content_type, List.keyfind(headers, "content-type", 0)}, + {:parse_content_type, {:ok, "application", subtype, type_params}} <- + {:parse_content_type, Plug.Conn.Utils.media_type(content_type)} do + final_id = get_final_id(final_url, id) - {:ok, "application", "ld+json", - %{"profile" => "https://www.w3.org/ns/activitystreams"}} -> - {:ok, body} + case {subtype, type_params} do + {"activity+json", _} -> + {:ok, final_id, body} - # pixelfed sometimes (and only sometimes) responds with http instead of https - {:ok, "application", "ld+json", - %{"profile" => "http://www.w3.org/ns/activitystreams"}} -> - {:ok, body} - - _ -> - {:error, {:content_type, content_type}} - end - - _ -> - {:error, {:content_type, nil}} - end + {"ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} -> + {:ok, final_id, body} + _ -> + {:error, {:content_type, content_type}} + end + else {:ok, %{status: code}} when code in [404, 410] -> {:error, {"Object has been deleted", id, code}} {:error, e} -> {:error, e} + {:has_content_type, _} -> + {:error, {:content_type, nil}} + + {:parse_content_type, e} -> + {:error, {:content_type, e}} + e -> {:error, e} end diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index bb4f4def3..f017bf51b 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -17,6 +17,8 @@ defmodule Pleroma.ReverseProxy do @failed_request_ttl :timer.seconds(60) @methods ~w(GET HEAD) + @allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], []) + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def max_read_duration_default, do: @max_read_duration @@ -253,6 +255,7 @@ defmodule Pleroma.ReverseProxy do headers |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end) |> build_resp_cache_headers(opts) + |> sanitise_content_type() |> build_resp_content_disposition_header(opts) |> build_csp_headers() |> Keyword.merge(Keyword.get(opts, :resp_headers, [])) @@ -282,6 +285,21 @@ defmodule Pleroma.ReverseProxy do end end + defp sanitise_content_type(headers) do + original_ct = get_content_type(headers) + + safe_ct = + Pleroma.Web.Plugs.Utils.get_safe_mime_type( + %{allowed_mime_types: @allowed_mime_types}, + original_ct + ) + + [ + {"content-type", safe_ct} + | Enum.filter(headers, fn {k, _v} -> k != "content-type" end) + ] + end + defp build_resp_content_disposition_header(headers, opts) do opt = Keyword.get(opts, :inline_content_types, @inline_content_types) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 99b6b5215..1158e9449 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -39,6 +39,8 @@ defmodule Pleroma.Upload do alias Pleroma.Web.ActivityPub.Utils require Logger + @mix_env Mix.env() + @type source :: Plug.Upload.t() | (data_uri_string :: String.t()) @@ -64,7 +66,7 @@ defmodule Pleroma.Upload do path: String.t() } - @always_enabled_filters [Pleroma.Upload.Filter.AnonymizeFilename] + @always_enabled_filters [Pleroma.Upload.Filter.Dedupe] defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path] @@ -228,6 +230,13 @@ defmodule Pleroma.Upload do defp url_from_spec(_upload, _base_url, {:url, url}), do: url + if @mix_env == :test do + defp choose_base_url(prim, sec \\ nil), + do: prim || sec || Pleroma.Web.Endpoint.url() <> "/media/" + else + defp choose_base_url(prim, sec \\ nil), do: prim || sec + end + def base_url do uploader = Config.get([Pleroma.Upload, :uploader]) upload_base_url = Config.get([Pleroma.Upload, :base_url]) @@ -235,7 +244,7 @@ defmodule Pleroma.Upload do case uploader do Pleroma.Uploaders.Local -> - upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" + choose_base_url(upload_base_url) Pleroma.Uploaders.S3 -> bucket = Config.get([Pleroma.Uploaders.S3, :bucket]) @@ -261,7 +270,7 @@ defmodule Pleroma.Upload do end _ -> - public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" + choose_base_url(public_endpoint, upload_base_url) end end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4a8ce2d3d..1e06bc809 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -22,6 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Upload alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF + alias Pleroma.Web.ActivityPub.ObjectValidators.UserValidator alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger @@ -1722,6 +1723,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), + {:valid, {:ok, _, _}} <- {:valid, UserValidator.validate(data, [])}, {:ok, data} <- user_data_from_user_object(data, additional) do {:ok, maybe_update_follow_information(data)} else @@ -1734,6 +1736,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.debug("Rejected user #{ap_id}: #{inspect(reason)}") {:error, e} + {:valid, reason} -> + Logger.debug("Data is not a valid user #{ap_id}: #{inspect(reason)}") + {:error, "Not a user"} + {:error, e} -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") {:error, e} @@ -1834,6 +1840,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + user = + if data.ap_id != ap_id do + User.get_cached_by_ap_id(data.ap_id) + else + user + end + if user do user |> User.remote_user_changeset(data) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 02a107c27..26d3dc592 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -6,10 +6,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do require Logger alias Pleroma.Config + alias Pleroma.Emoji.Pack @moduledoc "Detect new emojis by their shortcode and steals them" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @pack_name "stolen" + + # Config defaults + @size_limit 50_000 + @download_unknown_size false + + defp create_pack() do + with {:ok, pack} = Pack.create(@pack_name) do + Pack.save_metadata( + %{ + "description" => "Collection of emoji auto-stolen from other instances", + "homepage" => Pleroma.Web.Endpoint.url(), + "can-download" => false, + "share-files" => false + }, + pack + ) + end + end + + defp load_or_create_pack() do + case Pack.load_pack(@pack_name) do + {:ok, pack} -> {:ok, pack} + {:error, :enoent} -> create_pack() + e -> e + end + end + + defp add_emoji(shortcode, extension, filedata) do + {:ok, pack} = load_or_create_pack() + # Make final path infeasible to predict to thwart certain kinds of attacks + # (48 bits is slighty more than 8 base62 chars, thus 9 chars) + salt = + :crypto.strong_rand_bytes(6) + |> :crypto.bytes_to_integer() + |> Base62.encode() + |> String.pad_leading(9, "0") + + filename = shortcode <> "-" <> salt <> "." <> extension + + Pack.add_file(pack, shortcode, filename, filedata) + end + defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do @@ -20,31 +64,69 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do String.match?(shortcode, pattern) end - defp steal_emoji({shortcode, url}, emoji_dir_path) do + defp reject_emoji?({shortcode, _url}, installed_emoji) do + valid_shortcode? = String.match?(shortcode, ~r/^[a-zA-Z0-9_-]+$/) + + rejected_shortcode? = + [:mrf_steal_emoji, :rejected_shortcodes] + |> Config.get([]) + |> Enum.any?(fn pattern -> shortcode_matches?(shortcode, pattern) end) + + emoji_installed? = Enum.member?(installed_emoji, shortcode) + + !valid_shortcode? or rejected_shortcode? or emoji_installed? + end + + defp steal_emoji(%{} = response, {shortcode, extension}) do + case add_emoji(shortcode, extension, response.body) do + {:ok, _} -> + shortcode + + e -> + Logger.warning( + "MRF.StealEmojiPolicy: Failed to add #{shortcode} as #{extension}: #{inspect(e)}" + ) + + nil + end + end + + defp get_extension_if_safe(response) do + content_type = + :proplists.get_value("content-type", response.headers, MIME.from_path(response.url)) + + case content_type do + "image/" <> _ -> List.first(MIME.extensions(content_type)) + _ -> nil + end + end + + defp is_remote_size_within_limit?(url) do + with {:ok, %{status: status, headers: headers} = _response} when status in 200..299 <- + Pleroma.HTTP.request(:head, url, nil, [], []) do + content_length = :proplists.get_value("content-length", headers, nil) + size_limit = Config.get([:mrf_steal_emoji, :size_limit], @size_limit) + + accept_unknown = + Config.get([:mrf_steal_emoji, :download_unknown_size], @download_unknown_size) + + content_length <= size_limit or + (content_length == nil and accept_unknown) + else + _ -> false + end + end + + defp maybe_steal_emoji({shortcode, url}) do url = Pleroma.Web.MediaProxy.url(url) - with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do - size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) + with {:remote_size, true} <- {:remote_size, is_remote_size_within_limit?(url)}, + {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do + size_limit = Config.get([:mrf_steal_emoji, :size_limit], @size_limit) + extension = get_extension_if_safe(response) - if byte_size(response.body) <= size_limit do - extension = - url - |> URI.parse() - |> Map.get(:path) - |> Path.basename() - |> Path.extname() - - shortcode = Path.basename(shortcode) - file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) - - case File.write(file_path, response.body) do - :ok -> - shortcode - - e -> - Logger.warning("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") - nil - end + if byte_size(response.body) <= size_limit and extension do + steal_emoji(response, {shortcode, extension}) else Logger.debug( "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{size_limit} B)" @@ -66,29 +148,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - emoji_dir_path = - Config.get( - [:mrf_steal_emoji, :path], - Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") - ) - - File.mkdir_p(emoji_dir_path) - new_emojis = foreign_emojis - |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) - |> Enum.reject(fn {shortcode, _url} -> - String.contains?(shortcode, ["/", "\\", ".", ":"]) - end) - |> Enum.filter(fn {shortcode, _url} -> - reject_emoji? = - [:mrf_steal_emoji, :rejected_shortcodes] - |> Config.get([]) - |> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end) - - !reject_emoji? - end) - |> Enum.map(&steal_emoji(&1, emoji_dir_path)) + |> Enum.reject(&reject_emoji?(&1, installed_emoji)) + |> Enum.map(&maybe_steal_emoji(&1)) |> Enum.filter(& &1) if !Enum.empty?(new_emojis) do diff --git a/lib/pleroma/web/activity_pub/object_validators/user_validator.ex b/lib/pleroma/web/activity_pub/object_validators/user_validator.ex new file mode 100644 index 000000000..90b5404f3 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/user_validator.ex @@ -0,0 +1,92 @@ +# Akkoma: Magically expressive social media +# Copyright © 2024 Akkoma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.UserValidator do + @moduledoc """ + Checks whether ActivityPub data represents a valid user + + Users don't go through the same ingest pipeline like activities or other objects. + To ensure this can only match a user and no users match in the other pipeline, + this is a separate from the generic ObjectValidator. + """ + + @behaviour Pleroma.Web.ActivityPub.ObjectValidator.Validating + + alias Pleroma.Object.Containment + alias Pleroma.Signature + + @impl true + def validate(object, meta) + + def validate(%{"type" => type, "id" => _id} = data, meta) + when type in ["Person", "Organization", "Group", "Application"] do + with :ok <- validate_pubkey(data), + :ok <- validate_inbox(data), + :ok <- contain_collection_origin(data) do + {:ok, data, meta} + else + {:error, e} -> {:error, e} + e -> {:error, e} + end + end + + def validate(_, _), do: {:error, "Not a user object"} + + defp mabye_validate_owner(nil, _actor), do: :ok + defp mabye_validate_owner(actor, actor), do: :ok + defp mabye_validate_owner(_owner, _actor), do: :error + + defp validate_pubkey( + %{"id" => id, "publicKey" => %{"id" => pk_id, "publicKeyPem" => _key}} = data + ) + when id != nil do + with {_, {:ok, kactor}} <- {:key, Signature.key_id_to_actor_id(pk_id)}, + true <- id == kactor, + :ok <- mabye_validate_owner(Map.get(data, "owner"), id) do + :ok + else + {:key, _} -> + {:error, "Unable to determine actor id from key id"} + + false -> + {:error, "Key id does not relate to user id"} + + _ -> + {:error, "Actor does not own its public key"} + end + end + + # pubkey is optional atm + defp validate_pubkey(_data), do: :ok + + defp validate_inbox(%{"id" => id, "inbox" => inbox}) do + case Containment.same_origin(id, inbox) do + :ok -> :ok + :error -> {:error, "Inbox on different doamin"} + end + end + + defp validate_inbox(_), do: {:error, "No inbox"} + + defp check_field_value(%{"id" => id} = _data, value) do + Containment.same_origin(id, value) + end + + defp maybe_check_field(data, field) do + with val when val != nil <- data[field], + :ok <- check_field_value(data, val) do + :ok + else + nil -> :ok + _ -> {:error, "#{field} on different domain"} + end + end + + defp contain_collection_origin(data) do + Enum.reduce(["followers", "following", "featured"], :ok, fn + field, :ok -> maybe_check_field(data, field) + _, error -> error + end) + end +end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 64593767d..6628fcaf3 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -98,6 +98,10 @@ defmodule Pleroma.Web.Endpoint do at: "/", from: :pleroma, only: Pleroma.Web.static_paths(), + # JSON-LD is accepted by some servers for AP objects and activities, + # thus only enable it here instead of a global extension mapping + # (it's our only *.jsonld file anyway) + content_types: %{"litepub-0.1.jsonld" => "application/ld+json"}, # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength gzip: true, cache_control_for_etags: @static_cache_control, diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index c5087c42c..19411d58e 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Web.MediaProxy do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @mix_env Mix.env() + def cache_table, do: @cache_table @spec in_banned_urls(String.t()) :: boolean() @@ -144,8 +146,14 @@ defmodule Pleroma.Web.MediaProxy do if path = URI.parse(url_or_path).path, do: Path.basename(path) end - def base_url do - Config.get([:media_proxy, :base_url], Endpoint.url()) + if @mix_env == :test do + def base_url do + Config.get([:media_proxy, :base_url], Endpoint.url()) + end + else + def base_url do + Config.get!([:media_proxy, :base_url]) + end end defp proxy_url(path, sig_base64, url_base64, filename) do diff --git a/lib/pleroma/web/plugs/instance_static.ex b/lib/pleroma/web/plugs/instance_static.ex index 5f9a6ee83..b72b604a1 100644 --- a/lib/pleroma/web/plugs/instance_static.ex +++ b/lib/pleroma/web/plugs/instance_static.ex @@ -3,8 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.InstanceStatic do + import Plug.Conn + require Pleroma.Constants + alias Pleroma.Web.Plugs.Utils + @moduledoc """ This is a shim to call `Plug.Static` but with runtime `from` configuration. @@ -43,11 +47,25 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do conn end - defp call_static(conn, opts, from) do + defp set_static_content_type(conn, "/emoji/" <> _ = request_path) do + real_mime = MIME.from_path(request_path) + safe_mime = Utils.get_safe_mime_type(%{allowed_mime_types: ["image"]}, real_mime) + + put_resp_header(conn, "content-type", safe_mime) + end + + defp set_static_content_type(conn, request_path) do + put_resp_header(conn, "content-type", MIME.from_path(request_path)) + end + + defp call_static(%{request_path: request_path} = conn, opts, from) do opts = opts |> Map.put(:from, from) + |> Map.put(:set_content_type, false) - Plug.Static.call(conn, opts) + conn + |> set_static_content_type(request_path) + |> Pleroma.Web.Plugs.StaticNoCT.call(opts) end end diff --git a/lib/pleroma/web/plugs/static_no_content_type.ex b/lib/pleroma/web/plugs/static_no_content_type.ex new file mode 100644 index 000000000..ea00a2d5d --- /dev/null +++ b/lib/pleroma/web/plugs/static_no_content_type.ex @@ -0,0 +1,469 @@ +# This is almost identical to Plug.Static from Plug 1.15.3 (2024-01-16) +# It being copied is a temporary measure to fix an urgent bug without +# needing to wait for merge of a suitable patch upstream +# The differences are: +# - this leading comment +# - renaming of the module from 'Plug.Static' to 'Pleroma.Web.Plugs.StaticNoCT' +# - additon of set_content_type option + +defmodule Pleroma.Web.Plugs.StaticNoCT do + @moduledoc """ + A plug for serving static assets. + + It requires two options: + + * `:at` - the request path to reach for static assets. + It must be a string. + + * `:from` - the file system path to read static assets from. + It can be either: a string containing a file system path, an + atom representing the application name (where assets will + be served from `priv/static`), a tuple containing the + application name and the directory to serve assets from (besides + `priv/static`), or an MFA tuple. + + The preferred form is to use `:from` with an atom or tuple, since + it will make your application independent from the starting directory. + For example, if you pass: + + plug Plug.Static, from: "priv/app/path" + + Plug.Static will be unable to serve assets if you build releases + or if you change the current directory. Instead do: + + plug Plug.Static, from: {:app_name, "priv/app/path"} + + If a static asset cannot be found, `Plug.Static` simply forwards + the connection to the rest of the pipeline. + + ## Cache mechanisms + + `Plug.Static` uses etags for HTTP caching. This means browsers/clients + should cache assets on the first request and validate the cache on + following requests, not downloading the static asset once again if it + has not changed. The cache-control for etags is specified by the + `cache_control_for_etags` option and defaults to `"public"`. + + However, `Plug.Static` also supports direct cache control by using + versioned query strings. If the request query string starts with + "?vsn=", `Plug.Static` assumes the application is versioning assets + and does not set the `ETag` header, meaning the cache behaviour will + be specified solely by the `cache_control_for_vsn_requests` config, + which defaults to `"public, max-age=31536000"`. + + ## Options + + * `:encodings` - list of 2-ary tuples where first value is value of + the `Accept-Encoding` header and second is extension of the file to + be served if given encoding is accepted by client. Entries will be tested + in order in list, so entries higher in list will be preferred. Defaults + to: `[]`. + + In addition to setting this value directly it supports 2 additional + options for compatibility reasons: + + + `:brotli` - will append `{"br", ".br"}` to the encodings list. + + `:gzip` - will append `{"gzip", ".gz"}` to the encodings list. + + Additional options will be added in the above order (Brotli takes + preference over Gzip) to reflect older behaviour which was set due + to fact that Brotli in general provides better compression ratio than + Gzip. + + * `:cache_control_for_etags` - sets the cache header for requests + that use etags. Defaults to `"public"`. + + * `:etag_generation` - specify a `{module, function, args}` to be used + to generate an etag. The `path` of the resource will be passed to + the function, as well as the `args`. If this option is not supplied, + etags will be generated based off of file size and modification time. + Note it is [recommended for the etag value to be quoted](https://tools.ietf.org/html/rfc7232#section-2.3), + which Plug won't do automatically. + + * `:cache_control_for_vsn_requests` - sets the cache header for + requests starting with "?vsn=" in the query string. Defaults to + `"public, max-age=31536000"`. + + * `:only` - filters which requests to serve. This is useful to avoid + file system access on every request when this plug is mounted + at `"/"`. For example, if `only: ["images", "favicon.ico"]` is + specified, only files in the "images" directory and the + "favicon.ico" file will be served by `Plug.Static`. + Note that `Plug.Static` matches these filters against request + uri and not against the filesystem. When requesting + a file with name containing non-ascii or special characters, + you should use urlencoded form. For example, you should write + `only: ["file%20name"]` instead of `only: ["file name"]`. + Defaults to `nil` (no filtering). + + * `:only_matching` - a relaxed version of `:only` that will + serve any request as long as one of the given values matches the + given path. For example, `only_matching: ["images", "favicon"]` + will match any request that starts at "images" or "favicon", + be it "/images/foo.png", "/images-high/foo.png", "/favicon.ico" + or "/favicon-high.ico". Such matches are useful when serving + digested files at the root. Defaults to `nil` (no filtering). + + * `:headers` - other headers to be set when serving static assets. Specify either + an enum of key-value pairs or a `{module, function, args}` to return an enum. The + `conn` will be passed to the function, as well as the `args`. + + * `:content_types` - custom MIME type mapping. As a map with filename as key + and content type as value. For example: + `content_types: %{"apple-app-site-association" => "application/json"}`. + + * `:set_content_type` - by default Plug.Static (re)sets the content type header + using auto-detection and the `:content_types` map. But when set to `false` + no content-type header will be inserted instead retaining the original + value or lack thereof. This can be useful when custom logic for appropiate + content types is needed which cannot be reasonably expressed as a static + filename map. + + ## Examples + + This plug can be mounted in a `Plug.Builder` pipeline as follows: + + defmodule MyPlug do + use Plug.Builder + + plug Plug.Static, + at: "/public", + from: :my_app, + only: ~w(images robots.txt) + plug :not_found + + def not_found(conn, _) do + send_resp(conn, 404, "not found") + end + end + + """ + + @behaviour Plug + @allowed_methods ~w(GET HEAD) + + import Plug.Conn + alias Plug.Conn + + # In this module, the `:prim_file` Erlang module along with the `:file_info` + # record are used instead of the more common and Elixir-y `File` module and + # `File.Stat` struct, respectively. The reason behind this is performance: all + # the `File` operations pass through a single process in order to support node + # operations that we simply don't need when serving assets. + + require Record + Record.defrecordp(:file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl")) + + defmodule InvalidPathError do + defexception message: "invalid path for static asset", plug_status: 400 + end + + @impl true + def init(opts) do + from = + case Keyword.fetch!(opts, :from) do + {_, _} = from -> from + {_, _, _} = from -> from + from when is_atom(from) -> {from, "priv/static"} + from when is_binary(from) -> from + _ -> raise ArgumentError, ":from must be an atom, a binary or a tuple" + end + + encodings = + opts + |> Keyword.get(:encodings, []) + |> maybe_add("br", ".br", Keyword.get(opts, :brotli, false)) + |> maybe_add("gzip", ".gz", Keyword.get(opts, :gzip, false)) + + %{ + encodings: encodings, + only_rules: {Keyword.get(opts, :only, []), Keyword.get(opts, :only_matching, [])}, + qs_cache: Keyword.get(opts, :cache_control_for_vsn_requests, "public, max-age=31536000"), + et_cache: Keyword.get(opts, :cache_control_for_etags, "public"), + et_generation: Keyword.get(opts, :etag_generation, nil), + headers: Keyword.get(opts, :headers, %{}), + content_types: Keyword.get(opts, :content_types, %{}), + set_content_type: Keyword.get(opts, :set_content_type, true), + from: from, + at: opts |> Keyword.fetch!(:at) |> Plug.Router.Utils.split() + } + end + + @impl true + def call( + conn = %Conn{method: meth}, + %{at: at, only_rules: only_rules, from: from, encodings: encodings} = options + ) + when meth in @allowed_methods do + segments = subset(at, conn.path_info) + + if allowed?(only_rules, segments) do + segments = Enum.map(segments, &uri_decode/1) + + if invalid_path?(segments) do + raise InvalidPathError, "invalid path for static asset: #{conn.request_path}" + end + + path = path(from, segments) + range = get_req_header(conn, "range") + encoding = file_encoding(conn, path, range, encodings) + serve_static(encoding, conn, segments, range, options) + else + conn + end + end + + def call(conn, _options) do + conn + end + + defp uri_decode(path) do + # TODO: Remove rescue as this can't fail from Elixir v1.13 + try do + URI.decode(path) + rescue + ArgumentError -> + raise InvalidPathError + end + end + + defp allowed?(_only_rules, []), do: false + defp allowed?({[], []}, _list), do: true + + defp allowed?({full, prefix}, [h | _]) do + h in full or (prefix != [] and match?({0, _}, :binary.match(h, prefix))) + end + + defp maybe_put_content_type(conn, false, _, _), do: conn + + defp maybe_put_content_type(conn, _, types, filename) do + content_type = Map.get(types, filename) || MIME.from_path(filename) + + conn + |> put_resp_header("content-type", content_type) + end + + defp serve_static({content_encoding, file_info, path}, conn, segments, range, options) do + %{ + qs_cache: qs_cache, + et_cache: et_cache, + et_generation: et_generation, + headers: headers, + content_types: types, + set_content_type: set_content_type + } = options + + case put_cache_header(conn, qs_cache, et_cache, et_generation, file_info, path) do + {:stale, conn} -> + filename = List.last(segments) + + conn + |> maybe_put_content_type(set_content_type, types, filename) + |> put_resp_header("accept-ranges", "bytes") + |> maybe_add_encoding(content_encoding) + |> merge_headers(headers) + |> serve_range(file_info, path, range, options) + + {:fresh, conn} -> + conn + |> maybe_add_vary(options) + |> send_resp(304, "") + |> halt() + end + end + + defp serve_static(:error, conn, _segments, _range, _options) do + conn + end + + defp serve_range(conn, file_info, path, [range], options) do + file_info(size: file_size) = file_info + + with %{"bytes" => bytes} <- Plug.Conn.Utils.params(range), + {range_start, range_end} <- start_and_end(bytes, file_size) do + send_range(conn, path, range_start, range_end, file_size, options) + else + _ -> send_entire_file(conn, path, options) + end + end + + defp serve_range(conn, _file_info, path, _range, options) do + send_entire_file(conn, path, options) + end + + defp start_and_end("-" <> rest, file_size) do + case Integer.parse(rest) do + {last, ""} when last > 0 and last <= file_size -> {file_size - last, file_size - 1} + _ -> :error + end + end + + defp start_and_end(range, file_size) do + case Integer.parse(range) do + {first, "-"} when first >= 0 -> + {first, file_size - 1} + + {first, "-" <> rest} when first >= 0 -> + case Integer.parse(rest) do + {last, ""} when last >= first -> {first, min(last, file_size - 1)} + _ -> :error + end + + _ -> + :error + end + end + + defp send_range(conn, path, 0, range_end, file_size, options) when range_end == file_size - 1 do + send_entire_file(conn, path, options) + end + + defp send_range(conn, path, range_start, range_end, file_size, _options) do + length = range_end - range_start + 1 + + conn + |> put_resp_header("content-range", "bytes #{range_start}-#{range_end}/#{file_size}") + |> send_file(206, path, range_start, length) + |> halt() + end + + defp send_entire_file(conn, path, options) do + conn + |> maybe_add_vary(options) + |> send_file(200, path) + |> halt() + end + + defp maybe_add_encoding(conn, nil), do: conn + defp maybe_add_encoding(conn, ce), do: put_resp_header(conn, "content-encoding", ce) + + defp maybe_add_vary(conn, %{encodings: encodings}) do + # If we serve gzip or brotli at any moment, we need to set the proper vary + # header regardless of whether we are serving gzip content right now. + # See: http://www.fastly.com/blog/best-practices-for-using-the-vary-header/ + if encodings != [] do + update_in(conn.resp_headers, &[{"vary", "Accept-Encoding"} | &1]) + else + conn + end + end + + defp put_cache_header( + %Conn{query_string: "vsn=" <> _} = conn, + qs_cache, + _et_cache, + _et_generation, + _file_info, + _path + ) + when is_binary(qs_cache) do + {:stale, put_resp_header(conn, "cache-control", qs_cache)} + end + + defp put_cache_header(conn, _qs_cache, et_cache, et_generation, file_info, path) + when is_binary(et_cache) do + etag = etag_for_path(file_info, et_generation, path) + + conn = + conn + |> put_resp_header("cache-control", et_cache) + |> put_resp_header("etag", etag) + + if etag in get_req_header(conn, "if-none-match") do + {:fresh, conn} + else + {:stale, conn} + end + end + + defp put_cache_header(conn, _, _, _, _, _) do + {:stale, conn} + end + + defp etag_for_path(file_info, et_generation, path) do + case et_generation do + {module, function, args} -> + apply(module, function, [path | args]) + + nil -> + file_info(size: size, mtime: mtime) = file_info + < :erlang.phash2() |> Integer.to_string(16)::binary, ?">> + end + end + + defp file_encoding(conn, path, [_range], _encodings) do + # We do not support compression for range queries. + file_encoding(conn, path, nil, []) + end + + defp file_encoding(conn, path, _range, encodings) do + encoded = + Enum.find_value(encodings, fn {encoding, ext} -> + if file_info = accept_encoding?(conn, encoding) && regular_file_info(path <> ext) do + {encoding, file_info, path <> ext} + end + end) + + cond do + not is_nil(encoded) -> + encoded + + file_info = regular_file_info(path) -> + {nil, file_info, path} + + true -> + :error + end + end + + defp regular_file_info(path) do + case :prim_file.read_file_info(path) do + {:ok, file_info(type: :regular) = file_info} -> + file_info + + _ -> + nil + end + end + + defp accept_encoding?(conn, encoding) do + encoding? = &String.contains?(&1, [encoding, "*"]) + + Enum.any?(get_req_header(conn, "accept-encoding"), fn accept -> + accept |> Plug.Conn.Utils.list() |> Enum.any?(encoding?) + end) + end + + defp maybe_add(list, key, value, true), do: list ++ [{key, value}] + defp maybe_add(list, _key, _value, false), do: list + + defp path({module, function, arguments}, segments) + when is_atom(module) and is_atom(function) and is_list(arguments), + do: Enum.join([apply(module, function, arguments) | segments], "/") + + defp path({app, from}, segments) when is_atom(app) and is_binary(from), + do: Enum.join([Application.app_dir(app), from | segments], "/") + + defp path(from, segments), + do: Enum.join([from | segments], "/") + + defp subset([h | expected], [h | actual]), do: subset(expected, actual) + defp subset([], actual), do: actual + defp subset(_, _), do: [] + + defp invalid_path?(list) do + invalid_path?(list, :binary.compile_pattern(["/", "\\", ":", "\0"])) + end + + defp invalid_path?([h | _], _match) when h in [".", "..", ""], do: true + defp invalid_path?([h | t], match), do: String.contains?(h, match) or invalid_path?(t) + defp invalid_path?([], _match), do: false + + defp merge_headers(conn, {module, function, args}) do + merge_headers(conn, apply(module, function, [conn | args])) + end + + defp merge_headers(conn, headers) do + merge_resp_headers(conn, headers) + end +end diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 300c33068..746203087 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do require Logger alias Pleroma.Web.MediaProxy + alias Pleroma.Web.Plugs.Utils @behaviour Plug # no slashes @@ -28,10 +29,21 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do |> Keyword.put(:at, "/__unconfigured_media_plug") |> Plug.Static.init() - %{static_plug_opts: static_plug_opts} + config = Pleroma.Config.get(Pleroma.Upload) + allowed_mime_types = Keyword.fetch!(config, :allowed_mime_types) + uploader = Keyword.fetch!(config, :uploader) + + %{ + static_plug_opts: static_plug_opts, + allowed_mime_types: allowed_mime_types, + uploader: uploader + } end - def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do + def call( + %{request_path: <<"/", @path, "/", file::binary>>} = conn, + %{uploader: uploader} = opts + ) do conn = case fetch_query_params(conn) do %{query_params: %{"name" => name}} = conn -> @@ -44,10 +56,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do end |> merge_resp_headers([{"content-security-policy", "sandbox"}]) - config = Pleroma.Config.get(Pleroma.Upload) - - with uploader <- Keyword.fetch!(config, :uploader), - {:ok, get_method} <- uploader.get_file(file), + with {:ok, get_method} <- uploader.get_file(file), false <- media_is_banned(conn, get_method) do get_media(conn, get_method, opts) else @@ -68,13 +77,23 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do defp media_is_banned(_, _), do: false + defp set_content_type(conn, opts, filepath) do + real_mime = MIME.from_path(filepath) + clean_mime = Utils.get_safe_mime_type(opts, real_mime) + put_resp_header(conn, "content-type", clean_mime) + end + defp get_media(conn, {:static_dir, directory}, opts) do static_opts = Map.get(opts, :static_plug_opts) |> Map.put(:at, [@path]) |> Map.put(:from, directory) + |> Map.put(:set_content_type, false) - conn = Plug.Static.call(conn, static_opts) + conn = + conn + |> set_content_type(opts, conn.request_path) + |> Pleroma.Web.Plugs.StaticNoCT.call(static_opts) if conn.halted do conn diff --git a/lib/pleroma/web/plugs/utils.ex b/lib/pleroma/web/plugs/utils.ex new file mode 100644 index 000000000..770a3eeb2 --- /dev/null +++ b/lib/pleroma/web/plugs/utils.ex @@ -0,0 +1,14 @@ +# Akkoma: Magically expressive social media +# Copyright © 2024 Akkoma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.Utils do + @moduledoc """ + Some helper functions shared across several plugs + """ + + def get_safe_mime_type(%{allowed_mime_types: allowed_mime_types} = _opts, mime) do + [maintype | _] = String.split(mime, "/", parts: 2) + if maintype in allowed_mime_types, do: mime, else: "application/octet-stream" + end +end diff --git a/mix.exs b/mix.exs index 4e8b3160d..f501eec58 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("3.11.0"), + version: version("3.12.0"), elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), @@ -21,13 +21,13 @@ defmodule Pleroma.Mixfile do source_url: "https://akkoma.dev/AkkomaGang/akkoma", docs: [ source_url_pattern: "https://akkoma.dev/AkkomaGang/akkoma/blob/develop/%{path}#L%{line}", - logo: "priv/static/images/logo.png", - extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/**/*.md"), + logo: "priv/static/logo-512.png", + extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/docs/**/*.md"), groups_for_extras: [ - "Installation manuals": Path.wildcard("docs/installation/*.md"), - Configuration: Path.wildcard("docs/config/*.md"), - Administration: Path.wildcard("docs/admin/*.md"), - "Pleroma's APIs and Mastodon API extensions": Path.wildcard("docs/api/*.md") + "Installation manuals": Path.wildcard("docs/docs/installation/*.md"), + Configuration: Path.wildcard("docs/docs/config/*.md"), + Administration: Path.wildcard("docs/docs/admin/*.md"), + "Pleroma's APIs and Mastodon API extensions": Path.wildcard("docs/docs/api/*.md") ], main: "readme", output: "priv/static/doc" diff --git a/priv/templates/sample_config.eex b/priv/templates/sample_config.eex index 0068969ac..724c5a9d5 100644 --- a/priv/templates/sample_config.eex +++ b/priv/templates/sample_config.eex @@ -78,6 +78,8 @@ config :joken, default_signer: "<%= jwt_secret %>" config :pleroma, configurable_from_database: <%= db_configurable? %> +config :pleroma, Pleroma.Upload, <%= if Kernel.length(upload_filters) > 0 do -"config :pleroma, Pleroma.Upload, filters: #{inspect(upload_filters)}" +" filters: #{inspect(upload_filters)}," end %> + base_url: "<%= media_url %>" diff --git a/test/fixtures/bridgy/actor.json b/test/fixtures/bridgy/actor.json index 5b2d8982b..b4e859a82 100644 --- a/test/fixtures/bridgy/actor.json +++ b/test/fixtures/bridgy/actor.json @@ -70,7 +70,7 @@ "preferredUsername": "jk.nipponalba.scot", "summary": "", "publicKey": { - "id": "jk.nipponalba.scot", + "id": "https://fed.brid.gy/jk.nipponalba.scot#key", "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdarxwzxnNbJ2hneWOYHkYJowk\npyigQtxlUd0VjgSQHwxU9kWqfbrHBVADyTtcqi/4dAzQd3UnCI1TPNnn4LPZY9PW\noiWd3Zl1/EfLFxO7LU9GS7fcSLQkyj5JNhSlN3I8QPudZbybrgRDVZYooDe1D+52\n5KLGqC2ajrIVOiDRTQIDAQAB\n-----END PUBLIC KEY-----" }, "inbox": "https://fed.brid.gy/jk.nipponalba.scot/inbox", diff --git a/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json b/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json index ca76d6e17..70d750e5d 100644 --- a/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json +++ b/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json @@ -37,7 +37,7 @@ "sharedInbox": "https://osada.macgirvin.com/inbox" }, "publicKey": { - "id": "https://osada.macgirvin.com/channel/mike/public_key_pem", + "id": "https://osada.macgirvin.com/channel/mike", "owner": "https://osada.macgirvin.com/channel/mike", "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAskSyK2VwBNKbzZl9XNJk\nvxU5AAilmRArMmmKSzphdHaVBHakeafUfixvqNrQ/oX2srJvJKcghNmEMrJ6MJ7r\npeEndVOo7pcP4PwVjtnC06p3J711q5tBehqM25BfCLCrB2YqWF6c8zk3CPN3Na21\n8k5s4cO95N/rGN+Po0XFAX/HjKjlpgNpKRDrpxmXxTU8NZfAqeQGJ5oiMBZI9vVB\n+eU7t1L6F5/XWuUCeP4OMrG8oZX822AREba8rknS6DpkWGES0Rx2eNOyYTf6ue75\nI6Ek6rlO+da5wMWr+3BvYMq4JMIwTHzAO+ZqqJPFpzKSiVuAWb2DOX/MDFecVWJE\ntF/R60lONxe4e/00MPCoDdqkLKdwROsk1yGL7z4Zk6jOWFEhIcWy/d2Ya5CpPvS3\nu4wNN4jkYAjra+8TiloRELhV4gpcEk8nkyNwLXOhYm7zQ5sIc5rfXoIrFzALB86W\nG05Nnqg+77zZIaTZpD9qekYlaEt+0OVtt9TTIeTiudQ983l6mfKwZYymrzymH1dL\nVgxBRYo+Z53QOSLiSKELfTBZxEoP1pBw6RiOHXydmJ/39hGgc2YAY/5ADwW2F2yb\nJ7+gxG6bPJ3ikDLYcD4CB5iJQdnTcDsFt3jyHAT6wOCzFAYPbHUqtzHfUM30dZBn\nnJhQF8udPLcXLaj6GW75JacCAwEAAQ==\n-----END PUBLIC KEY-----\n" }, diff --git a/test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json b/test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json index 1df73f2c5..dbf74dfe1 100644 --- a/test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json +++ b/test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json @@ -3,7 +3,7 @@ "attributedTo": "http://mastodon.example.org/users/admin", "attachment": [], "content": "

this post was not actually written by Haelwenn

", - "id": "https://info.pleroma.site/activity2.json", + "id": "https://info.pleroma.site/activity3.json", "published": "2018-09-01T22:15:00Z", "tag": [], "to": [ diff --git a/test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json b/test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json index 11c79e11e..c354747cc 100644 --- a/test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json +++ b/test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json @@ -1 +1 @@ -{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1","https://hubzilla.example.org/apschema/v1.2"],"type":"Person","id":"https://hubzilla.example.org/channel/kaniini","preferredUsername":"kaniini","name":"kaniini","icon":{"type":"Image","mediaType":"image/jpeg","url":"https://hubzilla.example.org/photo/profile/l/281","height":300,"width":300},"url":{"type":"Link","mediaType":"text/html","href":"https://hubzilla.example.org/channel/kaniini"},"inbox":"https://hubzilla.example.org/inbox/kaniini","outbox":"https://hubzilla.example.org/outbox/kaniini","followers":"https://hubzilla.example.org/followers/kaniini","following":"https://hubzilla.example.org/following/kaniini","endpoints":{"sharedInbox":"https://hubzilla.example.org/inbox"},"publicKey":{"id":"https://hubzilla.example.org/channel/kaniini/public_key_pem","owner":"https://hubzilla.example.org/channel/kaniini","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXCDkQPw+1N8B2CUd5s2\nbYvjHt+t7soMNfUiRy0qGbgW46S45k5lCq1KpbFIX3sgGZ4OWjnXVbvjCJi4kl5M\nfm5DBXzpuu05AmjVl8hqk4GejajiE/1Nq0uWHPiOSFWispUjCzzCu65V+IsiE5JU\nvcL6WEf/pYNRq7gYqyT693F7+cO5/rVv9OScx5UOxbIuU1VXYhdHCqAMDJWadC89\nhePrcD3HOQKl06W2tDxHcWk6QjrdsUQGbNOgK/QIN9gSxA+rCFEvH5O0HAhI0aXq\ncOB+vysJUFLeQOAqmAKvKS5V6RqE1GqqT0pDWHack4EmQi0gkgVzo+45xoP6wfDl\nWwG88w21LNxGvGHuN4I8mg6cEoApqKQBSOj086UtfDfSlPC1B+PRD2phE5etucHd\nF/RIWN3SxVzU9BKIiaDm2gwOpvI8QuorQb6HDtZFO5NsSN3PnMnSywPe7kXl/469\nuQRYXrseqyOVIi6WjhvXkyWVKVE5CBz+S8wXHfKph+9YOyUcJeAVMijp9wrjBlMc\noSzOGu79oM7tpMSq/Xo6ePJ/glNOwZR+OKrg92Qp9BGTKDNwGrxuxP/9KwWtGLNf\nOMTtIkxtC3ubhxL3lBxOd7l+Bmum0UJV2f8ogkCgvTpIz05jMoyU8qWl6kkWNQlY\nDropXWaOfy7Lac+G4qlfSgsCAwEAAQ==\n-----END PUBLIC KEY-----\n"},"nomadicLocations":[{"id":"https://hubzilla.example.org/locs/kaniini","type":"nomadicLocation","locationAddress":"acct:kaniini@hubzilla.example.org","locationPrimary":true,"locationDeleted":false}],"signature":{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"],"type":"RsaSignature2017","nonce":"6b981a2f3bdcffc20252e3b131d4a4569fd2dea9fac543e5196136302f492694","creator":"https://hubzilla.example.org/channel/kaniini/public_key_pem","created":"2018-05-19T08:19:13Z","signatureValue":"ezpT4iCIUzJSeJa/Jsf4EkgbX9enWZG/0eliLXZcvkeCX9mZabaX9LMQRViP2GSlAJBHJu+UqK5LWaoWw9pYkQQHUL+43w2DeBxQicEcPqpT46j6pHuWptfwB8YHTC2/Pb56Y/jseU37j+FW8xVmcGZk4cPqJRLQNojwJlQiFOpBEd4Cel6081W12Pep578+6xBL+h92RJsWznA1gE/NV9dkCqoAoNdiORJg68sVTm0yYxPit2D/DLwXUFeBhC47EZtY3DtAOf7rADGwbquXKug/wtEI47R4p9dJvMWERSVW9O2FmDk8deUjRR3qO1iYGce8O+uMnnBHmuTcToRUHH7mxfMdqjfbcZ9DGBjKtLPSOyVPT9rENeyX8fsksmX0XhfHsNSWkmeDaU5/Au3IY75gDewiGzmzLOpRc6GUnHHro7lMpyMuo3lLZKjNVsFZbx+sXCYwORz5GAMuwIt/iCUdrsQsF5aycqfUAZrFBPguH6DVjbMUqyLvS78sDKiWqgWVhq9VDKse+WuQaJLGBDJNF9APoA6NDMjjIBZfmkGf2mV7ubIYihoOncUjahFqxU5306cNxAcdj2uNcwkgX4BCnBe/L2YsvMHhZrupzDewWWy4fxhktyoZ7VhLSl1I7fMPytjOpb9EIvng4DHGX2t+hKfon2rCGfECPavwiTM="}} +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1","https://hubzilla.example.org/apschema/v1.2"],"type":"Person","id":"https://hubzilla.example.org/channel/kaniini","preferredUsername":"kaniini","name":"kaniini","icon":{"type":"Image","mediaType":"image/jpeg","url":"https://hubzilla.example.org/photo/profile/l/281","height":300,"width":300},"url":{"type":"Link","mediaType":"text/html","href":"https://hubzilla.example.org/channel/kaniini"},"inbox":"https://hubzilla.example.org/inbox/kaniini","outbox":"https://hubzilla.example.org/outbox/kaniini","followers":"https://hubzilla.example.org/followers/kaniini","following":"https://hubzilla.example.org/following/kaniini","endpoints":{"sharedInbox":"https://hubzilla.example.org/inbox"},"publicKey":{"id":"https://hubzilla.example.org/channel/kaniini","owner":"https://hubzilla.example.org/channel/kaniini","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXCDkQPw+1N8B2CUd5s2\nbYvjHt+t7soMNfUiRy0qGbgW46S45k5lCq1KpbFIX3sgGZ4OWjnXVbvjCJi4kl5M\nfm5DBXzpuu05AmjVl8hqk4GejajiE/1Nq0uWHPiOSFWispUjCzzCu65V+IsiE5JU\nvcL6WEf/pYNRq7gYqyT693F7+cO5/rVv9OScx5UOxbIuU1VXYhdHCqAMDJWadC89\nhePrcD3HOQKl06W2tDxHcWk6QjrdsUQGbNOgK/QIN9gSxA+rCFEvH5O0HAhI0aXq\ncOB+vysJUFLeQOAqmAKvKS5V6RqE1GqqT0pDWHack4EmQi0gkgVzo+45xoP6wfDl\nWwG88w21LNxGvGHuN4I8mg6cEoApqKQBSOj086UtfDfSlPC1B+PRD2phE5etucHd\nF/RIWN3SxVzU9BKIiaDm2gwOpvI8QuorQb6HDtZFO5NsSN3PnMnSywPe7kXl/469\nuQRYXrseqyOVIi6WjhvXkyWVKVE5CBz+S8wXHfKph+9YOyUcJeAVMijp9wrjBlMc\noSzOGu79oM7tpMSq/Xo6ePJ/glNOwZR+OKrg92Qp9BGTKDNwGrxuxP/9KwWtGLNf\nOMTtIkxtC3ubhxL3lBxOd7l+Bmum0UJV2f8ogkCgvTpIz05jMoyU8qWl6kkWNQlY\nDropXWaOfy7Lac+G4qlfSgsCAwEAAQ==\n-----END PUBLIC KEY-----\n"},"nomadicLocations":[{"id":"https://hubzilla.example.org/locs/kaniini","type":"nomadicLocation","locationAddress":"acct:kaniini@hubzilla.example.org","locationPrimary":true,"locationDeleted":false}],"signature":{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"],"type":"RsaSignature2017","nonce":"6b981a2f3bdcffc20252e3b131d4a4569fd2dea9fac543e5196136302f492694","creator":"https://hubzilla.example.org/channel","created":"2018-05-19T08:19:13Z","signatureValue":"ezpT4iCIUzJSeJa/Jsf4EkgbX9enWZG/0eliLXZcvkeCX9mZabaX9LMQRViP2GSlAJBHJu+UqK5LWaoWw9pYkQQHUL+43w2DeBxQicEcPqpT46j6pHuWptfwB8YHTC2/Pb56Y/jseU37j+FW8xVmcGZk4cPqJRLQNojwJlQiFOpBEd4Cel6081W12Pep578+6xBL+h92RJsWznA1gE/NV9dkCqoAoNdiORJg68sVTm0yYxPit2D/DLwXUFeBhC47EZtY3DtAOf7rADGwbquXKug/wtEI47R4p9dJvMWERSVW9O2FmDk8deUjRR3qO1iYGce8O+uMnnBHmuTcToRUHH7mxfMdqjfbcZ9DGBjKtLPSOyVPT9rENeyX8fsksmX0XhfHsNSWkmeDaU5/Au3IY75gDewiGzmzLOpRc6GUnHHro7lMpyMuo3lLZKjNVsFZbx+sXCYwORz5GAMuwIt/iCUdrsQsF5aycqfUAZrFBPguH6DVjbMUqyLvS78sDKiWqgWVhq9VDKse+WuQaJLGBDJNF9APoA6NDMjjIBZfmkGf2mV7ubIYihoOncUjahFqxU5306cNxAcdj2uNcwkgX4BCnBe/L2YsvMHhZrupzDewWWy4fxhktyoZ7VhLSl1I7fMPytjOpb9EIvng4DHGX2t+hKfon2rCGfECPavwiTM="}} diff --git a/test/fixtures/tesla_mock/relay@mastdon.example.org.json b/test/fixtures/tesla_mock/relay@mastdon.example.org.json index c1fab7d3b..21dd405c8 100644 --- a/test/fixtures/tesla_mock/relay@mastdon.example.org.json +++ b/test/fixtures/tesla_mock/relay@mastdon.example.org.json @@ -11,7 +11,7 @@ "toot": "http://joinmastodon.org/ns#", "Emoji": "toot:Emoji" }], - "id": "http://mastodon.example.org/users/admin", + "id": "http://mastodon.example.org/users/relay", "type": "Application", "invisible": true, "following": "http://mastodon.example.org/users/admin/following", @@ -24,8 +24,8 @@ "url": "http://mastodon.example.org/@admin", "manuallyApprovesFollowers": false, "publicKey": { - "id": "http://mastodon.example.org/users/admin#main-key", - "owner": "http://mastodon.example.org/users/admin", + "id": "http://mastodon.example.org/users/relay#main-key", + "owner": "http://mastodon.example.org/users/relay", "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n" }, "attachment": [{ diff --git a/test/fixtures/users_mock/friendica_followers.json b/test/fixtures/users_mock/friendica_followers.json index 7b86b5fe2..02b287e23 100644 --- a/test/fixtures/users_mock/friendica_followers.json +++ b/test/fixtures/users_mock/friendica_followers.json @@ -13,7 +13,7 @@ "directMessage": "litepub:directMessage" } ], - "id": "http://localhost:8080/followers/fuser3", + "id": "http://remote.org/followers/fuser3", "type": "OrderedCollection", "totalItems": 296 } diff --git a/test/fixtures/users_mock/friendica_following.json b/test/fixtures/users_mock/friendica_following.json index 7c526befc..0908e78f0 100644 --- a/test/fixtures/users_mock/friendica_following.json +++ b/test/fixtures/users_mock/friendica_following.json @@ -13,7 +13,7 @@ "directMessage": "litepub:directMessage" } ], - "id": "http://localhost:8080/following/fuser3", + "id": "http://remote.org/following/fuser3", "type": "OrderedCollection", "totalItems": 32 } diff --git a/test/fixtures/users_mock/masto_closed_followers.json b/test/fixtures/users_mock/masto_closed_followers.json index da296892d..ccc32d15e 100644 --- a/test/fixtures/users_mock/masto_closed_followers.json +++ b/test/fixtures/users_mock/masto_closed_followers.json @@ -1,7 +1,7 @@ { "@context": "https://www.w3.org/ns/activitystreams", - "id": "http://localhost:4001/users/masto_closed/followers", + "id": "http://remote.org/users/masto_closed/followers", "type": "OrderedCollection", "totalItems": 437, - "first": "http://localhost:4001/users/masto_closed/followers?page=1" + "first": "http://remote.org/users/masto_closed/followers?page=1" } diff --git a/test/fixtures/users_mock/masto_closed_followers_page.json b/test/fixtures/users_mock/masto_closed_followers_page.json index 04ab0c4d3..e4f1b3ac0 100644 --- a/test/fixtures/users_mock/masto_closed_followers_page.json +++ b/test/fixtures/users_mock/masto_closed_followers_page.json @@ -1 +1 @@ -{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://localhost:4001/users/masto_closed/followers?page=2","partOf":"http://localhost:4001/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} +{"@context":"https://www.w3.org/ns/activitystreams","id":"http://remote.org/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://remote.org/users/masto_closed/followers?page=2","partOf":"http://remote.org/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} diff --git a/test/fixtures/users_mock/masto_closed_following.json b/test/fixtures/users_mock/masto_closed_following.json index 146d49f9c..34e9e9fe8 100644 --- a/test/fixtures/users_mock/masto_closed_following.json +++ b/test/fixtures/users_mock/masto_closed_following.json @@ -1,7 +1,7 @@ { "@context": "https://www.w3.org/ns/activitystreams", - "id": "http://localhost:4001/users/masto_closed/following", + "id": "http://remote.org/users/masto_closed/following", "type": "OrderedCollection", "totalItems": 152, - "first": "http://localhost:4001/users/masto_closed/following?page=1" + "first": "http://remote.org/users/masto_closed/following?page=1" } diff --git a/test/fixtures/users_mock/masto_closed_following_page.json b/test/fixtures/users_mock/masto_closed_following_page.json index 8d8324699..d398ae3cf 100644 --- a/test/fixtures/users_mock/masto_closed_following_page.json +++ b/test/fixtures/users_mock/masto_closed_following_page.json @@ -1 +1 @@ -{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://localhost:4001/users/masto_closed/following?page=2","partOf":"http://localhost:4001/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} +{"@context":"https://www.w3.org/ns/activitystreams","id":"http://remote.org/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://remote.org/users/masto_closed/following?page=2","partOf":"http://remote.org/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} diff --git a/test/fixtures/users_mock/pleroma_followers.json b/test/fixtures/users_mock/pleroma_followers.json index db71d084b..9611990ee 100644 --- a/test/fixtures/users_mock/pleroma_followers.json +++ b/test/fixtures/users_mock/pleroma_followers.json @@ -1,14 +1,14 @@ { "type": "OrderedCollection", "totalItems": 527, - "id": "http://localhost:4001/users/fuser2/followers", + "id": "http://remote.org/users/fuser2/followers", "first": { "type": "OrderedCollectionPage", "totalItems": 527, - "partOf": "http://localhost:4001/users/fuser2/followers", + "partOf": "http://remote.org/users/fuser2/followers", "orderedItems": [], - "next": "http://localhost:4001/users/fuser2/followers?page=2", - "id": "http://localhost:4001/users/fuser2/followers?page=1" + "next": "http://remote.org/users/fuser2/followers?page=2", + "id": "http://remote.org/users/fuser2/followers?page=1" }, "@context": [ "https://www.w3.org/ns/activitystreams", diff --git a/test/fixtures/users_mock/pleroma_following.json b/test/fixtures/users_mock/pleroma_following.json index 33d087703..27fadbc94 100644 --- a/test/fixtures/users_mock/pleroma_following.json +++ b/test/fixtures/users_mock/pleroma_following.json @@ -1,14 +1,14 @@ { "type": "OrderedCollection", "totalItems": 267, - "id": "http://localhost:4001/users/fuser2/following", + "id": "http://remote.org/users/fuser2/following", "first": { "type": "OrderedCollectionPage", "totalItems": 267, - "partOf": "http://localhost:4001/users/fuser2/following", + "partOf": "http://remote.org/users/fuser2/following", "orderedItems": [], - "next": "http://localhost:4001/users/fuser2/following?page=2", - "id": "http://localhost:4001/users/fuser2/following?page=1" + "next": "http://remote.org/users/fuser2/following?page=2", + "id": "http://remote.org/users/fuser2/following?page=1" }, "@context": [ "https://www.w3.org/ns/activitystreams", diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs index 5a5a68053..17b2e3267 100644 --- a/test/mix/tasks/pleroma/instance_test.exs +++ b/test/mix/tasks/pleroma/instance_test.exs @@ -39,6 +39,8 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do tmp_path() <> "setup.psql", "--domain", "test.pleroma.social", + "--media-url", + "https://media.pleroma.social/media", "--instance-name", "Pleroma", "--admin-email", @@ -69,8 +71,6 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do "./test/../test/instance/static/", "--strip-uploads", "y", - "--dedupe-uploads", - "n", "--anonymize-uploads", "n" ]) @@ -92,6 +92,7 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do assert generated_config =~ "configurable_from_database: true" assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]" assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]" + assert generated_config =~ "base_url: \"https://media.pleroma.social/media\"" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.exists?(Path.expand("./test/instance/static/robots.txt")) end diff --git a/test/mix/tasks/pleroma/uploads_test.exs b/test/mix/tasks/pleroma/uploads_test.exs index 67fb642c1..d00e25a37 100644 --- a/test/mix/tasks/pleroma/uploads_test.exs +++ b/test/mix/tasks/pleroma/uploads_test.exs @@ -16,7 +16,6 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do Mix.shell(Mix.Shell.IO) end) - File.mkdir_p!("test/uploads") :ok end diff --git a/test/pleroma/collections/collections_fetcher_test.exs b/test/pleroma/collections/collections_fetcher_test.exs index 7a582a3d7..ff1aa84db 100644 --- a/test/pleroma/collections/collections_fetcher_test.exs +++ b/test/pleroma/collections/collections_fetcher_test.exs @@ -12,11 +12,14 @@ defmodule Akkoma.Collections.FetcherTest do end test "it should extract items from an embedded array in a Collection" do + ap_id = "https://example.com/collection/ordered_array" + unordered_collection = "test/fixtures/collections/unordered_array.json" |> File.read!() - - ap_id = "https://example.com/collection/ordered_array" + |> Jason.decode!() + |> Map.put("id", ap_id) + |> Jason.encode!(pretty: true) Tesla.Mock.mock(fn %{ diff --git a/test/pleroma/emoji/pack_test.exs b/test/pleroma/emoji/pack_test.exs index 4d769789d..f5d2e2eef 100644 --- a/test/pleroma/emoji/pack_test.exs +++ b/test/pleroma/emoji/pack_test.exs @@ -93,7 +93,9 @@ defmodule Pleroma.Emoji.PackTest do assert updated_pack.files_count == 1 end - test "load_pack/1 ignores path traversal in a forged pack name", %{pack: pack} do - assert {:ok, ^pack} = Pack.load_pack("../../../../../dump_pack") + test "load_pack/1 panics on path traversal in a forged pack name" do + assert_raise(RuntimeError, "Invalid or malicious pack name: ../../../../../dump_pack", fn -> + Pack.load_pack("../../../../../dump_pack") + end) end end diff --git a/test/pleroma/object/containment_test.exs b/test/pleroma/object/containment_test.exs index fb2fb7d49..f8f40a3ac 100644 --- a/test/pleroma/object/containment_test.exs +++ b/test/pleroma/object/containment_test.exs @@ -17,16 +17,58 @@ defmodule Pleroma.Object.ContainmentTest do end describe "general origin containment" do - test "works for completely actorless posts" do - assert :error == - Containment.contain_origin("https://glaceon.social/users/monorail", %{ + test "handles completly actorless objects gracefully" do + assert :ok == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ "deleted" => "2019-10-30T05:48:50.249606Z", "formerType" => "Note", - "id" => "https://glaceon.social/users/monorail/statuses/103049757364029187", + "id" => "https://glaceon.social/statuses/123", "type" => "Tombstone" }) end + test "errors for spoofed actors" do + assert :error == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ + "actor" => "https://otp.akkoma.dev/users/you", + "id" => "https://glaceon.social/statuses/123", + "type" => "Note" + }) + end + + test "errors for spoofed attributedTo" do + assert :error == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ + "attributedTo" => "https://otp.akkoma.dev/users/you", + "id" => "https://glaceon.social/statuses/123", + "type" => "Note" + }) + end + + test "accepts valid actors" do + assert :ok == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ + "actor" => "https://glaceon.social/users/monorail", + "attributedTo" => "https://glaceon.social/users/monorail", + "id" => "https://glaceon.social/statuses/123", + "type" => "Note" + }) + + assert :ok == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ + "actor" => "https://glaceon.social/users/monorail", + "id" => "https://glaceon.social/statuses/123", + "type" => "Note" + }) + + assert :ok == + Containment.contain_origin("https://glaceon.social/statuses/123", %{ + "attributedTo" => "https://glaceon.social/users/monorail", + "id" => "https://glaceon.social/statuses/123", + "type" => "Note" + }) + end + test "contain_origin_from_id() catches obvious spoofing attempts" do data = %{ "id" => "http://example.com/~alyssa/activities/1234.json" @@ -63,6 +105,56 @@ defmodule Pleroma.Object.ContainmentTest do ) end + test "contain_id_to_fetch() refuses alternate IDs within the same origin domain" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json", + "url" => "http://example.com/@alyssa/status/1234" + } + + :error = + Containment.contain_id_to_fetch( + "http://example.com/~alyssa/activities/1234", + data + ) + end + + test "contain_id_to_fetch() allows matching IDs" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json/" + } + + :ok = + Containment.contain_id_to_fetch( + "http://example.com/~alyssa/activities/1234.json/", + data + ) + + :ok = + Containment.contain_id_to_fetch( + "http://example.com/~alyssa/activities/1234.json", + data + ) + end + + test "contain_id_to_fetch() allows display URLs" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json", + "url" => "http://example.com/@alyssa/status/1234" + } + + :ok = + Containment.contain_id_to_fetch( + "http://example.com/@alyssa/status/1234", + data + ) + + :ok = + Containment.contain_id_to_fetch( + "http://example.com/@alyssa/status/1234/", + data + ) + end + test "users cannot be collided through fake direction spoofing attempts" do _user = insert(:user, %{ diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 8cf0bce48..4c4831af3 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -14,6 +14,17 @@ defmodule Pleroma.Object.FetcherTest do import Mock import Tesla.Mock + defp spoofed_object_with_ids( + id \\ "https://patch.cx/objects/spoof", + actor_id \\ "https://patch.cx/users/rin" + ) do + File.read!("test/fixtures/spoofed-object.json") + |> Jason.decode!() + |> Map.put("id", id) + |> Map.put("actor", actor_id) + |> Jason.encode!() + end + setup do mock(fn %{method: :get, url: "https://mastodon.example.org/users/userisgone"} -> @@ -22,6 +33,32 @@ defmodule Pleroma.Object.FetcherTest do %{method: :get, url: "https://mastodon.example.org/users/userisgone404"} -> %Tesla.Env{status: 404} + # Spoof: wrong Content-Type + %{ + method: :get, + url: "https://patch.cx/objects/spoof_content_type.json" + } -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/objects/spoof_content_type.json", + headers: [{"content-type", "application/json"}], + body: spoofed_object_with_ids("https://patch.cx/objects/spoof_content_type.json") + } + + # Spoof: no Content-Type + %{ + method: :get, + url: "https://patch.cx/objects/spoof_content_type" + } -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/objects/spoof_content_type", + headers: [], + body: spoofed_object_with_ids("https://patch.cx/objects/spoof_content_type") + } + + # Spoof: mismatching ids + # Variant 1: Non-exisitng fake id %{ method: :get, url: @@ -29,8 +66,75 @@ defmodule Pleroma.Object.FetcherTest do } -> %Tesla.Env{ status: 200, - headers: [{"content-type", "application/json"}], - body: File.read!("test/fixtures/spoofed-object.json") + url: + "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids() + } + + %{method: :get, url: "https://patch.cx/objects/spoof"} -> + %Tesla.Env{ + status: 404, + url: "https://patch.cx/objects/spoof", + headers: [], + body: "Not found" + } + + # Varaint 2: two-stage payload + %{method: :get, url: "https://patch.cx/media/spoof_stage1.json"} -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/media/spoof_stage1.json", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids("https://patch.cx/media/spoof_stage2.json") + } + + %{method: :get, url: "https://patch.cx/media/spoof_stage2.json"} -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/media/spoof_stage2.json", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids("https://patch.cx/media/unpredictable.json") + } + + # Spoof: cross-domain redirect with original domain id + %{method: :get, url: "https://patch.cx/objects/spoof_media_redirect1"} -> + %Tesla.Env{ + status: 200, + url: "https://media.patch.cx/objects/spoof", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids("https://patch.cx/objects/spoof_media_redirect1") + } + + # Spoof: cross-domain redirect with final domain id + %{method: :get, url: "https://patch.cx/objects/spoof_media_redirect2"} -> + %Tesla.Env{ + status: 200, + url: "https://media.patch.cx/objects/spoof_media_redirect2", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids("https://media.patch.cx/objects/spoof_media_redirect2") + } + + # No-Spoof: same domain redirect + %{method: :get, url: "https://patch.cx/objects/spoof_redirect"} -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/objects/spoof_redirect", + headers: [{"content-type", "application/activity+json"}], + body: spoofed_object_with_ids("https://patch.cx/objects/spoof_redirect") + } + + # Spoof: Actor from another domain + %{method: :get, url: "https://patch.cx/objects/spoof_foreign_actor"} -> + %Tesla.Env{ + status: 200, + url: "https://patch.cx/objects/spoof_foreign_actor", + headers: [{"content-type", "application/activity+json"}], + body: + spoofed_object_with_ids( + "https://patch.cx/objects/spoof_foreign_actor", + "https://not.patch.cx/users/rin" + ) } env -> @@ -46,6 +150,7 @@ defmodule Pleroma.Object.FetcherTest do %{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} -> %Tesla.Env{ status: 200, + url: "https://social.sakamoto.gq/objects/f20f2497-66d9-4a52-a2e1-1be2a39c32c1", body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"), headers: HttpRequestMock.activitypub_object_headers() } @@ -129,6 +234,71 @@ defmodule Pleroma.Object.FetcherTest do end end + describe "fetcher security and auth checks" do + test "it does not fetch a spoofed object without content type" do + assert {:error, {:content_type, nil}} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_content_type" + ) + end + + test "it does not fetch a spoofed object with wrong content type" do + assert {:error, {:content_type, _}} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_content_type.json" + ) + end + + test "it does not fetch a spoofed object with id different from URL" do + assert {:error, "Object's ActivityPub id/url does not match final fetch URL"} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json" + ) + + assert {:error, "Object's ActivityPub id/url does not match final fetch URL"} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/media/spoof_stage1.json" + ) + end + + test "it does not fetch an object via cross-domain redirects (initial id)" do + assert {:error, {:cross_domain_redirect, true}} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_media_redirect1" + ) + end + + test "it does not fetch an object via cross-domain redirects (final id)" do + assert {:error, {:cross_domain_redirect, true}} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_media_redirect2" + ) + end + + test "it accepts same-domain redirects" do + assert {:ok, %{"id" => id} = _object} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_redirect" + ) + + assert id == "https://patch.cx/objects/spoof_redirect" + end + + test "it does not fetch a spoofed object with a foreign actor" do + assert {:error, "Object containment failed."} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://patch.cx/objects/spoof_foreign_actor" + ) + end + + test "it does not fetch from localhost" do + assert {:error, "Trying to fetch local resource"} = + Fetcher.fetch_and_contain_remote_object_from_id( + Pleroma.Web.Endpoint.url() <> "/spoof_local" + ) + end + end + describe "fetching an object" do test "it fetches an object" do {:ok, object} = @@ -155,13 +325,6 @@ defmodule Pleroma.Object.FetcherTest do ) end - test "it does not fetch a spoofed object uploaded on an instance as an attachment" do - assert {:error, _} = - Fetcher.fetch_object_from_id( - "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json" - ) - end - test "does not fetch anything from a rejected instance" do clear_config([:mrf_simple, :reject], [{"evil.example.org", "i said so"}]) @@ -583,12 +746,13 @@ defmodule Pleroma.Object.FetcherTest do } -> %Tesla.Env{ status: 200, + url: "https://mastodon.social/2", headers: [{"content-type", "application/activity+json"}], body: "{}" } end) - assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2") + assert {:ok, _, "{}"} = Fetcher.get_object("https://mastodon.social/2") end test "should return ok if the content type is application/ld+json with a profile" do @@ -599,6 +763,7 @@ defmodule Pleroma.Object.FetcherTest do } -> %Tesla.Env{ status: 200, + url: "https://mastodon.social/2", headers: [ {"content-type", "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""} @@ -607,24 +772,7 @@ defmodule Pleroma.Object.FetcherTest do } end) - assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2") - - Tesla.Mock.mock(fn - %{ - method: :get, - url: "https://mastodon.social/2" - } -> - %Tesla.Env{ - status: 200, - headers: [ - {"content-type", - "application/ld+json; profile=\"http://www.w3.org/ns/activitystreams\""} - ], - body: "{}" - } - end) - - assert {:ok, "{}"} = Fetcher.get_object("https://mastodon.social/2") + assert {:ok, _, "{}"} = Fetcher.get_object("https://mastodon.social/2") end test "should not return ok with other content types" do @@ -635,6 +783,7 @@ defmodule Pleroma.Object.FetcherTest do } -> %Tesla.Env{ status: 200, + url: "https://mastodon.social/2", headers: [{"content-type", "application/json"}], body: "{}" } @@ -643,5 +792,23 @@ defmodule Pleroma.Object.FetcherTest do assert {:error, {:content_type, "application/json"}} = Fetcher.get_object("https://mastodon.social/2") end + + test "returns the url after redirects" do + Tesla.Mock.mock(fn + %{ + method: :get, + url: "https://mastodon.social/5" + } -> + %Tesla.Env{ + status: 200, + url: "https://mastodon.social/7", + headers: [{"content-type", "application/activity+json"}], + body: "{}" + } + end) + + assert {:ok, "https://mastodon.social/7", "{}"} = + Fetcher.get_object("https://mastodon.social/5") + end end end diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs index 8320660a5..4b0fec1bd 100644 --- a/test/pleroma/object_test.exs +++ b/test/pleroma/object_test.exs @@ -22,6 +22,13 @@ defmodule Pleroma.ObjectTest do :ok end + # Only works for a single attachment but that's all we need here + defp get_attachment_filepath(note, uploads_dir) do + %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = note + filename = href |> Path.basename() + "#{uploads_dir}/#{filename}" + end + test "returns an object by it's AP id" do object = insert(:note) found_object = Object.get_by_ap_id(object.data["id"]) @@ -95,14 +102,13 @@ defmodule Pleroma.ObjectTest do {:ok, %Object{} = attachment} = Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) - %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = - note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) - path = href |> Path.dirname() |> Path.basename() + path = get_attachment_filepath(note, uploads_dir) - assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + assert File.exists?("#{path}") Object.delete(note) @@ -111,7 +117,7 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(note.id).data["deleted"] refute Object.get_by_id(attachment.id) == nil - assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + assert File.exists?("#{path}") end test "in subdirectories" do @@ -129,14 +135,13 @@ defmodule Pleroma.ObjectTest do {:ok, %Object{} = attachment} = Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) - %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = - note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) - path = href |> Path.dirname() |> Path.basename() + path = get_attachment_filepath(note, uploads_dir) - assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + assert File.exists?("#{path}") Object.delete(note) @@ -145,7 +150,7 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(note.id).data["deleted"] assert Object.get_by_id(attachment.id) == nil - assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute File.exists?("#{path}") end test "with dedupe enabled" do @@ -168,13 +173,11 @@ defmodule Pleroma.ObjectTest do {:ok, %Object{} = attachment} = Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) - %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = - note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) - filename = Path.basename(href) + path = get_attachment_filepath(note, uploads_dir) - assert {:ok, files} = File.ls(uploads_dir) - assert filename in files + assert File.exists?("#{path}") Object.delete(note) @@ -182,8 +185,8 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(note.id).data["deleted"] assert Object.get_by_id(attachment.id) == nil - assert {:ok, files} = File.ls(uploads_dir) - refute filename in files + # what if another test runs concurrently using the same image file? + refute File.exists?("#{path}") end test "with objects that have legacy data.url attribute" do @@ -203,14 +206,13 @@ defmodule Pleroma.ObjectTest do {:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id}) - %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = - note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) - path = href |> Path.dirname() |> Path.basename() + path = get_attachment_filepath(note, uploads_dir) - assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + assert File.exists?("#{path}") Object.delete(note) @@ -219,7 +221,7 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(note.id).data["deleted"] assert Object.get_by_id(attachment.id) == nil - assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute File.exists?("#{path}") end test "With custom base_url" do @@ -238,14 +240,13 @@ defmodule Pleroma.ObjectTest do {:ok, %Object{} = attachment} = Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) - %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = - note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) - path = href |> Path.dirname() |> Path.basename() + path = get_attachment_filepath(note, uploads_dir) - assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + assert File.exists?("#{path}") Object.delete(note) @@ -254,7 +255,7 @@ defmodule Pleroma.ObjectTest do assert Object.get_by_id(note.id).data["deleted"] assert Object.get_by_id(attachment.id) == nil - assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + refute File.exists?("#{path}") end end diff --git a/test/pleroma/reverse_proxy_test.exs b/test/pleroma/reverse_proxy_test.exs index e3e2a1571..fc6ae42bc 100644 --- a/test/pleroma/reverse_proxy_test.exs +++ b/test/pleroma/reverse_proxy_test.exs @@ -75,13 +75,16 @@ defmodule Pleroma.ReverseProxyTest do Tesla.Mock.mock(fn %{method: :head, url: "/head"} -> %Tesla.Env{ status: 200, - headers: [{"content-type", "text/html; charset=utf-8"}], + headers: [{"content-type", "image/png"}], body: "" } end) conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head") - assert html_response(conn, 200) == "" + + assert conn.status == 200 + assert Conn.get_resp_header(conn, "content-type") == ["image/png"] + assert conn.resp_body == "" end end @@ -252,4 +255,38 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end + + describe "content-type sanitisation" do + test "preserves video type", %{conn: conn} do + Tesla.Mock.mock(fn %{method: :get, url: "/content"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "video/mp4"}], + body: "test" + } + end) + + conn = ReverseProxy.call(Map.put(conn, :method, "GET"), "/content") + + assert conn.status == 200 + assert Conn.get_resp_header(conn, "content-type") == ["video/mp4"] + assert conn.resp_body == "test" + end + + test "replaces application type", %{conn: conn} do + Tesla.Mock.mock(fn %{method: :get, url: "/content"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: "test" + } + end) + + conn = ReverseProxy.call(Map.put(conn, :method, "GET"), "/content") + + assert conn.status == 200 + assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] + assert conn.resp_body == "test" + end + end end diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs index ad6065b43..27a2d1b97 100644 --- a/test/pleroma/upload_test.exs +++ b/test/pleroma/upload_test.exs @@ -188,7 +188,7 @@ defmodule Pleroma.UploadTest do refute data["name"] == "an [image.jpg" end - test "escapes invalid characters in url" do + test "mangles the filename" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ @@ -200,23 +200,8 @@ defmodule Pleroma.UploadTest do {:ok, data} = Upload.store(file) [attachment_url | _] = data["url"] - assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg" - end - - test "escapes reserved uri characters" do - File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - - file = %Plug.Upload{ - content_type: "image/jpeg", - path: Path.absname("test/fixtures/image_tmp.jpg"), - filename: ":?#[]@!$&\\'()*+,;=.jpg" - } - - {:ok, data} = Upload.store(file) - [attachment_url | _] = data["url"] - - assert Path.basename(attachment_url["href"]) == - "%3A%3F%23%5B%5D%40%21%24%26%5C%27%28%29%2A%2B%2C%3B%3D.jpg" + refute Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg" + refute Path.basename(attachment_url["href"]) == "an… image.jpg" end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 5a0d77cab..96ca8d0fd 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -326,9 +326,9 @@ defmodule Pleroma.UserTest do insert(:user, %{ local: false, nickname: "fuser2", - ap_id: "http://localhost:4001/users/fuser2", - follower_address: "http://localhost:4001/users/fuser2/followers", - following_address: "http://localhost:4001/users/fuser2/following" + ap_id: "http://remote.org/users/fuser2", + follower_address: "http://remote.org/users/fuser2/followers", + following_address: "http://remote.org/users/fuser2/following" }) {:ok, user, followed} = User.follow(user, followed, :follow_accept) @@ -2177,8 +2177,8 @@ defmodule Pleroma.UserTest do describe "sync followers count" do setup do - user1 = insert(:user, local: false, ap_id: "http://localhost:4001/users/masto_closed") - user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2") + user1 = insert(:user, local: false, ap_id: "http://remote.org/users/masto_closed") + user2 = insert(:user, local: false, ap_id: "http://remote.org/users/fuser2") insert(:user, local: true) insert(:user, local: false, is_active: false) {:ok, user1: user1, user2: user2} @@ -2272,8 +2272,8 @@ defmodule Pleroma.UserTest do other_user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", + follower_address: "http://remote.org/users/masto_closed/followers", + following_address: "http://remote.org/users/masto_closed/following", ap_enabled: true ) @@ -2294,8 +2294,8 @@ defmodule Pleroma.UserTest do other_user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", + follower_address: "http://remote.org/users/masto_closed/followers", + following_address: "http://remote.org/users/masto_closed/following", ap_enabled: true ) @@ -2316,8 +2316,8 @@ defmodule Pleroma.UserTest do other_user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following", + follower_address: "http://remote.org/users/masto_closed/followers", + following_address: "http://remote.org/users/masto_closed/following", ap_enabled: true ) diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 5d5388cf5..5ad6d4716 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -312,7 +312,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end test "fetches user featured collection using the first property" do - featured_url = "https://friendica.example.com/raha/collections/featured" + featured_url = "https://friendica.example.com/featured/raha" first_url = "https://friendica.example.com/featured/raha?page=1" featured_data = @@ -350,7 +350,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end test "fetches user featured when it has string IDs" do - featured_url = "https://example.com/alisaie/collections/featured" + featured_url = "https://example.com/users/alisaie/collections/featured" dead_url = "https://example.com/users/alisaie/statuses/108311386746229284" featured_data = @@ -1304,14 +1304,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do %{test_file: test_file} end - test "strips / from filename", %{test_file: file} do - file = %Plug.Upload{file | filename: "../../../../../nested/bad.jpg"} - {:ok, %Object{} = object} = ActivityPub.upload(file) - [%{"href" => href}] = object.data["url"] - assert Regex.match?(~r"/bad.jpg$", href) - refute Regex.match?(~r"/nested/", href) - end - test "sets a description if given", %{test_file: file} do {:ok, %Object{} = object} = ActivityPub.upload(file, description: "a cool file") assert object.data["name"] == "a cool file" @@ -1651,8 +1643,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/fuser2/followers", - following_address: "http://localhost:4001/users/fuser2/following" + follower_address: "http://remote.org/users/fuser2/followers", + following_address: "http://remote.org/users/fuser2/following" ) {:ok, info} = ActivityPub.fetch_follow_information_for_user(user) @@ -1663,7 +1655,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do test "detects hidden followers" do mock(fn env -> case env.url do - "http://localhost:4001/users/masto_closed/followers?page=1" -> + "http://remote.org/users/masto_closed/followers?page=1" -> %Tesla.Env{status: 403, body: ""} _ -> @@ -1674,8 +1666,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following" + follower_address: "http://remote.org/users/masto_closed/followers", + following_address: "http://remote.org/users/masto_closed/following" ) {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) @@ -1686,7 +1678,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do test "detects hidden follows" do mock(fn env -> case env.url do - "http://localhost:4001/users/masto_closed/following?page=1" -> + "http://remote.org/users/masto_closed/following?page=1" -> %Tesla.Env{status: 403, body: ""} _ -> @@ -1697,8 +1689,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following" + follower_address: "http://remote.org/users/masto_closed/followers", + following_address: "http://remote.org/users/masto_closed/following" ) {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) @@ -1710,8 +1702,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user = insert(:user, local: false, - follower_address: "http://localhost:8080/followers/fuser3", - following_address: "http://localhost:8080/following/fuser3" + follower_address: "http://remote.org/followers/fuser3", + following_address: "http://remote.org/following/fuser3" ) {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) @@ -1724,28 +1716,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do test "doesn't crash when follower and following counters are hidden" do mock(fn env -> case env.url do - "http://localhost:4001/users/masto_hidden_counters/following" -> + "http://remote.org/users/masto_hidden_counters/following" -> json( %{ "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "http://localhost:4001/users/masto_hidden_counters/followers" + "id" => "http://remote.org/users/masto_hidden_counters/following" }, headers: HttpRequestMock.activitypub_object_headers() ) - "http://localhost:4001/users/masto_hidden_counters/following?page=1" -> + "http://remote.org/users/masto_hidden_counters/following?page=1" -> %Tesla.Env{status: 403, body: ""} - "http://localhost:4001/users/masto_hidden_counters/followers" -> + "http://remote.org/users/masto_hidden_counters/followers" -> json( %{ "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "http://localhost:4001/users/masto_hidden_counters/following" + "id" => "http://remote.org/users/masto_hidden_counters/followers" }, headers: HttpRequestMock.activitypub_object_headers() ) - "http://localhost:4001/users/masto_hidden_counters/followers?page=1" -> + "http://remote.org/users/masto_hidden_counters/followers?page=1" -> %Tesla.Env{status: 403, body: ""} end end) @@ -1753,8 +1745,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user = insert(:user, local: false, - follower_address: "http://localhost:4001/users/masto_hidden_counters/followers", - following_address: "http://localhost:4001/users/masto_hidden_counters/following" + follower_address: "http://remote.org/users/masto_hidden_counters/followers", + following_address: "http://remote.org/users/masto_hidden_counters/following" ) {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index 59baa3a43..ba5087f1b 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -7,9 +7,57 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do alias Pleroma.Config alias Pleroma.Emoji + alias Pleroma.Emoji.Pack alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy + defp has_pack?() do + case Pack.load_pack("stolen") do + {:ok, _pack} -> true + {:error, :enoent} -> false + end + end + + defp has_emoji?(shortcode) do + case Pack.load_pack("stolen") do + {:ok, pack} -> Map.has_key?(pack.files, shortcode) + {:error, :enoent} -> false + end + end + + defmacro mock_tesla( + url \\ "https://example.org/emoji/firedfox.png", + status \\ 200, + headers \\ [], + get_body \\ File.read!("test/fixtures/image.jpg") + ) do + quote do + Tesla.Mock.mock(fn + %{method: :head, url: unquote(url)} -> + %Tesla.Env{ + status: unquote(status), + body: nil, + url: unquote(url), + headers: unquote(headers) + } + + %{method: :get, url: unquote(url)} -> + %Tesla.Env{ + status: unquote(status), + body: unquote(get_body), + url: unquote(url), + headers: unquote(headers) + } + end) + end + end + setup do + clear_config(:mrf_steal_emoji, + hosts: ["example.org"], + size_limit: 284_468, + download_unknown_size: true + ) + emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen") Emoji.reload() @@ -26,41 +74,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do File.rm_rf!(emoji_path) end) - [message: message, path: emoji_path] + [message: message] end test "does nothing by default", %{message: message} do refute "firedfox" in installed() + clear_config(:mrf_steal_emoji, []) assert {:ok, _message} = StealEmojiPolicy.filter(message) refute "firedfox" in installed() end test "Steals emoji on unknown shortcode from allowed remote host", %{ - message: message, - path: path + message: message } do refute "firedfox" in installed() - refute File.exists?(path) + refute has_pack?() - Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} - end) - - clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + mock_tesla() assert {:ok, _message} = StealEmojiPolicy.filter(message) assert "firedfox" in installed() - assert File.exists?(path) + assert has_pack?() - assert path - |> Path.join("firedfox.png") - |> File.exists?() + assert has_emoji?("firedfox") end - test "rejects invalid shortcodes", %{path: path} do + test "rejects invalid shortcodes" do message = %{ "type" => "Create", "object" => %{ @@ -69,31 +111,38 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do } } - fullpath = Path.join(path, "fired/fox.png") - - Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} - end) - - clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + mock_tesla() refute "firedfox" in installed() - refute File.exists?(path) + refute has_pack?() assert {:ok, _message} = StealEmojiPolicy.filter(message) refute "fired/fox" in installed() - refute File.exists?(fullpath) + refute has_emoji?("fired/fox") + end + + test "prefers content-type header for extension" do + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.fud"}], + "actor" => "https://example.org/users/admin" + } + } + + mock_tesla("https://example.org/emoji/firedfox.fud", 200, [{"content-type", "image/gif"}]) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + assert "firedfox" in installed() + assert has_emoji?("firedfox") end test "reject regex shortcode", %{message: message} do refute "firedfox" in installed() - clear_config(:mrf_steal_emoji, - hosts: ["example.org"], - size_limit: 284_468, - rejected_shortcodes: [~r/firedfox/] - ) + clear_config([:mrf_steal_emoji, :rejected_shortcodes], [~r/firedfox/]) assert {:ok, _message} = StealEmojiPolicy.filter(message) @@ -103,11 +152,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do test "reject string shortcode", %{message: message} do refute "firedfox" in installed() - clear_config(:mrf_steal_emoji, - hosts: ["example.org"], - size_limit: 284_468, - rejected_shortcodes: ["firedfox"] - ) + clear_config([:mrf_steal_emoji, :rejected_shortcodes], ["firedfox"]) assert {:ok, _message} = StealEmojiPolicy.filter(message) @@ -117,11 +162,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do test "reject if size is above the limit", %{message: message} do refute "firedfox" in installed() - Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} - end) + mock_tesla() - clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000) + clear_config([:mrf_steal_emoji, :size_limit], 50_000) assert {:ok, _message} = StealEmojiPolicy.filter(message) @@ -131,11 +174,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do test "reject if host returns error", %{message: message} do refute "firedfox" in installed() - Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> - {:ok, %Tesla.Env{status: 404, body: "Not found"}} - end) - - clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + mock_tesla("https://example.org/emoji/firedfox.png", 404, [], "Not found") ExUnit.CaptureLog.capture_log(fn -> assert {:ok, _message} = StealEmojiPolicy.filter(message) @@ -144,5 +183,44 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do refute "firedfox" in installed() end + test "reject unknown size", %{message: message} do + clear_config([:mrf_steal_emoji, :download_unknown_size], false) + mock_tesla() + + refute "firedfox" in installed() + + ExUnit.CaptureLog.capture_log(fn -> + assert {:ok, _message} = StealEmojiPolicy.filter(message) + end) =~ + "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png: {:remote_size, false}" + + refute "firedfox" in installed() + end + + test "reject too large content-size before download", %{message: message} do + clear_config([:mrf_steal_emoji, :download_unknown_size], false) + mock_tesla("https://example.org/emoji/firedfox.png", 200, [{"content-length", 2 ** 30}]) + + refute "firedfox" in installed() + + ExUnit.CaptureLog.capture_log(fn -> + assert {:ok, _message} = StealEmojiPolicy.filter(message) + end) =~ + "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png: {:remote_size, false}" + + refute "firedfox" in installed() + end + + test "accepts content-size below limit", %{message: message} do + clear_config([:mrf_steal_emoji, :download_unknown_size], false) + mock_tesla("https://example.org/emoji/firedfox.png", 200, [{"content-length", 2}]) + + refute "firedfox" in installed() + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + assert "firedfox" in installed() + end + defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end) end diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index 9150b8d41..f8dec09d3 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -11,6 +11,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do import Pleroma.Factory describe "attachments" do + test "works with apng" do + attachment = + %{ + "mediaType" => "image/apng", + "name" => "", + "type" => "Document", + "url" => + "https://media.misskeyusercontent.com/io/2859c26e-cd43-4550-848b-b6243bc3fe28.apng" + } + + assert {:ok, attachment} = + AttachmentValidator.cast_and_validate(attachment) + |> Ecto.Changeset.apply_action(:insert) + + assert attachment.mediaType == "image/apng" + end + test "works with honkerific attachments" do attachment = %{ "mediaType" => "", diff --git a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs index 7f02bff8f..8829597eb 100644 --- a/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/mascot_controller_test.exs @@ -64,6 +64,10 @@ defmodule Pleroma.Web.PleromaAPI.MascotControllerTest do assert json_response_and_validate_schema(ret_conn, 200) + %{"url" => uploaded_url} = Jason.decode!(ret_conn.resp_body) + + assert uploaded_url != nil and is_binary(uploaded_url) + user = User.get_cached_by_id(user.id) conn = @@ -72,6 +76,6 @@ defmodule Pleroma.Web.PleromaAPI.MascotControllerTest do |> get("/api/v1/pleroma/mascot") assert %{"url" => url, "type" => "image"} = json_response_and_validate_schema(conn, 200) - assert url =~ "an_image" + assert url == uploaded_url end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 6772a7421..cc0e22af1 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -572,6 +572,7 @@ defmodule HttpRequestMock do }} end + # Mastodon status via display URL def get( "http://mastodon.example.org/@admin/99541947525187367", _, @@ -581,6 +582,23 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, + url: "http://mastodon.example.org/@admin/99541947525187367", + body: File.read!("test/fixtures/mastodon-note-object.json"), + headers: activitypub_object_headers() + }} + end + + # same status via its canonical ActivityPub id + def get( + "http://mastodon.example.org/users/admin/statuses/99541947525187367", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + url: "http://mastodon.example.org/users/admin/statuses/99541947525187367", body: File.read!("test/fixtures/mastodon-note-object.json"), headers: activitypub_object_headers() }} @@ -964,7 +982,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}} end - def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do + def get("http://remote.org/users/masto_closed/followers", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -973,7 +991,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do + def get("http://remote.org/users/masto_closed/followers?page=1", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -982,7 +1000,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:4001/users/masto_closed/following", _, _, _) do + def get("http://remote.org/users/masto_closed/following", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -991,7 +1009,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do + def get("http://remote.org/users/masto_closed/following?page=1", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -1000,7 +1018,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:8080/followers/fuser3", _, _, _) do + def get("http://remote.org/followers/fuser3", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -1009,7 +1027,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:8080/following/fuser3", _, _, _) do + def get("http://remote.org/following/fuser3", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -1018,7 +1036,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:4001/users/fuser2/followers", _, _, _) do + def get("http://remote.org/users/fuser2/followers", _, _, _) do {:ok, %Tesla.Env{ status: 200, @@ -1027,7 +1045,7 @@ defmodule HttpRequestMock do }} end - def get("http://localhost:4001/users/fuser2/following", _, _, _) do + def get("http://remote.org/users/fuser2/following", _, _, _) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/test_helper.exs b/test/test_helper.exs index 0fc7a86b9..22a0f33ee 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -9,6 +9,10 @@ Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) {:ok, _} = Application.ensure_all_started(:ex_machina) +# Prepare and later automatically cleanup upload dir +uploads_dir = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads") +File.mkdir_p!(uploads_dir) + ExUnit.after_suite(fn _results -> uploads = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads") File.rm_rf!(uploads)