Merge branch 'develop' into stable

This commit is contained in:
FloatingGhost 2024-03-30 11:44:44 +00:00
commit 14515d8d4a
56 changed files with 2170 additions and 358 deletions

View file

@ -4,6 +4,41 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## 2024.03
## Added
- CLI tasks best-effort checking for past abuse of the recent spoofing exploit
- new `:mrf_steal_emoji, :download_unknown_size` option; defaults to `false`
## Changed
- `Pleroma.Upload, :base_url` now MUST be configured explicitly if used;
use of the same domain as the instance is **strongly** discouraged
- `:media_proxy, :base_url` now MUST be configured explicitly if used;
use of the same domain as the instance is **strongly** discouraged
- StealEmoji:
- now uses the pack.json format;
existing users must migrate with an out-of-band script (check release notes)
- only steals shortcodes recognised as valid
- URLs of stolen emoji is no longer predictable
- The `Dedupe` upload filter is now always active;
`AnonymizeFilenames` is again opt-in
- received AP data is sanity checked before we attempt to parse it as a user
- Uploads, emoji and media proxy now restrict Content-Type headers to a safe subset
- Akkoma will no longer fetch and parse objects hosted on the same domain
## Fixed
- Critical security issue allowing Akkoma to be used as a vector for
(depending on configuration) impersonation of other users or creation
of bogus users and posts on the upload domain
- Critical security issue letting Akkoma fall for the above impersonation
payloads due to lack of strict id checking
- Critical security issue allowing domains redirect to to pose as the initial domain
(e.g. with media proxy's fallback redirects)
- refetched objects can no longer attribute themselves to third-party actors
(this had no externally visible effect since actor info is read from the Create activity)
- our litepub JSON-LD schema is now served with the correct content type
- remote APNG attachments are now recognised as images
## 2024.02
## Added

View file

@ -1,16 +1,21 @@
# Pleroma backend security policy
## Supported versions
Currently, Pleroma offers bugfixes and security patches only for the latest minor release.
| Version | Support
|---------| --------
| 2.2 | Bugfixes and security patches
# Akkoma backend security handling
## Reporting a vulnerability
Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities.
Please send an email (preferably encrypted) or
a DM via our IRC to one of the following people:
| Forgejo nick | IRC nick | Email | GPG |
| ------------ | ------------- | ------------- | --------------------------------------- |
| floatinghost | FloatingGhost | *see GPG key* | https://coffee-and-dreams.uk/pubkey.asc |
## Announcements
New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at <https://pleroma.social/announcements/tags/security/feed.xml>.
New releases and security issues are announced at
[meta.akkoma.dev](https://meta.akkoma.dev/c/releases) and
[@akkoma@ihatebeinga.live](https://ihatebeinga.live/akkoma).
Both also offer RSS feeds
([meta](https://meta.akkoma.dev/c/releases/7.rss),
[fedi](https://ihatebeinga.live/users/akkoma.rss))
so you can keep an eye on it without any accounts.

View file

@ -61,11 +61,12 @@
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
filters: [Pleroma.Upload.Filter.Dedupe],
filters: [],
link_name: false,
proxy_remote: false,
filename_display_max_length: 30,
base_url: nil
base_url: nil,
allowed_mime_types: ["image", "audio", "video"]
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
@ -148,18 +149,38 @@
format: "$metadata[$level] $message",
metadata: [:request_id]
# ———————————————————————————————————————————————————————————————
# W A R N I N G
# ———————————————————————————————————————————————————————————————
#
# Whenever adding a privileged new custom type for e.g.
# ActivityPub objects, ALWAYS map their extension back
# to "application/octet-stream".
# Else files served by us can automatically end up with
# those privileged types causing severe security hazards.
# (We need those mappings so Phoenix can assoiate its format
# (the "extension") to incoming requests of those MIME types)
#
# ———————————————————————————————————————————————————————————————
config :mime, :types, %{
"application/xml" => ["xml"],
"application/xrd+xml" => ["xrd+xml"],
"application/jrd+json" => ["jrd+json"],
"application/activity+json" => ["activity+json"],
"application/ld+json" => ["activity+json"]
"application/ld+json" => ["activity+json"],
# Can be removed when bumping MIME past 2.0.5
# see https://akkoma.dev/AkkomaGang/akkoma/issues/657
"image/apng" => ["apng"]
}
config :mime, :extensions, %{
"activity+json" => "application/activity+json"
"xrd+xml" => "text/plain",
"jrd+json" => "text/plain",
"activity+json" => "text/plain"
}
# ———————————————————————————————————————————————————————————————
config :tesla, :adapter, {Tesla.Adapter.Finch, name: MyFinch}
# Configures http settings, upstream proxy etc.

View file

@ -105,6 +105,19 @@
"https://cdn-host.com"
]
},
%{
key: :allowed_mime_types,
label: "Allowed MIME types",
type: {:list, :string},
description:
"List of MIME (main) types uploads are allowed to identify themselves with. Other types may still be uploaded, but will identify as a generic binary to clients. WARNING: Loosening this over the defaults can lead to security issues. Removing types is safe, but only add to the list if you are sure you know what you are doing.",
suggestions: [
"image",
"audio",
"video",
"font"
]
},
%{
key: :proxy_remote,
type: :boolean,

View file

@ -0,0 +1,56 @@
# Security-related tasks
{! administration/CLI_tasks/general_cli_task_info.include !}
!!! danger
Many of these tasks were written in response to a patched exploit.
It is recommended to run those very soon after installing its respective security update.
Over time with db migrations they might become less accurate or be removed altogether.
If you never ran an affected version, theres no point in running them.
## Spoofed AcitivityPub objects exploit (2024-03, fixed in 3.11.1)
### Search for uploaded spoofing payloads
Scans local uploads for spoofing payloads.
If the instance is not using the local uploader it was not affected.
Attachments wil be scanned anyway in case local uploader was used in the past.
!!! note
This cannot reliably detect payloads attached to deleted posts.
=== "OTP"
```sh
./bin/pleroma_ctl security spoof-uploaded
```
=== "From Source"
```sh
mix pleroma.security spoof-uploaded
```
### Search for counterfeit posts in database
Scans all notes in the database for signs of being spoofed.
!!! note
Spoofs targeting local accounts can be detected rather reliably
(with some restrictions documented in the tasks logs).
Counterfeit posts from remote users cannot. A best-effort attempt is made, but
a thorough attacker can avoid this and it may yield a small amount of false positives.
Should you find counterfeit posts of local users, let other admins know so they can delete the too.
=== "OTP"
```sh
./bin/pleroma_ctl security spoof-inserted
```
=== "From Source"
```sh
mix pleroma.security spoof-inserted
```

View file

@ -236,7 +236,9 @@ config :pleroma, :mrf_user_allowlist, %{
#### :mrf_steal_emoji
* `hosts`: List of hosts to steal emojis from
* `rejected_shortcodes`: Regex-list of shortcodes to reject
* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
* `size_limit`: File size limit (in bytes), checked before download if possible (and remote server honest),
otherwise or again checked before saving emoji to the disk
* `download_unknown_size`: whether to download an emoji when the remote server doesnt report its size in advance
#### :mrf_activity_expiration
@ -396,7 +398,8 @@ This section describe PWA manifest instance-specific values. Currently this opti
## :media_proxy
* `enabled`: Enables proxying of remote media to the instances proxy
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
* `base_url`: The base URL to access a user-uploaded file.
Using a (sub)domain distinct from the instance endpoint is **strongly** recommended.
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
* `whitelist`: List of hosts with scheme to bypass the mediaproxy (e.g. `https://example.com`)
* `invalidation`: options for remove media from cache after delete object:
@ -597,8 +600,9 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
* `uploader`: Which one of the [uploaders](#uploaders) to use.
* `filters`: List of [upload filters](#upload-filters) to use.
* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to host the media files via another domain or are using a 3rd party S3 provider.
* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
* `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
Using a (sub)domain distinct from the instance endpoint is **strongly** recommended.
* `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
* `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
@ -638,17 +642,18 @@ config :ex_aws, :s3,
### Upload filters
#### Pleroma.Upload.Filter.AnonymizeFilename
This filter replaces the filename (not the path) of an upload. For complete obfuscation, add
`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename.
* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
#### Pleroma.Upload.Filter.Dedupe
**Always** active; cannot be turned off.
Renames files to their hash and prevents duplicate files filling up the disk.
No specific configuration.
#### Pleroma.Upload.Filter.AnonymizeFilename
This filter replaces the declared filename (not the path) of an upload.
* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
#### Pleroma.Upload.Filter.Exiftool
This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.

View file

@ -17,6 +17,16 @@ This sets the Akkoma application server to only listen to the localhost interfac
This sets the `secure` flag on Akkomas session cookie. This makes sure, that the cookie is only accepted over encrypted HTTPs connections. This implicitly renames the cookie from `pleroma_key` to `__Host-pleroma-key` which enforces some restrictions. (see [cookie prefixes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#Cookie_prefixes))
### `Pleroma.Upload, :uploader, :base_url`
> Recommended value: *anything on a different domain than the instance endpoint; e.g. https://media.myinstance.net/*
Uploads are user controlled and (unless youre running a true single-user
instance) should therefore not be considered trusted. But the domain is used
as a pivilege boundary e.g. by HTTP content security policy and ActivityPub.
Having uploads on the same domain enabled several past vulnerabilities
able to be exploited by malicious users.
### `:http_security`
> Recommended value: `true`

View file

@ -6,7 +6,16 @@ With the `mediaproxy` function you can use nginx to cache this content, so users
## Activate it
* Edit your nginx config and add the following location:
* Edit your nginx config and add the following location to your main server block:
```
location /proxy {
return 404;
}
```
* Set up a subdomain for the proxy with its nginx config on the same machine
*(the latter is not strictly required, but for simplicity well assume so)*
* In this subdomains server block add
```
location /proxy {
proxy_cache akkoma_media_cache;
@ -26,9 +35,9 @@ config :pleroma, :media_proxy,
enabled: true,
proxy_opts: [
redirect_on_failure: true
]
#base_url: "https://cache.akkoma.social"
],
base_url: "https://cache.akkoma.social"
```
If you want to use a subdomain to serve the files, uncomment `base_url`, change the url and add a comma after `true` in the previous line.
You **really** should use a subdomain to serve proxied files; while we will fix bugs resulting from this, serving arbitrary remote content on your main domain namespace is a significant attack surface.
* Restart nginx and Akkoma

View file

@ -75,9 +75,48 @@ server {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
location ~ ^/(media|proxy) {
return 404;
}
location / {
proxy_pass http://phoenix;
}
}
# Upload and MediaProxy Subdomain
# (see main domain setup for more details)
server {
server_name media.example.tld;
listen 80;
listen [::]:80;
location / {
return 301 https://$server_name$request_uri;
}
}
server {
server_name media.example.tld;
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_trusted_certificate /etc/letsencrypt/live/media.example.tld/chain.pem;
ssl_certificate /etc/letsencrypt/live/media.example.tld/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/media.example.tld/privkey.pem;
# .. copy all other the ssl_* and gzip_* stuff from main domain
# the nginx default is 1m, not enough for large media uploads
client_max_body_size 16m;
ignore_invalid_headers off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
location ~ ^/(media|proxy) {
proxy_cache akkoma_media_cache;
@ -91,4 +130,8 @@ server {
chunked_transfer_encoding on;
proxy_pass http://phoenix;
}
location / {
return 404;
}
}

View file

@ -20,6 +20,7 @@ def run(["gen" | rest]) do
output: :string,
output_psql: :string,
domain: :string,
media_url: :string,
instance_name: :string,
admin_email: :string,
notify_email: :string,
@ -35,8 +36,7 @@ def run(["gen" | rest]) do
listen_ip: :string,
listen_port: :string,
strip_uploads: :string,
anonymize_uploads: :string,
dedupe_uploads: :string
anonymize_uploads: :string
],
aliases: [
o: :output,
@ -64,6 +64,14 @@ def run(["gen" | rest]) do
":"
) ++ [443]
media_url =
get_option(
options,
:media_url,
"What base url will uploads use? (e.g https://media.example.com/media)\n" <>
" Generally this should NOT use the same domain as the instance "
)
name =
get_option(
options,
@ -186,14 +194,6 @@ def run(["gen" | rest]) do
"n"
) === "y"
dedupe_uploads =
get_option(
options,
:dedupe_uploads,
"Do you want to deduplicate uploaded files? (y/n)",
"n"
) === "y"
Config.put([:instance, :static_dir], static_dir)
secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@ -207,6 +207,7 @@ def run(["gen" | rest]) do
EEx.eval_file(
template_dir <> "/sample_config.eex",
domain: domain,
media_url: media_url,
port: port,
email: email,
notify_email: notify_email,
@ -230,8 +231,7 @@ def run(["gen" | rest]) do
upload_filters:
upload_filters(%{
strip: strip_uploads,
anonymize: anonymize_uploads,
dedupe: dedupe_uploads
anonymize: anonymize_uploads
})
)
@ -319,13 +319,6 @@ defp upload_filters(filters) when is_map(filters) do
enabled_filters
end
enabled_filters =
if filters.dedupe do
enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
else
enabled_filters
end
enabled_filters
end
end

View file

@ -0,0 +1,330 @@
# Akkoma: Magically expressive social media
# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Security do
use Mix.Task
import Ecto.Query
import Mix.Pleroma
alias Pleroma.Config
require Logger
@shortdoc """
Security-related tasks, like e.g. checking for signs past exploits were abused.
"""
# Constants etc
defp local_id_prefix(), do: Pleroma.Web.Endpoint.url() <> "/"
defp local_id_pattern(), do: local_id_prefix() <> "%"
@activity_exts ["activity+json", "activity%2Bjson"]
defp activity_ext_url_patterns() do
for e <- @activity_exts do
for suf <- ["", "?%"] do
# Escape literal % for use in SQL patterns
ee = String.replace(e, "%", "\\%")
"%.#{ee}#{suf}"
end
end
|> List.flatten()
end
# Search for malicious uploads exploiting the lack of Content-Type sanitisation from before 2024-03
def run(["spoof-uploaded"]) do
Logger.put_process_level(self(), :notice)
start_pleroma()
IO.puts("""
+------------------------+
| SPOOF SEARCH UPLOADS |
+------------------------+
Checking if any uploads are using privileged types.
NOTE if attachment deletion is enabled, payloads used
in the past may no longer exist.
""")
do_spoof_uploaded()
end
# Fuzzy search for potentially counterfeit activities in the database resulting from the same exploit
def run(["spoof-inserted"]) do
Logger.put_process_level(self(), :notice)
start_pleroma()
IO.puts("""
+----------------------+
| SPOOF SEARCH NOTES |
+----------------------+
Starting fuzzy search for counterfeit activities.
NOTE this can not guarantee detecting all counterfeits
and may yield a small percentage of false positives.
""")
do_spoof_inserted()
end
# +-----------------------------+
# | S P O O F - U P L O A D E D |
# +-----------------------------+
defp do_spoof_uploaded() do
files =
case Config.get!([Pleroma.Upload, :uploader]) do
Pleroma.Uploaders.Local ->
uploads_search_spoofs_local_dir(Config.get!([Pleroma.Uploaders.Local, :uploads]))
_ ->
IO.puts("""
NOTE:
Not using local uploader; thus not affected by this exploit.
It's impossible to check for files, but in case local uploader was used before
or to check if anyone futilely attempted a spoof, notes will still be scanned.
""")
[]
end
emoji = uploads_search_spoofs_local_dir(Config.get!([:instance, :static_dir]))
post_attachs = uploads_search_spoofs_notes()
not_orphaned_urls =
post_attachs
|> Enum.map(fn {_u, _a, url} -> url end)
|> MapSet.new()
orphaned_attachs = upload_search_orphaned_attachments(not_orphaned_urls)
IO.puts("\nSearch concluded; here are the results:")
pretty_print_list_with_title(emoji, "Emoji")
pretty_print_list_with_title(files, "Uploaded Files")
pretty_print_list_with_title(post_attachs, "(Not Deleted) Post Attachments")
pretty_print_list_with_title(orphaned_attachs, "Orphaned Uploads")
IO.puts("""
In total found
#{length(emoji)} emoji
#{length(files)} uploads
#{length(post_attachs)} not deleted posts
#{length(orphaned_attachs)} orphaned attachments
""")
end
defp uploads_search_spoofs_local_dir(dir) do
local_dir = String.replace_suffix(dir, "/", "")
IO.puts("Searching for suspicious files in #{local_dir}...")
glob_ext = "{" <> Enum.join(@activity_exts, ",") <> "}"
Path.wildcard(local_dir <> "/**/*." <> glob_ext, match_dot: true)
|> Enum.map(fn path ->
String.replace_prefix(path, local_dir <> "/", "")
end)
|> Enum.sort()
end
defp uploads_search_spoofs_notes() do
IO.puts("Now querying DB for posts with spoofing attachments. This might take a while...")
patterns = [local_id_pattern() | activity_ext_url_patterns()]
# if jsonb_array_elemsts in FROM can be used with normal Ecto functions, idk how
"""
SELECT DISTINCT a.data->>'actor', a.id, url->>'href'
FROM public.objects AS o JOIN public.activities AS a
ON o.data->>'id' = a.data->>'object',
jsonb_array_elements(o.data->'attachment') AS attachs,
jsonb_array_elements(attachs->'url') AS url
WHERE o.data->>'type' = 'Note' AND
o.data->>'id' LIKE $1::text AND (
url->>'href' LIKE $2::text OR
url->>'href' LIKE $3::text OR
url->>'href' LIKE $4::text OR
url->>'href' LIKE $5::text
)
ORDER BY a.data->>'actor', a.id, url->>'href';
"""
|> Pleroma.Repo.query!(patterns, timeout: :infinity)
|> map_raw_id_apid_tuple()
end
defp upload_search_orphaned_attachments(not_orphaned_urls) do
IO.puts("""
Now querying DB for orphaned spoofing attachment (i.e. their post was deleted,
but if :cleanup_attachments was not enabled traces remain in the database)
This might take a bit...
""")
patterns = activity_ext_url_patterns()
"""
SELECT DISTINCT attach.id, url->>'href'
FROM public.objects AS attach,
jsonb_array_elements(attach.data->'url') AS url
WHERE (attach.data->>'type' = 'Image' OR
attach.data->>'type' = 'Document')
AND (
url->>'href' LIKE $1::text OR
url->>'href' LIKE $2::text OR
url->>'href' LIKE $3::text OR
url->>'href' LIKE $4::text
)
ORDER BY attach.id, url->>'href';
"""
|> Pleroma.Repo.query!(patterns, timeout: :infinity)
|> then(fn res -> Enum.map(res.rows, fn [id, url] -> {id, url} end) end)
|> Enum.filter(fn {_, url} -> !(url in not_orphaned_urls) end)
end
# +-----------------------------+
# | S P O O F - I N S E R T E D |
# +-----------------------------+
defp do_spoof_inserted() do
IO.puts("""
Searching for local posts whose Create activity has no ActivityPub id...
This is a pretty good indicator, but only for spoofs of local actors
and only if the spoofing happened after around late 2021.
""")
idless_create =
search_local_notes_without_create_id()
|> Enum.sort()
IO.puts("Done.\n")
IO.puts("""
Now trying to weed out other poorly hidden spoofs.
This can't detect all and may have some false positives.
""")
likely_spoofed_posts_set = MapSet.new(idless_create)
sus_pattern_posts =
search_sus_notes_by_id_patterns()
|> Enum.filter(fn r -> !(r in likely_spoofed_posts_set) end)
IO.puts("Done.\n")
IO.puts("""
Finally, searching for spoofed, local user accounts.
(It's impossible to detect spoofed remote users)
""")
spoofed_users = search_bogus_local_users()
pretty_print_list_with_title(sus_pattern_posts, "Maybe Spoofed Posts")
pretty_print_list_with_title(idless_create, "Likely Spoofed Posts")
pretty_print_list_with_title(spoofed_users, "Spoofed local user accounts")
IO.puts("""
In total found:
#{length(spoofed_users)} bogus users
#{length(idless_create)} likely spoofed posts
#{length(sus_pattern_posts)} maybe spoofed posts
""")
end
defp search_local_notes_without_create_id() do
Pleroma.Object
|> where([o], fragment("?->>'id' LIKE ?", o.data, ^local_id_pattern()))
|> join(:inner, [o], a in Pleroma.Activity,
on: fragment("?->>'object' = ?->>'id'", a.data, o.data)
)
|> where([o, a], fragment("NOT (? \\? 'id') OR ?->>'id' IS NULL", a.data, a.data))
|> select([o, a], {a.id, fragment("?->>'id'", o.data)})
|> order_by([o, a], a.id)
|> Pleroma.Repo.all()
end
defp search_sus_notes_by_id_patterns() do
[ep1, ep2, ep3, ep4] = activity_ext_url_patterns()
Pleroma.Object
|> where(
[o],
# for local objects we know exactly how a genuine id looks like
# (though a thorough attacker can emulate this)
# for remote posts, use some best-effort patterns
fragment(
"""
(?->>'id' LIKE ? AND ?->>'id' NOT SIMILAR TO
? || 'objects/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}')
""",
o.data,
^local_id_pattern(),
o.data,
^local_id_prefix()
) or
fragment("?->>'id' LIKE ?", o.data, "%/emoji/%") or
fragment("?->>'id' LIKE ?", o.data, "%/media/%") or
fragment("?->>'id' LIKE ?", o.data, "%/proxy/%") or
fragment("?->>'id' LIKE ?", o.data, ^ep1) or
fragment("?->>'id' LIKE ?", o.data, ^ep2) or
fragment("?->>'id' LIKE ?", o.data, ^ep3) or
fragment("?->>'id' LIKE ?", o.data, ^ep4)
)
|> join(:inner, [o], a in Pleroma.Activity,
on: fragment("?->>'object' = ?->>'id'", a.data, o.data)
)
|> select([o, a], {a.id, fragment("?->>'id'", o.data)})
|> order_by([o, a], a.id)
|> Pleroma.Repo.all()
end
defp search_bogus_local_users() do
Pleroma.User.Query.build(%{})
|> where([u], u.local == false and like(u.ap_id, ^local_id_pattern()))
|> order_by([u], u.ap_id)
|> select([u], u.ap_id)
|> Pleroma.Repo.all()
end
# +-----------------------------------+
# | module-specific utility functions |
# +-----------------------------------+
defp pretty_print_list_with_title(list, title) do
title_len = String.length(title)
title_underline = String.duplicate("=", title_len)
IO.puts(title)
IO.puts(title_underline)
pretty_print_list(list)
end
defp pretty_print_list([]), do: IO.puts("")
defp pretty_print_list([{a, o} | rest])
when (is_binary(a) or is_number(a)) and is_binary(o) do
IO.puts(" {#{a}, #{o}}")
pretty_print_list(rest)
end
defp pretty_print_list([{u, a, o} | rest])
when is_binary(a) and is_binary(u) and is_binary(o) do
IO.puts(" {#{u}, #{a}, #{o}}")
pretty_print_list(rest)
end
defp pretty_print_list([e | rest]) when is_binary(e) do
IO.puts(" #{e}")
pretty_print_list(rest)
end
defp pretty_print_list([e | rest]), do: pretty_print_list([inspect(e) | rest])
defp map_raw_id_apid_tuple(res) do
user_prefix = local_id_prefix() <> "users/"
Enum.map(res.rows, fn
[uid, aid, oid] ->
{
String.replace_prefix(uid, user_prefix, ""),
FlakeId.to_string(aid),
oid
}
end)
end
end

View file

@ -26,12 +26,37 @@ defmodule Pleroma.Emoji.Pack do
alias Pleroma.Emoji.Pack
alias Pleroma.Utils
# Invalid/Malicious names are supposed to be filtered out before path joining,
# but there are many entrypoints to affected functions so as the code changes
# we might accidentally let an unsanitised name slip through.
# To make sure, use the below which crash the process otherwise.
# ALWAYS use this when constructing paths from external name!
# (name meaning it must be only a single path component)
defp path_join_name_safe(dir, name) do
if to_string(name) != Path.basename(name) or name in ["..", ".", ""] do
raise "Invalid or malicious pack name: #{name}"
else
Path.join(dir, name)
end
end
# ALWAYS use this to join external paths
# (which are allowed to have several components)
defp path_join_safe(dir, path) do
{:ok, safe_path} = Path.safe_relative(path)
Path.join(dir, safe_path)
end
@spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
def create(name) do
with :ok <- validate_not_empty([name]),
dir <- Path.join(emoji_path(), name),
dir <- path_join_name_safe(emoji_path(), name),
:ok <- File.mkdir(dir) do
save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
save_pack(%__MODULE__{
path: dir,
pack_file: Path.join(dir, "pack.json")
})
end
end
@ -65,7 +90,7 @@ def show(opts) do
{:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
def delete(name) do
with :ok <- validate_not_empty([name]),
pack_path <- Path.join(emoji_path(), name) do
pack_path <- path_join_name_safe(emoji_path(), name) do
File.rm_rf(pack_path)
end
end
@ -89,7 +114,7 @@ defp unpack_zip_emojies(zip_files) do
end)
end
@spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
@spec add_file(t(), String.t(), Path.t(), Plug.Upload.t() | binary()) ::
{:ok, t()}
| {:error, File.posix() | atom()}
def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
@ -107,7 +132,7 @@ def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"}
Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
emoji_file = %Plug.Upload{
filename: item[:filename],
path: Path.join(tmp_dir, item[:path])
path: path_join_safe(tmp_dir, item[:path])
}
{:ok, updated_pack} =
@ -137,6 +162,14 @@ def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"}
end
def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
try_add_file(pack, shortcode, filename, file)
end
def add_file(%Pack{} = pack, shortcode, filename, filedata) when is_binary(filedata) do
try_add_file(pack, shortcode, filename, filedata)
end
defp try_add_file(%Pack{} = pack, shortcode, filename, file) do
with :ok <- validate_not_empty([shortcode, filename]),
:ok <- validate_emoji_not_exists(shortcode),
{:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
@ -189,6 +222,7 @@ def import_from_filesystem do
{:ok, results} <- File.ls(emoji_path) do
names =
results
# items come from File.ls, thus safe
|> Enum.map(&Path.join(emoji_path, &1))
|> Enum.reject(fn path ->
File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
@ -287,8 +321,8 @@ def update_metadata(name, data) do
@spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
def load_pack(name) do
name = Path.basename(name)
pack_file = Path.join([emoji_path(), name, "pack.json"])
pack_dir = path_join_name_safe(emoji_path(), name)
pack_file = Path.join(pack_dir, "pack.json")
with {:ok, _} <- File.stat(pack_file),
{:ok, pack_data} <- File.read(pack_file) do
@ -412,7 +446,13 @@ defp downloadable?(pack) do
end
defp create_archive_and_cache(pack, hash) do
files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
files = [
~c"pack.json"
| Enum.map(pack.files, fn {_, file} ->
{:ok, file} = Path.safe_relative(file)
to_charlist(file)
end)
]
{:ok, {_, result}} =
:zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)])
@ -474,7 +514,7 @@ defp validate_not_empty(list) do
end
defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
file_path = Path.join(pack.path, filename)
file_path = path_join_safe(pack.path, filename)
create_subdirs(file_path)
with {:ok, _} <- File.copy(upload_path, file_path) do
@ -482,6 +522,12 @@ defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
end
end
defp save_file(file_data, pack, filename) when is_binary(file_data) do
file_path = path_join_safe(pack.path, filename)
create_subdirs(file_path)
File.write(file_path, file_data, [:binary])
end
defp put_emoji(pack, shortcode, filename) do
files = Map.put(pack.files, shortcode, filename)
%{pack | files: files, files_count: length(Map.keys(files))}
@ -493,8 +539,8 @@ defp delete_emoji(pack, shortcode) do
end
defp rename_file(pack, filename, new_filename) do
old_path = Path.join(pack.path, filename)
new_path = Path.join(pack.path, new_filename)
old_path = path_join_safe(pack.path, filename)
new_path = path_join_safe(pack.path, new_filename)
create_subdirs(new_path)
with :ok <- File.rename(old_path, new_path) do
@ -512,7 +558,7 @@ defp create_subdirs(file_path) do
defp remove_file(pack, shortcode) do
with {:ok, filename} <- get_filename(pack, shortcode),
emoji <- Path.join(pack.path, filename),
emoji <- path_join_safe(pack.path, filename),
:ok <- File.rm(emoji) do
remove_dir_if_empty(emoji, filename)
end
@ -530,7 +576,7 @@ defp remove_dir_if_empty(emoji, filename) do
defp get_filename(pack, shortcode) do
with %{^shortcode => filename} when is_binary(filename) <- pack.files,
file_path <- Path.join(pack.path, filename),
file_path <- path_join_safe(pack.path, filename),
{:ok, _} <- File.stat(file_path) do
{:ok, filename}
else
@ -568,7 +614,7 @@ defp validate_downloadable(pack) do
end
defp copy_as(remote_pack, local_name) do
path = Path.join(emoji_path(), local_name)
path = path_join_name_safe(emoji_path(), local_name)
%__MODULE__{
name: local_name,

View file

@ -11,6 +11,9 @@ defmodule Pleroma.Object.Containment do
Object containment is an important step in validating remote objects to prevent
spoofing, therefore removal of object containment functions is NOT recommended.
"""
alias Pleroma.Web.ActivityPub.Transmogrifier
def get_actor(%{"actor" => actor}) when is_binary(actor) do
actor
end
@ -47,6 +50,31 @@ def get_object(_) do
defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
defp compare_uris(_id_uri, _other_uri), do: :error
defp compare_uris_exact(uri, uri), do: :ok
defp compare_uris_exact(%URI{} = id, %URI{} = other),
do: compare_uris_exact(URI.to_string(id), URI.to_string(other))
defp compare_uris_exact(id_uri, other_uri)
when is_binary(id_uri) and is_binary(other_uri) do
norm_id = String.replace_suffix(id_uri, "/", "")
norm_other = String.replace_suffix(other_uri, "/", "")
if norm_id == norm_other, do: :ok, else: :error
end
@doc """
Checks whether an URL to fetch from is from the local server.
We never want to fetch from ourselves; if its not in the database
it cant be authentic and must be a counterfeit.
"""
def contain_local_fetch(id) do
case compare_uris(URI.parse(id), Pleroma.Web.Endpoint.struct_url()) do
:ok -> :error
_ -> :ok
end
end
@doc """
Checks that an imported AP object's actor matches the host it came from.
"""
@ -62,8 +90,31 @@ def contain_origin(id, %{"actor" => _actor} = params) do
def contain_origin(id, %{"attributedTo" => actor} = params),
do: contain_origin(id, Map.put(params, "actor", actor))
def contain_origin(_id, _data), do: :error
def contain_origin(_id, _data), do: :ok
@doc """
Check whether the fetch URL (after redirects) exactly (sans tralining slash) matches either
the canonical ActivityPub id or the objects url field (for display URLs from *key and Mastodon)
Since this is meant to be used for fetches, anonymous or transient objects are not accepted here.
"""
def contain_id_to_fetch(url, %{"id" => id} = data) when is_binary(id) do
with {:id, :error} <- {:id, compare_uris_exact(id, url)},
# "url" can be a "Link" object and this is checked before full normalisation
display_url <- Transmogrifier.fix_url(data)["url"],
true <- display_url != nil do
compare_uris_exact(display_url, url)
else
{:id, :ok} -> :ok
_ -> :error
end
end
def contain_id_to_fetch(_url, _data), do: :error
@doc """
Check whether the object id is from the same host as another id
"""
def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(other_id) do
id_uri = URI.parse(id)
other_uri = URI.parse(other_id)
@ -85,4 +136,12 @@ def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
do: contain_origin(id, object)
def contain_child(_), do: :ok
@doc "Checks whether two URIs belong to the same domain"
def same_origin(id1, id2) do
uri1 = URI.parse(id1)
uri2 = URI.parse(id2)
compare_uris(uri1, uri2)
end
end

View file

@ -18,6 +18,16 @@ defmodule Pleroma.Object.Fetcher do
require Logger
require Pleroma.Constants
@moduledoc """
This module deals with correctly fetching Acitivity Pub objects in a safe way.
The core function is `fetch_and_contain_remote_object_from_id/1` which performs
the actual fetch and common safety and authenticity checks. Other `fetch_*`
function use the former and perform some additional tasks
"""
@mix_env Mix.env()
defp touch_changeset(changeset) do
updated_at =
NaiveDateTime.utc_now()
@ -103,18 +113,26 @@ defp reinject_object(%Object{} = object, new_data) do
end
end
@doc "Assumes object already is in our database and refetches from remote to update (e.g. for polls)"
def refetch_object(%Object{data: %{"id" => id}} = object) do
with {:local, false} <- {:local, Object.local?(object)},
{:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
{:id, true} <- {:id, new_data["id"] == id},
{:ok, object} <- reinject_object(object, new_data) do
{:ok, object}
else
{:local, true} -> {:ok, object}
{:id, false} -> {:error, "Object id changed on refetch"}
e -> {:error, e}
end
end
# Note: will create a Create activity, which we need internally at the moment.
@doc """
Fetches a new object and puts it through the processing pipeline for inbound objects
Note: will also insert a fake Create activity, since atm we internally
need everything to be traced back to a Create activity.
"""
def fetch_object_from_id(id, options \\ []) do
with %URI{} = uri <- URI.parse(id),
# let's check the URI is even vaguely valid first
@ -127,7 +145,6 @@ def fetch_object_from_id(id, options \\ []) do
{_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
{_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
params <- prepare_activity_params(data),
{_, :ok} <- {:containment, Containment.contain_origin(id, params)},
{_, {:ok, activity}} <-
{:transmogrifier, Transmogrifier.handle_incoming(params, options)},
{_, _data, %Object{} = object} <-
@ -140,9 +157,6 @@ def fetch_object_from_id(id, options \\ []) do
{:scheme, false} ->
{:error, "URI Scheme Invalid"}
{:containment, _} ->
{:error, "Object containment failed."}
{:transmogrifier, {:error, {:reject, e}}} ->
{:reject, e}
@ -185,6 +199,7 @@ defp prepare_activity_params(data) do
|> Maps.put_if_present("bcc", data["bcc"])
end
@doc "Identical to `fetch_object_from_id/2` but just directly returns the object or on error `nil`"
def fetch_object_from_id!(id, options \\ []) do
with {:ok, object} <- fetch_object_from_id(id, options) do
object
@ -235,6 +250,7 @@ defp maybe_date_fetch(headers, date) do
end
end
@doc "Fetches arbitrary remote object and performs basic safety and authenticity checks"
def fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(%{"id" => id}),
@ -244,18 +260,29 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
Logger.debug("Fetching object #{id} via AP")
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{:ok, body} <- get_object(id),
{_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
{:ok, final_id, body} <- get_object(id),
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
unless Instances.reachable?(id) do
Instances.set_reachable(id)
{_, :ok} <- {:strict_id, Containment.contain_id_to_fetch(final_id, data)},
{_, :ok} <- {:containment, Containment.contain_origin(final_id, data)} do
unless Instances.reachable?(final_id) do
Instances.set_reachable(final_id)
end
{:ok, data}
else
{:strict_id, _} ->
{:error, "Object's ActivityPub id/url does not match final fetch URL"}
{:scheme, _} ->
{:error, "Unsupported URI scheme"}
{:local_fetch, _} ->
{:error, "Trying to fetch local resource"}
{:containment, _} ->
{:error, "Object containment failed."}
{:error, e} ->
{:error, e}
@ -267,6 +294,32 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
defp check_crossdomain_redirect(final_host, original_url)
# HOPEFULLY TEMPORARY
# Basically none of our Tesla mocks in tests set the (supposed to
# exist for Tesla proper) url parameter for their responses
# causing almost every fetch in test to fail otherwise
if @mix_env == :test do
defp check_crossdomain_redirect(nil, _) do
{:cross_domain_redirect, false}
end
end
defp check_crossdomain_redirect(final_host, original_url) do
{:cross_domain_redirect, final_host != URI.parse(original_url).host}
end
if @mix_env == :test do
defp get_final_id(nil, initial_url), do: initial_url
defp get_final_id("", initial_url), do: initial_url
end
defp get_final_id(final_url, _intial_url) do
final_url
end
@doc "Do NOT use; only public for use in tests"
def get_object(id) do
date = Pleroma.Signature.signed_date()
@ -275,37 +328,42 @@ def get_object(id) do
|> maybe_date_fetch(date)
|> sign_fetch(id, date)
case HTTP.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, body}
with {:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 <-
HTTP.get(id, headers),
remote_host <-
URI.parse(final_url).host,
{:cross_domain_redirect, false} <-
check_crossdomain_redirect(remote_host, id),
{:has_content_type, {_, content_type}} <-
{:has_content_type, List.keyfind(headers, "content-type", 0)},
{:parse_content_type, {:ok, "application", subtype, type_params}} <-
{:parse_content_type, Plug.Conn.Utils.media_type(content_type)} do
final_id = get_final_id(final_url, id)
{:ok, "application", "ld+json",
%{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, body}
case {subtype, type_params} do
{"activity+json", _} ->
{:ok, final_id, body}
# pixelfed sometimes (and only sometimes) responds with http instead of https
{:ok, "application", "ld+json",
%{"profile" => "http://www.w3.org/ns/activitystreams"}} ->
{:ok, body}
_ ->
{:error, {:content_type, content_type}}
end
_ ->
{:error, {:content_type, nil}}
end
{"ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, final_id, body}
_ ->
{:error, {:content_type, content_type}}
end
else
{:ok, %{status: code}} when code in [404, 410] ->
{:error, {"Object has been deleted", id, code}}
{:error, e} ->
{:error, e}
{:has_content_type, _} ->
{:error, {:content_type, nil}}
{:parse_content_type, e} ->
{:error, {:content_type, e}}
e ->
{:error, e}
end

View file

@ -17,6 +17,8 @@ defmodule Pleroma.ReverseProxy do
@failed_request_ttl :timer.seconds(60)
@methods ~w(GET HEAD)
@allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], [])
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def max_read_duration_default, do: @max_read_duration
@ -253,6 +255,7 @@ defp build_resp_headers(headers, opts) do
headers
|> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
|> build_resp_cache_headers(opts)
|> sanitise_content_type()
|> build_resp_content_disposition_header(opts)
|> build_csp_headers()
|> Keyword.merge(Keyword.get(opts, :resp_headers, []))
@ -282,6 +285,21 @@ defp build_resp_cache_headers(headers, _opts) do
end
end
defp sanitise_content_type(headers) do
original_ct = get_content_type(headers)
safe_ct =
Pleroma.Web.Plugs.Utils.get_safe_mime_type(
%{allowed_mime_types: @allowed_mime_types},
original_ct
)
[
{"content-type", safe_ct}
| Enum.filter(headers, fn {k, _v} -> k != "content-type" end)
]
end
defp build_resp_content_disposition_header(headers, opts) do
opt = Keyword.get(opts, :inline_content_types, @inline_content_types)

View file

@ -39,6 +39,8 @@ defmodule Pleroma.Upload do
alias Pleroma.Web.ActivityPub.Utils
require Logger
@mix_env Mix.env()
@type source ::
Plug.Upload.t()
| (data_uri_string :: String.t())