diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7bcca08..1a3afd0fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Pleroma Admin API: emoji packs and files routes changed. +- **Breaking:** Sensitive/NSFW statuses no longer disable link previews. - Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. diff --git a/config/config.exs b/config/config.exs index 2e6b0796a..d53663d36 100644 --- a/config/config.exs +++ b/config/config.exs @@ -677,7 +677,18 @@ config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 -config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true +config :pleroma, Pleroma.Plugs.RemoteIp, + enabled: true, + headers: ["x-forwarded-for"], + proxies: [], + reserved: [ + "127.0.0.0/8", + "::1/128", + "fc00::/7", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] config :pleroma, :static_fe, enabled: false diff --git a/config/description.exs b/config/description.exs index ac3dfbb2b..3902b9632 100644 --- a/config/description.exs +++ b/config/description.exs @@ -44,11 +44,13 @@ }, %{ key: "git", + label: "Git Repository URL", type: :string, description: "URL of the git repository of the frontend" }, %{ key: "build_url", + label: "Build URL", type: :string, description: "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.", @@ -56,6 +58,7 @@ }, %{ key: "build_dir", + label: "Build directory", type: :string, description: "The directory inside the zip file " } @@ -3262,20 +3265,22 @@ %{ key: :headers, type: {:list, :string}, - description: - "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`." + description: """ + A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`. + """ }, %{ key: :proxies, type: {:list, :string}, description: - "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`." + "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128." }, %{ key: :reserved, type: {:list, :string}, - description: - "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)." + description: """ + A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]` + """ } ] }, @@ -3681,9 +3686,7 @@ type: :map, description: "A map containing available frontends and parameters for their installation.", - children: [ - frontend_options - ] + children: frontend_options } ] }, diff --git a/docs/administration/backup.md b/docs/administration/backup.md index be57bf74a..cfedf5b58 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -5,20 +5,24 @@ 1. Stop the Pleroma service. 2. Go to the working directory of Pleroma (default is `/opt/pleroma`) 3. Run `sudo -Hu postgres pg_dump -d --format=custom -f ` (make sure the postgres user has write access to the destination file) -4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. +4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. 5. Restart the Pleroma service. ## Restore/Move -1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name. +1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. -5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'CREATE DATABASE ;';` -6. Run `sudo -Hu postgres pg_restore -d -v -1 ` -7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. -8. Restart the Pleroma service. -9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries. +5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;'` +6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. + + Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. + +7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d -v -1 ` +8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. +9. Restart the Pleroma service. +10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries. [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 42e5fe808..ea7dfec98 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -426,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start Available options: * `enabled` - Enable/disable the plug. Defaults to `false`. -* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`. -* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`. -* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network). +* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`. +* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128. +* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`. ### :rate_limit diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index c27ad1065..8979db2f8 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do html_body = """

New account for review: @#{account.nickname}

#{HTML.strip_tags(account.registration_reason)}
- Visit AdminFE + Visit AdminFE """ new() diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 0ac9050d0..987022156 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -7,48 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. """ + alias Pleroma.Config import Plug.Conn @behaviour Plug - @headers ~w[ - x-forwarded-for - ] - - # https://en.wikipedia.org/wiki/Localhost - # https://en.wikipedia.org/wiki/Private_network - @reserved ~w[ - 127.0.0.0/8 - ::1/128 - fc00::/7 - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 - ] - def init(_), do: nil def call(%{remote_ip: original_remote_ip} = conn, _) do - config = Pleroma.Config.get(__MODULE__, []) - - if Keyword.get(config, :enabled, false) do - %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config)) + if Config.get([__MODULE__, :enabled]) do + %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts()) assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip) else conn end end - defp remote_ip_opts(config) do - headers = config |> Keyword.get(:headers, @headers) |> MapSet.new() - reserved = Keyword.get(config, :reserved, @reserved) + defp remote_ip_opts do + headers = Config.get([__MODULE__, :headers], []) |> MapSet.new() + reserved = Config.get([__MODULE__, :reserved], []) proxies = - config - |> Keyword.get(:proxies, []) + Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&InetCidr.parse/1) + |> Enum.map(&maybe_add_cidr/1) {headers, proxies} end + + defp maybe_add_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse(proxy, true) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index aacd58d03..eb44cffec 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -790,7 +790,17 @@ defp restrict_replies(query, %{ [activity, object] in query, where: fragment( - "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?", + """ + ?->>'type' != 'Create' -- This isn't a Create + OR ?->>'inReplyTo' is null -- this isn't a reply + OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, + -- unless they are the author (because authors + -- are also part of the recipients). This leads + -- to a bug that self-replies by friends won't + -- show up. + OR ? = ? -- The actor is us + """, + activity.data, object.data, ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)], activity.recipients, diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index d7a19df4a..d67b594b5 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -57,7 +57,6 @@ defp get_tld(host) do def fetch_data_for_object(object) do with true <- Config.get([:rich_media, :enabled]), - false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), diff --git a/test/emails/admin_email_test.exs b/test/emails/admin_email_test.exs index e24231e27..155057f3e 100644 --- a/test/emails/admin_email_test.exs +++ b/test/emails/admin_email_test.exs @@ -63,7 +63,7 @@ test "new unapproved registration email" do assert res.html_body == """

New account for review: @#{account.nickname}

Plz let me in
- Visit AdminFE + Visit AdminFE """ end end diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 752ab32e7..6d01c812d 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -3,13 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.RemoteIpTest do - use ExUnit.Case, async: true + use ExUnit.Case use Plug.Test alias Pleroma.Plugs.RemoteIp - import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2] - setup do: clear_config(RemoteIp) + import Pleroma.Tests.Helpers, only: [clear_config: 2] + + setup do: + clear_config(RemoteIp, + enabled: true, + headers: ["x-forwarded-for"], + proxies: [], + reserved: [ + "127.0.0.0/8", + "::1/128", + "fc00::/7", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] + ) test "disabled" do Pleroma.Config.put(RemoteIp, enabled: false) @@ -25,8 +39,6 @@ test "disabled" do end test "enabled" do - Pleroma.Config.put(RemoteIp, enabled: true) - conn = conn(:get, "/") |> put_req_header("x-forwarded-for", "1.1.1.1") @@ -54,8 +66,6 @@ test "custom headers" do end test "custom proxies" do - Pleroma.Config.put(RemoteIp, enabled: true) - conn = conn(:get, "/") |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2") @@ -72,4 +82,27 @@ test "custom proxies" do assert conn.remote_ip == {1, 1, 1, 1} end + + test "proxies set without CIDR format" do + Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"]) + + conn = + conn(:get, "/") + |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1") + |> RemoteIp.call(nil) + + assert conn.remote_ip == {1, 1, 1, 1} + end + + test "proxies set `nonsensical` CIDR" do + Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"]) + Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"]) + + conn = + conn(:get, "/") + |> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1") + |> RemoteIp.call(nil) + + assert conn.remote_ip == {1, 1, 1, 1} + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 7bdad3810..804305a13 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -2177,4 +2177,84 @@ test "does nothing with a clashing nickname and the same ap id" do assert user.nickname == orig_user.nickname end end + + describe "reply filtering" do + test "`following` still contains announcements by friends" do + user = insert(:user) + followed = insert(:user) + not_followed = insert(:user) + + User.follow(user, followed) + + {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"}) + + {:ok, not_followed_to_followed} = + CommonAPI.post(not_followed, %{ + status: "Also hello", + in_reply_to_status_id: followed_post.id + }) + + {:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed) + + params = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "following") + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + activities = + [user.ap_id | User.following(user)] + |> ActivityPub.fetch_activities(params) + + followed_post_id = followed_post.id + retoot_id = retoot.id + + assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities + + assert length(activities) == 2 + end + + # This test is skipped because, while this is the desired behavior, + # there seems to be no good way to achieve it with the method that + # we currently use for detecting to who a reply is directed. + # This is a TODO and should be fixed by a later rewrite of the code + # in question. + @tag skip: true + test "`following` still contains self-replies by friends" do + user = insert(:user) + followed = insert(:user) + not_followed = insert(:user) + + User.follow(user, followed) + + {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"}) + {:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"}) + + {:ok, _followed_to_not_followed} = + CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id}) + + {:ok, _followed_self_reply} = + CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id}) + + params = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "following") + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + activities = + [user.ap_id | User.following(user)] + |> ActivityPub.fetch_activities(params) + + assert length(activities) == 2 + end + end end diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 8264a9c41..4b97bd66b 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -64,41 +64,6 @@ test "crawls valid, complete URLs" do Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end - test "refuses to crawl URLs from posts marked sensitive" do - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{ - status: "http://example.com/ogp", - sensitive: true - }) - - %Object{} = object = Object.normalize(activity) - - assert object.data["sensitive"] - - Config.put([:rich_media, :enabled], true) - - assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - end - - test "refuses to crawl URLs from posts tagged NSFW" do - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{ - status: "http://example.com/ogp #nsfw" - }) - - %Object{} = object = Object.normalize(activity) - - assert object.data["sensitive"] - - Config.put([:rich_media, :enabled], true) - - assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - end - test "refuses to crawl URLs of private network from posts" do user = insert(:user)