From ee3052a2d8fda37e27f31c8d824ce7ac174b993c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 22 Sep 2020 14:20:19 +0200 Subject: [PATCH 01/26] ActivityPub: Return Announces when filtering by `following`. --- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++- test/web/activity_pub/activity_pub_test.exs | 80 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index aacd58d03..eb44cffec 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -790,7 +790,17 @@ defp restrict_replies(query, %{ [activity, object] in query, where: fragment( - "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?", + """ + ?->>'type' != 'Create' -- This isn't a Create + OR ?->>'inReplyTo' is null -- this isn't a reply + OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, + -- unless they are the author (because authors + -- are also part of the recipients). This leads + -- to a bug that self-replies by friends won't + -- show up. + OR ? = ? -- The actor is us + """, + activity.data, object.data, ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)], activity.recipients, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 7bdad3810..804305a13 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -2177,4 +2177,84 @@ test "does nothing with a clashing nickname and the same ap id" do assert user.nickname == orig_user.nickname end end + + describe "reply filtering" do + test "`following` still contains announcements by friends" do + user = insert(:user) + followed = insert(:user) + not_followed = insert(:user) + + User.follow(user, followed) + + {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"}) + + {:ok, not_followed_to_followed} = + CommonAPI.post(not_followed, %{ + status: "Also hello", + in_reply_to_status_id: followed_post.id + }) + + {:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed) + + params = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "following") + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + activities = + [user.ap_id | User.following(user)] + |> ActivityPub.fetch_activities(params) + + followed_post_id = followed_post.id + retoot_id = retoot.id + + assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities + + assert length(activities) == 2 + end + + # This test is skipped because, while this is the desired behavior, + # there seems to be no good way to achieve it with the method that + # we currently use for detecting to who a reply is directed. + # This is a TODO and should be fixed by a later rewrite of the code + # in question. + @tag skip: true + test "`following` still contains self-replies by friends" do + user = insert(:user) + followed = insert(:user) + not_followed = insert(:user) + + User.follow(user, followed) + + {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"}) + {:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"}) + + {:ok, _followed_to_not_followed} = + CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id}) + + {:ok, _followed_self_reply} = + CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id}) + + params = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "following") + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + activities = + [user.ap_id | User.following(user)] + |> ActivityPub.fetch_activities(params) + + assert length(activities) == 2 + end + end end From 7ac4d1e7ee831cbbda535c00b7dc9aca053183be Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Mon, 28 Sep 2020 18:13:45 +0300 Subject: [PATCH 02/26] Remove extra nesting and add labels in `:frontends` setting --- config/description.exs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/description.exs b/config/description.exs index ac3dfbb2b..be9cb0a16 100644 --- a/config/description.exs +++ b/config/description.exs @@ -44,11 +44,13 @@ }, %{ key: "git", + label: "Git", type: :string, description: "URL of the git repository of the frontend" }, %{ key: "build_url", + label: "Build URL", type: :string, description: "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.", @@ -56,6 +58,7 @@ }, %{ key: "build_dir", + label: "Build Dir", type: :string, description: "The directory inside the zip file " } @@ -3681,9 +3684,7 @@ type: :map, description: "A map containing available frontends and parameters for their installation.", - children: [ - frontend_options - ] + children: frontend_options } ] }, From ba7f9459b4798388eb4e441d096302c018354033 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 28 Sep 2020 18:22:59 -0500 Subject: [PATCH 03/26] Revert Rich Media censorship for sensitive statuses The #NSFW hashtag test was broken anyway. --- lib/pleroma/web/rich_media/helpers.ex | 1 - test/web/rich_media/helpers_test.exs | 35 --------------------------- 2 files changed, 36 deletions(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index d7a19df4a..d67b594b5 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -57,7 +57,6 @@ defp get_tld(host) do def fetch_data_for_object(object) do with true <- Config.get([:rich_media, :enabled]), - false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 8264a9c41..4b97bd66b 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -64,41 +64,6 @@ test "crawls valid, complete URLs" do Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end - test "refuses to crawl URLs from posts marked sensitive" do - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{ - status: "http://example.com/ogp", - sensitive: true - }) - - %Object{} = object = Object.normalize(activity) - - assert object.data["sensitive"] - - Config.put([:rich_media, :enabled], true) - - assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - end - - test "refuses to crawl URLs from posts tagged NSFW" do - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{ - status: "http://example.com/ogp #nsfw" - }) - - %Object{} = object = Object.normalize(activity) - - assert object.data["sensitive"] - - Config.put([:rich_media, :enabled], true) - - assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - end - test "refuses to crawl URLs of private network from posts" do user = insert(:user) From 0e57f133650ffe3cdd191a49a2c9f0ae6d2e1e54 Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Tue, 29 Sep 2020 20:32:55 +0000 Subject: [PATCH 04/26] Apply 1 suggestion(s) to 1 file(s) --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index be9cb0a16..8656e4575 100644 --- a/config/description.exs +++ b/config/description.exs @@ -44,7 +44,7 @@ }, %{ key: "git", - label: "Git", + label: "Git Repository URL" type: :string, description: "URL of the git repository of the frontend" }, From 1b2f7139c87557e63743fb592a7015adaf277f93 Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Tue, 29 Sep 2020 20:33:04 +0000 Subject: [PATCH 05/26] Apply 1 suggestion(s) to 1 file(s) --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 8656e4575..14478a119 100644 --- a/config/description.exs +++ b/config/description.exs @@ -58,7 +58,7 @@ }, %{ key: "build_dir", - label: "Build Dir", + label: "Build directory" type: :string, description: "The directory inside the zip file " } From fe1975064446121a6d1f6c709b1a31c30aaa121a Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Wed, 30 Sep 2020 00:49:08 +0300 Subject: [PATCH 06/26] Fix comma typo --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 14478a119..dd9e46c08 100644 --- a/config/description.exs +++ b/config/description.exs @@ -44,7 +44,7 @@ }, %{ key: "git", - label: "Git Repository URL" + label: "Git Repository URL", type: :string, description: "URL of the git repository of the frontend" }, From d96b8923f631d26469de9264c3e493a69cf605dd Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Wed, 30 Sep 2020 00:50:53 +0300 Subject: [PATCH 07/26] Fix typo --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index dd9e46c08..6fa78a5d1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -58,7 +58,7 @@ }, %{ key: "build_dir", - label: "Build directory" + label: "Build directory", type: :string, description: "The directory inside the zip file " } From 35ee759e74d0737598311d8e4245168f981812d3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 5 Oct 2020 11:48:41 -0500 Subject: [PATCH 08/26] Add helper function to convert single IPs into CIDR format if they were not provided that way --- lib/pleroma/plugs/remote_ip.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 0ac9050d0..d1b1f793a 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -47,8 +47,19 @@ defp remote_ip_opts(config) do config |> Keyword.get(:proxies, []) |> Enum.concat(reserved) - |> Enum.map(&InetCidr.parse/1) + |> Enum.map(&maybe_add_cidr/1) {headers, proxies} end + + defp maybe_add_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse(proxy) + end end From 21a73ce9f2c577c046e1cd17ef1a2b6636d01585 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 14:54:21 -0500 Subject: [PATCH 09/26] Document the NSFW link preview change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae5d0eda..8fc1750d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Pleroma Admin API: emoji packs and files routes changed. +- **Breaking:** Sensitive/NSFW statuses no longer disable link previews. - Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. From 7aff2b47c56c5b41620445b7d49c429eb1866164 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 15:47:11 -0500 Subject: [PATCH 10/26] Fix docs for default headers used by RemoteIp. We only use X-Forwarded-For by default. --- config/description.exs | 5 +++-- docs/configuration/cheatsheet.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/description.exs b/config/description.exs index ac3dfbb2b..f6331dd30 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3262,8 +3262,9 @@ %{ key: :headers, type: {:list, :string}, - description: - "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`." + description: """ + A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Default: `["x-forwarded-for"]`. + """ }, %{ key: :proxies, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 42e5fe808..e0194525d 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -426,7 +426,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start Available options: * `enabled` - Enable/disable the plug. Defaults to `false`. -* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`. +* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Defaults to `["x-forwarded-for"]`. * `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`. * `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network). From d43d05005ae4e8b0f069111baee867492d4f0c52 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:02:46 -0500 Subject: [PATCH 11/26] Move hardcoded default configuration into config.exs --- config/config.exs | 13 ++++++++++++- lib/pleroma/plugs/remote_ip.ex | 31 +++++++------------------------ test/plugs/remote_ip_test.exs | 24 +++++++++++++++++------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/config/config.exs b/config/config.exs index 2e6b0796a..d53663d36 100644 --- a/config/config.exs +++ b/config/config.exs @@ -677,7 +677,18 @@ config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 -config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true +config :pleroma, Pleroma.Plugs.RemoteIp, + enabled: true, + headers: ["x-forwarded-for"], + proxies: [], + reserved: [ + "127.0.0.0/8", + "::1/128", + "fc00::/7", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] config :pleroma, :static_fe, enabled: false diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index d1b1f793a..9487efa5f 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -7,45 +7,28 @@ defmodule Pleroma.Plugs.RemoteIp do This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. """ + alias Pleroma.Config import Plug.Conn @behaviour Plug - @headers ~w[ - x-forwarded-for - ] - - # https://en.wikipedia.org/wiki/Localhost - # https://en.wikipedia.org/wiki/Private_network - @reserved ~w[ - 127.0.0.0/8 - ::1/128 - fc00::/7 - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 - ] - def init(_), do: nil def call(%{remote_ip: original_remote_ip} = conn, _) do - config = Pleroma.Config.get(__MODULE__, []) - - if Keyword.get(config, :enabled, false) do - %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config)) + if Config.get([__MODULE__, :enabled]) do + %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts()) assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip) else conn end end - defp remote_ip_opts(config) do - headers = config |> Keyword.get(:headers, @headers) |> MapSet.new() - reserved = Keyword.get(config, :reserved, @reserved) + defp remote_ip_opts() do + headers = Config.get([__MODULE__, :headers], []) |> MapSet.new() + reserved = Config.get([__MODULE__, :reserved], []) proxies = - config - |> Keyword.get(:proxies, []) + Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) |> Enum.map(&maybe_add_cidr/1) diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 752ab32e7..2dd1ac1f8 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -3,13 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.RemoteIpTest do - use ExUnit.Case, async: true + use ExUnit.Case use Plug.Test alias Pleroma.Plugs.RemoteIp - import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2] - setup do: clear_config(RemoteIp) + import Pleroma.Tests.Helpers, only: [clear_config: 2] + + setup do: + clear_config(RemoteIp, + enabled: true, + headers: ["x-forwarded-for"], + proxies: [], + reserved: [ + "127.0.0.0/8", + "::1/128", + "fc00::/7", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16" + ] + ) test "disabled" do Pleroma.Config.put(RemoteIp, enabled: false) @@ -25,8 +39,6 @@ test "disabled" do end test "enabled" do - Pleroma.Config.put(RemoteIp, enabled: true) - conn = conn(:get, "/") |> put_req_header("x-forwarded-for", "1.1.1.1") @@ -54,8 +66,6 @@ test "custom headers" do end test "custom proxies" do - Pleroma.Config.put(RemoteIp, enabled: true) - conn = conn(:get, "/") |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2") From 9783e9cd8023533d05bf78e3db3375102a199fc0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:08:26 -0500 Subject: [PATCH 12/26] Add test for an entry without CIDR format --- test/plugs/remote_ip_test.exs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 2dd1ac1f8..849c7fc3d 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -82,4 +82,15 @@ test "custom proxies" do assert conn.remote_ip == {1, 1, 1, 1} end + + test "proxies set without CIDR format" do + Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"]) + + conn = + conn(:get, "/") + |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1") + |> RemoteIp.call(nil) + + assert conn.remote_ip == {1, 1, 1, 1} + end end From b8c05f4876b8f48bcd93d7e5d60539101329065a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:21:27 -0500 Subject: [PATCH 13/26] Improve descriptions for reserved and proxies --- config/description.exs | 7 ++++--- docs/configuration/cheatsheet.md | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/description.exs b/config/description.exs index f6331dd30..2c7d91ccc 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3270,13 +3270,14 @@ key: :proxies, type: {:list, :string}, description: - "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`." + "A list of upstream proxy IP subnets in CIDR notation. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128." }, %{ key: :reserved, type: {:list, :string}, - description: - "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)." + description: """ + A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]` + """ } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index e0194525d..22333c4f8 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -427,8 +427,8 @@ Available options: * `enabled` - Enable/disable the plug. Defaults to `false`. * `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Defaults to `["x-forwarded-for"]`. -* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`. -* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network). +* `proxies` - A list of upstream proxy IP subnets in CIDR notation. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128. +* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`. ### :rate_limit From 7a2ed2fc90dd16a5ef45c4dd44a6e09bba035299 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:26:31 -0500 Subject: [PATCH 14/26] Credo --- lib/pleroma/plugs/remote_ip.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 9487efa5f..51cc87ad8 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -23,7 +23,7 @@ def call(%{remote_ip: original_remote_ip} = conn, _) do end end - defp remote_ip_opts() do + defp remote_ip_opts do headers = Config.get([__MODULE__, :headers], []) |> MapSet.new() reserved = Config.get([__MODULE__, :reserved], []) From e08eb4aba07ce843f3f1149b8c70fb6b4d855c44 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:29:56 -0500 Subject: [PATCH 15/26] Don't leak internal variables in the docs. They're useless to users. --- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 2c7d91ccc..71cb5d913 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3263,7 +3263,7 @@ key: :headers, type: {:list, :string}, description: """ - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Default: `["x-forwarded-for"]`. + A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`. """ }, %{ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 22333c4f8..7f1dc0fe6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -426,7 +426,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start Available options: * `enabled` - Enable/disable the plug. Defaults to `false`. -* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Defaults to `["x-forwarded-for"]`. +* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`. * `proxies` - A list of upstream proxy IP subnets in CIDR notation. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128. * `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`. From b90eda3d8bfa1faf5bdabce9539b601476abed94 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Oct 2020 17:36:29 -0500 Subject: [PATCH 16/26] Improve description yet again --- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 71cb5d913..c6916ad14 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3270,7 +3270,7 @@ key: :proxies, type: {:list, :string}, description: - "A list of upstream proxy IP subnets in CIDR notation. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128." + "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128." }, %{ key: :reserved, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 7f1dc0fe6..ea7dfec98 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -427,7 +427,7 @@ Available options: * `enabled` - Enable/disable the plug. Defaults to `false`. * `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`. -* `proxies` - A list of upstream proxy IP subnets in CIDR notation. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128. +* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128. * `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`. From d0eca5b12518b0b98ef53003d60b08a78decf35f Mon Sep 17 00:00:00 2001 From: feld Date: Wed, 7 Oct 2020 19:16:53 +0000 Subject: [PATCH 17/26] Apply 2 suggestion(s) to 2 file(s) --- lib/pleroma/plugs/remote_ip.ex | 2 +- test/plugs/remote_ip_test.exs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 51cc87ad8..987022156 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -43,6 +43,6 @@ defp maybe_add_cidr(proxy) when is_binary(proxy) do InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" end - InetCidr.parse(proxy) + InetCidr.parse(proxy, true) end end diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 849c7fc3d..2da9f616b 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -92,5 +92,18 @@ test "proxies set without CIDR format" do |> RemoteIp.call(nil) assert conn.remote_ip == {1, 1, 1, 1} + + test "proxies set `nonsensical` CIDR" do + Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"]) + Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"]) + + conn = + conn(:get, "/") + |> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1") + |> RemoteIp.call(nil) + + assert conn.remote_ip == {1, 1, 1, 1} + end + end end From 8bfc5d9a0cf96739a6a73eae3c1d96277da8ae1b Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 7 Oct 2020 19:32:09 +0000 Subject: [PATCH 18/26] Apply 1 suggestion(s) to 1 file(s) --- test/plugs/remote_ip_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 2da9f616b..5f1b8a539 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -92,6 +92,7 @@ test "proxies set without CIDR format" do |> RemoteIp.call(nil) assert conn.remote_ip == {1, 1, 1, 1} + end test "proxies set `nonsensical` CIDR" do Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"]) From 6ee20eb3285a99fab880150a9dfeebadc46fde76 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 7 Oct 2020 19:32:42 +0000 Subject: [PATCH 19/26] Apply 1 suggestion(s) to 1 file(s) --- test/plugs/remote_ip_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index 5f1b8a539..b45baf75f 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -104,7 +104,6 @@ test "proxies set `nonsensical` CIDR" do |> RemoteIp.call(nil) assert conn.remote_ip == {1, 1, 1, 1} - end end end From a702f9fb5bff78c99014838eb8f678c30913bd59 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Oct 2020 15:07:03 -0500 Subject: [PATCH 20/26] Lint --- test/plugs/remote_ip_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs index b45baf75f..6d01c812d 100644 --- a/test/plugs/remote_ip_test.exs +++ b/test/plugs/remote_ip_test.exs @@ -104,6 +104,5 @@ test "proxies set `nonsensical` CIDR" do |> RemoteIp.call(nil) assert conn.remote_ip == {1, 1, 1, 1} - end end From a02ed3920f43b4dd67737e5989f65f3298546ea0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 15:51:56 -0500 Subject: [PATCH 21/26] Improve backup/restore documentation --- docs/administration/backup.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index be57bf74a..10a86c563 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -5,7 +5,7 @@ 1. Stop the Pleroma service. 2. Go to the working directory of Pleroma (default is `/opt/pleroma`) 3. Run `sudo -Hu postgres pg_dump -d --format=custom -f ` (make sure the postgres user has write access to the destination file) -4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. +4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. 5. Restart the Pleroma service. ## Restore/Move @@ -14,11 +14,15 @@ 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. -5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'CREATE DATABASE ;';` -6. Run `sudo -Hu postgres pg_restore -d -v -1 ` -7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. -8. Restart the Pleroma service. -9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries. +5. Drop the existing database. `sudo -Hu postgres psql -c 'DROP DATABASE ;` +6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. + + Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. + +7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d -v -1 ` +8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. +9. Restart the Pleroma service. +10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries. [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. From 5f38e03c8a487da3e9f7f480a684b0dad6a3c086 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 15:54:28 -0500 Subject: [PATCH 22/26] Don't always need to drop --- docs/administration/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 10a86c563..1d9b82fcc 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -14,7 +14,7 @@ 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. -5. Drop the existing database. `sudo -Hu postgres psql -c 'DROP DATABASE ;` +5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;` 6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. From e0c5adcc206818e6d135fb5f827017edd0d4044e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 15:56:37 -0500 Subject: [PATCH 23/26] Syntax error --- docs/administration/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 1d9b82fcc..8e5db8c57 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -14,7 +14,7 @@ 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. -5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;` +5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;'` 6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. From 1e5fec8d45ac9d16b80986db99abc32ddb7bb87b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Oct 2020 16:00:42 -0500 Subject: [PATCH 24/26] Ask user to use matching values for database setup --- docs/administration/backup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 8e5db8c57..b49ff07fb 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -10,14 +10,14 @@ ## Restore/Move -1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name. +1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). 2. Stop the Pleroma service. 3. Go to the working directory of Pleroma (default is `/opt/pleroma`) 4. Copy the above mentioned files back to their original position. 5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;'` 6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. - Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. + Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. 7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d -v -1 ` 8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. From ec217ca638859d6c4b8071102e7e0546db762990 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 9 Oct 2020 11:18:48 -0500 Subject: [PATCH 25/26] Spelling --- docs/administration/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index b49ff07fb..cfedf5b58 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -17,7 +17,7 @@ 5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE ;'` 6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. - Alernatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. + Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed. 7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d -v -1 ` 8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. From 3ca98878d27478037233a92f72adb3fbade62035 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 9 Oct 2020 17:08:05 -0500 Subject: [PATCH 26/26] Deep link to the user account in AdminFE in account confirmation emails --- lib/pleroma/emails/admin_email.ex | 2 +- test/emails/admin_email_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index c27ad1065..8979db2f8 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do html_body = """

New account for review: @#{account.nickname}

#{HTML.strip_tags(account.registration_reason)}
- Visit AdminFE + Visit AdminFE """ new() diff --git a/test/emails/admin_email_test.exs b/test/emails/admin_email_test.exs index e24231e27..155057f3e 100644 --- a/test/emails/admin_email_test.exs +++ b/test/emails/admin_email_test.exs @@ -63,7 +63,7 @@ test "new unapproved registration email" do assert res.html_body == """

New account for review: @#{account.nickname}

Plz let me in
- Visit AdminFE + Visit AdminFE """ end end