From dcd30e3cebc61c5b84cee123f450536003869dd0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 9 Aug 2019 16:49:09 +0300 Subject: [PATCH 01/48] Mastodon API: Set follower/following counters to 0 when hiding followers/following is enabled We are already doing that in AP representation, so I think we should do it here as well for consistency. --- CHANGELOG.md | 4 +++ .../web/mastodon_api/views/account_view.ex | 11 ++++++-- test/web/mastodon_api/account_view_test.exs | 27 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1008e6171..139af1fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.0.5] - 2019-08-?? +### Fixed +- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set + ## [1.0.4] - 2019-08-01 ### Fixed - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 62c516f8e..edd37de2f 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -71,6 +71,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do image = User.avatar_url(user) |> MediaProxy.url() header = User.banner_url(user) |> MediaProxy.url() user_info = User.get_cached_user_info(user) + + following_count = + ((!user.info.hide_follows or opts[:for] == user) && user_info.following_count) || 0 + + followers_count = + ((!user.info.hide_followers or opts[:for] == user) && user_info.follower_count) || 0 + bot = (user.info.source_data["type"] || "Person") in ["Application", "Service"] emojis = @@ -101,8 +108,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do display_name: display_name, locked: user_info.locked, created_at: Utils.to_masto_date(user.inserted_at), - followers_count: user_info.follower_count, - following_count: user_info.following_count, + followers_count: followers_count, + following_count: following_count, statuses_count: user_info.note_count, note: bio || "", url: user.ap_id, diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index de6aeec72..f8a354272 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -275,4 +275,31 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do result = AccountView.render("account.json", %{user: user}) refute result.display_name == " username " end + + describe "hiding follows/following" do + test "shows when follows/following are hidden and sets follower/following count to 0" do + user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + other_user = insert(:user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + assert %{ + followers_count: 0, + following_count: 0, + pleroma: %{hide_follows: true, hide_followers: true} + } = AccountView.render("account.json", %{user: user}) + end + + test "shows actual follower/following count to the account owner" do + user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + other_user = insert(:user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + assert %{ + followers_count: 1, + following_count: 1 + } = AccountView.render("account.json", %{user: user, for: user}) + end + end end From ba21d515d6ace80109b5f38934b33def8243e797 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 10 Aug 2019 16:27:46 +0300 Subject: [PATCH 02/48] Mastodon API: Fix thread mute detection It was calling CommonAPI.thread_muted? with post author's account instead of viewer's one. --- CHANGELOG.md | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 139af1fd6..8de2f2124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.5] - 2019-08-?? ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set +- Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c183c9f4a..286fd08c7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -160,7 +160,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do thread_muted? = case activity.thread_muted? do thread_muted? when is_boolean(thread_muted?) -> thread_muted? - nil -> CommonAPI.thread_muted?(user, activity) + nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false end attachment_data = object.data["attachment"] || [] diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 152ba8137..7efdc7adb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2684,8 +2684,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "conversation muting" do setup do + post_user = insert(:user) user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{"status" => "HIE"}) + + {:ok, activity} = CommonAPI.post(post_user, %{"status" => "HIE"}) [user: user, activity: activity] end From 6525fbac95f83a6050fcea44ad819387c42aa368 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 20:29:30 +0000 Subject: [PATCH 03/48] Return profile URL when available instead of actor URI for MastodonAPI mention URL Fixes #1165 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index edd37de2f..1cd8b62c8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: user.ap_id + url: User.profile_url(user) || user.ap_id } end From b29dcc8c1b74a84e49d290cbea9b1c2f93f1cc5f Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 20:55:37 +0000 Subject: [PATCH 04/48] Simplify logic to mention.js `url` field `User.profile_url` already fallbacks to ap_id --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 1cd8b62c8..b6cbb454b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: User.profile_url(user) || user.ap_id + url: User.profile_url(user) } end From a3654d947967749211ff7d9d68c1365cfa439c52 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 7 Aug 2019 21:40:53 +0000 Subject: [PATCH 05/48] Return profile URL in MastodonAPI's `url` field --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index b6cbb454b..69895fd71 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do following_count: following_count, statuses_count: user_info.note_count, note: bio || "", - url: user.ap_id, + url: User.profile_url(user), avatar: image, avatar_static: image, header: header, From 951d8c28d105e8bff6d46af6088a250acf613ab0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:30:55 +0000 Subject: [PATCH 06/48] update changelog for thibg's change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de2f2124..35582f5fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted +- Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate ## [1.0.4] - 2019-08-01 ### Fixed From 65bd927cf2c98a1e942f817b730f696d5f200cf8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 3 Aug 2019 22:56:20 +0200 Subject: [PATCH 07/48] templates/layout/app.html.eex: Style anchors [ci skip] --- lib/pleroma/web/templates/layout/app.html.eex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index b3cf9ed11..5836ec1e0 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -36,6 +36,11 @@ margin-bottom: 20px; } + a { + color: color: #d8a070; + text-decoration: none; + } + form { width: 100%; } From d2c3c64da5b4d85f63e96e3b0fc752dc50259ddd Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:31:49 +0000 Subject: [PATCH 08/48] update changelog for lanodan's change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35582f5fc..624a348cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate +- Templates: properly style anchor tags ## [1.0.4] - 2019-08-01 ### Fixed From f270e332082210c59e5a9024d929a8877bc8e9d1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 Jul 2019 06:19:27 +0200 Subject: [PATCH 09/48] tasks/pleroma/instance.ex: Change :upload_dir to :uploads_dir Closes: https://git.pleroma.social/pleroma/pleroma/issues/1058 --- lib/mix/tasks/pleroma/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index a27c4b897..2ae16adc0 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -149,7 +149,7 @@ defmodule Mix.Tasks.Pleroma.Instance do uploads_dir = get_option( options, - :upload_dir, + :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) ) From 54d4ceec5ce82ecea8bfd2fade7e618feca8a211 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 2 Aug 2019 23:30:47 +0200 Subject: [PATCH 10/48] tasks/pleroma/user.ex: Fix documentation of --max-use and --expire-at Closes: https://git.pleroma.social/pleroma/pleroma/issues/1155 [ci skip] --- lib/mix/tasks/pleroma/user.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 8a78b4fe6..6c7e34a1c 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -31,8 +31,8 @@ defmodule Mix.Tasks.Pleroma.User do mix pleroma.user invite [OPTION...] Options: - - `--expires_at DATE` - last day on which token is active (e.g. "2019-04-05") - - `--max_use NUMBER` - maximum numbers of token uses + - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05") + - `--max-use NUMBER` - maximum numbers of token uses ## List generated invites From 1310cdc24f31b8fb6858b0c70514a4e316638dbe Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 3 Aug 2019 18:12:38 +0000 Subject: [PATCH 11/48] Handle MRF rejections of incoming AP activities --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +++ test/web/federator_test.exs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fd5521c98..fcecbe21d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -274,6 +274,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else {:fake, true, activity} -> {:ok, activity} + + {:error, message} -> + {:error, message} end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 0f43bc8f2..e42d376cb 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -213,5 +213,21 @@ defmodule Pleroma.Web.FederatorTest do :error = Federator.incoming_ap_doc(params) end + + test "it does not crash if MRF rejects the post" do + policies = Pleroma.Config.get([:instance, :rewrite_policy]) + mrf_keyword_policy = Pleroma.Config.get(:mrf_keyword) + Pleroma.Config.put([:mrf_keyword, :reject], ["lain"]) + Pleroma.Config.put([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.KeywordPolicy) + + params = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + assert Federator.incoming_ap_doc(params) == :error + + Pleroma.Config.put([:instance, :rewrite_policy], policies) + Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) + end end end From 8a8fe57670db773d9f70a0248653661f93771457 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 3 Aug 2019 23:17:17 +0000 Subject: [PATCH 12/48] tasks: relay: add list task --- CHANGELOG.md | 3 +++ lib/mix/tasks/pleroma/relay.ex | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 624a348cd..655c01972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags +### Added +- Relays: Added a task to list relay subscriptions. + ## [1.0.4] - 2019-08-01 ### Fixed - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index 83ed0ed02..c7324fff6 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Relay do use Mix.Task import Mix.Pleroma + alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @shortdoc "Manages remote relays" @@ -22,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Relay do ``mix pleroma.relay unfollow `` Example: ``mix pleroma.relay unfollow https://example.org/relay`` + + ## List relay subscriptions + + ``mix pleroma.relay list`` """ def run(["follow", target]) do start_pleroma() @@ -44,4 +49,19 @@ defmodule Mix.Tasks.Pleroma.Relay do {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}") end end + + def run(["list"]) do + start_pleroma() + + with %User{} = user <- Relay.get_actor() do + user.following + |> Enum.each(fn entry -> + URI.parse(entry) + |> Map.get(:host) + |> shell_info() + end) + else + e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") + end + end end From e08b97853f20c6d0571cf451d18defd0bdb2393f Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Wed, 10 Jul 2019 01:42:41 +0545 Subject: [PATCH 13/48] add listener port and ip option for 'pleroma.instance gen' and enable its test --- lib/mix/tasks/pleroma/instance.ex | 26 +++++++++++++++++-- priv/templates/sample_config.eex | 1 + .../tasks/{instance.exs => instance_test.exs} | 15 +++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) rename test/tasks/{instance.exs => instance_test.exs} (83%) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 2ae16adc0..9080adb52 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -34,6 +34,8 @@ defmodule Mix.Tasks.Pleroma.Instance do - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part - `--uploads-dir` - the directory uploads go in when using a local uploader - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.) + - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1 + - `--listen-port` - the port the app should listen to, defaults to 4000 """ def run(["gen" | rest]) do @@ -56,7 +58,9 @@ defmodule Mix.Tasks.Pleroma.Instance do indexable: :string, db_configurable: :string, uploads_dir: :string, - static_dir: :string + static_dir: :string, + listen_ip: :string, + listen_port: :string ], aliases: [ o: :output, @@ -146,6 +150,22 @@ defmodule Mix.Tasks.Pleroma.Instance do "n" ) === "y" + listen_port = + get_option( + options, + :listen_port, + "What port will the app listen to (leave it if you are using the default setup with nginx)?", + 4000 + ) + + listen_ip = + get_option( + options, + :listen_ip, + "What ip will the app listen to (leave it if you are using the default setup with nginx)?", + "127.0.0.1" + ) + uploads_dir = get_option( options, @@ -186,7 +206,9 @@ defmodule Mix.Tasks.Pleroma.Instance do db_configurable?: db_configurable?, static_dir: static_dir, uploads_dir: uploads_dir, - rum_enabled: rum_enabled + rum_enabled: rum_enabled, + listen_ip: listen_ip, + listen_port: listen_port ) result_psql = diff --git a/priv/templates/sample_config.eex b/priv/templates/sample_config.eex index 2d4a49328..054aa6c22 100644 --- a/priv/templates/sample_config.eex +++ b/priv/templates/sample_config.eex @@ -11,6 +11,7 @@ end %> config :pleroma, Pleroma.Web.Endpoint, url: [host: "<%= domain %>", scheme: "https", port: <%= port %>], + http: [ip: {<%= String.replace(listen_ip, ".", ", ") %>}, port: <%= listen_port %>], secret_key_base: "<%= secret %>", signing_salt: "<%= signing_salt %>" diff --git a/test/tasks/instance.exs b/test/tasks/instance_test.exs similarity index 83% rename from test/tasks/instance.exs rename to test/tasks/instance_test.exs index 1875f52a3..229ecc9c1 100644 --- a/test/tasks/instance.exs +++ b/test/tasks/instance_test.exs @@ -38,7 +38,17 @@ defmodule Pleroma.InstanceTest do "--indexable", "y", "--db-configurable", - "y" + "y", + "--rum", + "y", + "--listen-port", + "4000", + "--listen-ip", + "127.0.0.1", + "--uploads-dir", + "test/uploads", + "--static-dir", + "instance/static/" ]) end @@ -56,10 +66,11 @@ defmodule Pleroma.InstanceTest do assert generated_config =~ "username: \"dbuser\"" assert generated_config =~ "password: \"dbpass\"" assert generated_config =~ "dynamic_configuration: true" + assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() end defp generated_setup_psql do - ~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\n) + ~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\nCREATE EXTENSION IF NOT EXISTS rum;\n) end end From b9def3758ae97490f8c46a77a03c344568106376 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 22 Jul 2019 14:33:58 +0000 Subject: [PATCH 14/48] Feature/1087 wildcard option for blocks --- lib/pleroma/user.ex | 12 +-- lib/pleroma/web/activity_pub/mrf.ex | 10 ++ .../web/activity_pub/mrf/simple_policy.ex | 55 ++++++++--- lib/pleroma/web/activity_pub/publisher.ex | 9 +- test/user_test.exs | 42 +++++++++ test/web/activity_pub/mrf/mrf_test.exs | 46 +++++++++ .../activity_pub/mrf/simple_policy_test.exs | 93 +++++++++++++++++++ 7 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 test/web/activity_pub/mrf/mrf_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f7191762f..ed5403219 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -836,15 +836,15 @@ defmodule Pleroma.User do def mutes?(nil, _), do: false def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id) - def blocks?(user, %{ap_id: ap_id}) do - blocks = user.info.blocks - domain_blocks = user.info.domain_blocks + def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do + blocks = info.blocks + + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(info.domain_blocks) + %{host: host} = URI.parse(ap_id) Enum.member?(blocks, ap_id) || - Enum.any?(domain_blocks, fn domain -> - host == domain - end) + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, host) end def subscribed_to?(user, %{ap_id: ap_id}) do diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 10ceef715..dd204b21c 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -25,4 +25,14 @@ defmodule Pleroma.Web.ActivityPub.MRF do defp get_policies(policy) when is_atom(policy), do: [policy] defp get_policies(policies) when is_list(policies), do: policies defp get_policies(_), do: [] + + @spec subdomains_regex([String.t()]) :: [Regex.t()] + def subdomains_regex(domains) when is_list(domains) do + for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$) + end + + @spec subdomain_match?([Regex.t()], String.t()) :: boolean() + def subdomain_match?(domains, host) do + Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) + end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 433d23c5f..2cf63d3db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -4,22 +4,29 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF @moduledoc "Filter activities depending on their origin instance" - @behaviour Pleroma.Web.ActivityPub.MRF + @behaviour MRF defp check_accept(%{host: actor_host} = _actor_info, object) do - accepts = Pleroma.Config.get([:mrf_simple, :accept]) + accepts = + Pleroma.Config.get([:mrf_simple, :accept]) + |> MRF.subdomains_regex() cond do accepts == [] -> {:ok, object} actor_host == Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} - Enum.member?(accepts, actor_host) -> {:ok, object} + MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} true -> {:reject, nil} end end defp check_reject(%{host: actor_host} = _actor_info, object) do - if Enum.member?(Pleroma.Config.get([:mrf_simple, :reject]), actor_host) do + rejects = + Pleroma.Config.get([:mrf_simple, :reject]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(rejects, actor_host) do {:reject, nil} else {:ok, object} @@ -31,8 +38,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do %{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object ) when length(child_attachment) > 0 do + media_removal = + Pleroma.Config.get([:mrf_simple, :media_removal]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_removal]), actor_host) do + if MRF.subdomain_match?(media_removal, actor_host) do child_object = Map.delete(object["object"], "attachment") Map.put(object, "object", child_object) else @@ -51,8 +62,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do "object" => child_object } = object ) do + media_nsfw = + Pleroma.Config.get([:mrf_simple, :media_nsfw]) + |> MRF.subdomains_regex() + object = - if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do + if MRF.subdomain_match?(media_nsfw, actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] child_object = Map.put(child_object, "tag", tags) child_object = Map.put(child_object, "sensitive", true) @@ -67,12 +82,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_media_nsfw(_actor_info, object), do: {:ok, object} defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do + timeline_removal = + Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]) + |> MRF.subdomains_regex() + object = - with true <- - Enum.member?( - Pleroma.Config.get([:mrf_simple, :federated_timeline_removal]), - actor_host - ), + with true <- MRF.subdomain_match?(timeline_removal, actor_host), user <- User.get_cached_by_ap_id(object["actor"]), true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do to = @@ -94,7 +109,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + report_removal = + Pleroma.Config.get([:mrf_simple, :report_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(report_removal, actor_host) do {:reject, nil} else {:ok, object} @@ -104,7 +123,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_report_removal(_actor_info, object), do: {:ok, object} defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + avatar_removal = + Pleroma.Config.get([:mrf_simple, :avatar_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(avatar_removal, actor_host) do {:ok, Map.delete(object, "icon")} else {:ok, object} @@ -114,7 +137,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_avatar_removal(_actor_info, object), do: {:ok, object} defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do - if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + banner_removal = + Pleroma.Config.get([:mrf_simple, :banner_removal]) + |> MRF.subdomains_regex() + + if MRF.subdomain_match?(banner_removal, actor_host) do {:ok, Map.delete(object, "image")} else {:ok, object} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index a05e03263..3d181f57c 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -87,8 +87,13 @@ defmodule Pleroma.Web.ActivityPub.Publisher do if public do true else - inbox_info = URI.parse(inbox) - !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host) + %{host: host} = URI.parse(inbox) + + quarantined_instances = + Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + + !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) end end diff --git a/test/user_test.exs b/test/user_test.exs index 198a97fae..7bd745104 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -799,6 +799,48 @@ defmodule Pleroma.UserTest do assert User.blocks?(user, collateral_user) end + test "does not block domain with same end" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "does not block domain with same end if wildcard added" do + user = insert(:user) + + collateral_user = + insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + refute User.blocks?(user, collateral_user) + end + + test "blocks domain with wildcard for subdomain" do + user = insert(:user) + + user_from_subdomain = + insert(:user, %{ap_id: "https://subdomain.awful-and-rude-instance.com/user/bully"}) + + user_with_two_subdomains = + insert(:user, %{ + ap_id: "https://subdomain.second_subdomain.awful-and-rude-instance.com/user/bully" + }) + + user_domain = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) + + {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + + assert User.blocks?(user, user_from_subdomain) + assert User.blocks?(user, user_with_two_subdomains) + assert User.blocks?(user, user_domain) + end + test "unblocks domains" do user = insert(:user) collateral_user = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs new file mode 100644 index 000000000..a9cdf5317 --- /dev/null +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -0,0 +1,46 @@ +defmodule Pleroma.Web.ActivityPub.MRFTest do + use ExUnit.Case, async: true + alias Pleroma.Web.ActivityPub.MRF + + test "subdomains_regex/1" do + assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ + ~r/^unsafe.tld$/, + ~r/^(.*\.)*unsafe.tld$/ + ] + end + + describe "subdomain_match/2" do + test "common domains" do + regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) + + assert regexes == [~r/^unsafe.tld$/, ~r/^unsafe2.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "unsafe2.tld") + + refute MRF.subdomain_match?(regexes, "example.com") + end + + test "wildcard domains with one subdomain" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "unsafe.tldanother") + end + + test "wildcard domains with two subdomains" do + regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + + assert regexes == [~r/^(.*\.)*unsafe.tld$/] + + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.anotherunsafe.tld") + refute MRF.subdomain_match?(regexes, "sub.unsafe.tldanother") + end + end +end diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 0fd68e103..8e86d2219 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -49,6 +49,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_removal], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> Map.put("object", Map.delete(media_message["object"], "attachment"))} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end describe "when :media_nsfw" do @@ -74,6 +87,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :media_nsfw], ["*.remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "sensitive"], true)} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_media_message do @@ -106,6 +133,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(report_message) == {:reject, nil} assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :report_removal], ["*.remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end end defp build_report_message do @@ -146,6 +182,27 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + test "match with wildcard domain" do + {actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + Config.put([:mrf_simple, :federated_timeline_removal], ["*." <> ftl_message_actor_host]) + local_message = build_local_message() + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + assert actor.follower_address in ftl_message["to"] + refute actor.follower_address in ftl_message["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + test "has a matching host but only as:Public in to" do {_actor, ftl_message} = build_ftl_actor_and_message() @@ -192,6 +249,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(remote_message) == {:reject, nil} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :reject], ["*.remote.instance"]) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end end describe "when :accept" do @@ -224,6 +289,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :accept], ["*.remote.instance"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end end describe "when :avatar_removal" do @@ -251,6 +326,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do refute filtered["icon"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :avatar_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end end describe "when :banner_removal" do @@ -278,6 +362,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do refute filtered["image"] end + + test "match with wildcard domain" do + Config.put([:mrf_simple, :banner_removal], ["*.remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end end defp build_local_message do From a7b947534cc998486b82c721849bfc9aceb5178f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:38:04 +0000 Subject: [PATCH 15/48] chase changelog updates --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 655c01972..8c3666cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Relays: Added a task to list relay subscriptions. +- MRF (Simple Policy): Support for wildcard domains. +- Support for wildcard domains in user domain blocks setting. +- Configuration: `quarantined_instances` support wildcard domains. ## [1.0.4] - 2019-08-01 ### Fixed From b85840b536f9be391120db5d3e0e7934c8ed9390 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:12:42 +0300 Subject: [PATCH 16/48] Stop depending on the embedded object in restrict_favorited_by --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fcecbe21d..f9ab06bf8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -733,8 +733,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data) + [_activity, object] in query, + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) ) end From 1807d3a115edf12510b2ca735c7aea3d3f57e5f8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:23:58 +0300 Subject: [PATCH 17/48] OStatus Announce Representer: Do not depend on the object being embedded in the Create activity --- lib/pleroma/web/ostatus/activity_representer.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 95037125d..2f23e5730 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -182,6 +182,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + retweeted_object = Object.normalize(retweeted_activity) retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) @@ -196,7 +197,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, {:id, h.(activity.data["id"])}, {:title, ['#{user.nickname} repeated a notice']}, - {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']}, + {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], From e5161961bdf41a0c0a6f051940b31c0b4f258473 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:36:13 +0300 Subject: [PATCH 18/48] ActivityPub tests: remove assertions of embedded object being updated, because the objects are no longer supposed to be embedded --- test/web/activity_pub/activity_pub_test.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 76586ee4a..97c946924 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -679,9 +679,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert like_activity == same_like_activity assert object.data["likes"] == [user.ap_id] - [note_activity] = Activity.get_all_create_by_object_ap_id(object.data["id"]) - assert note_activity.data["object"]["like_count"] == 1 - {:ok, _like_activity, object} = ActivityPub.like(user_two, object) assert object.data["like_count"] == 2 end From f0c32364b7678ce9c234f2df994b9107c922a23d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 00:58:48 +0300 Subject: [PATCH 19/48] OStatus tests: stop relying on embedded objects --- test/web/ostatus/ostatus_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 77bccdaa1..41cf188ca 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -196,7 +196,7 @@ defmodule Pleroma.Web.OStatusTest do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == user.ap_id assert retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 + assert Object.normalize(retweeted_activity).data["announcement_count"] == 1 end test "handle incoming retweets - Mastodon, salmon" do From 7e412fd88a57643bd3826d84d7bd46beec980188 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 7 Aug 2019 01:02:29 +0300 Subject: [PATCH 20/48] Do not rembed the object after updating it --- CHANGELOG.md | 5 +---- lib/pleroma/web/activity_pub/utils.ex | 17 +---------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3666cc5..23319d4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags +- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. ### Added - Relays: Added a task to list relay subscriptions. @@ -32,10 +33,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.2] - 2019-07-28 ### Fixed - Not being able to pin unlisted posts -- Mastodon API: represent poll IDs as strings -- MediaProxy: fix matching filenames -- MediaProxy: fix filename encoding -- Migrations: fix a sporadic migration failure - Metadata rendering errors resulting in the entire page being inaccessible - Federation/MediaProxy not working with instances that have wrong certificate order - ActivityPub S2S: remote user deletions now work the same as local user deletions. diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 514266cee..b9fcae192 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -251,20 +251,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do def insert_full_object(map), do: {:ok, map, nil} - def update_object_in_activities(%{data: %{"id" => id}} = object) do - # TODO - # Update activities that already had this. Could be done in a seperate process. - # Alternatively, just don't do this and fetch the current object each time. Most - # could probably be taken from cache. - relevant_activities = Activity.get_all_create_by_object_ap_id(id) - - Enum.map(relevant_activities, fn activity -> - new_activity_data = activity.data |> Map.put("object", object.data) - changeset = Changeset.change(activity, data: new_activity_data) - Repo.update(changeset) - end) - end - #### Like-related helpers @doc """ @@ -347,8 +333,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put("#{property}_count", length(element)) |> Map.put("#{property}s", element), changeset <- Changeset.change(object, data: new_data), - {:ok, object} <- Object.update_and_set_cache(changeset), - _ <- update_object_in_activities(object) do + {:ok, object} <- Object.update_and_set_cache(changeset) do {:ok, object} end end From 48bd4ee933e549daa3637ac4741aa4439af543ef Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 10 Aug 2019 18:47:40 +0000 Subject: [PATCH 21/48] Strip internal fields including likes from incoming and outgoing activities --- CHANGELOG.md | 2 + lib/mix/tasks/pleroma/database.ex | 36 ++++++++++++++++++ .../web/activity_pub/transmogrifier.ex | 38 +++---------------- test/tasks/database_test.exs | 36 ++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 30 +++++++++++---- 5 files changed, 101 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23319d4e8..9ea0052c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (Simple Policy): Support for wildcard domains. - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. +- Mix Tasks: `mix pleroma.database fix_likes_collections` +- Federation: Remove `likes` from objects. ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index e91fb31d1..990651da4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -35,6 +35,10 @@ defmodule Mix.Tasks.Pleroma.Database do ## Remove duplicated items from following and update followers count for all users mix pleroma.database update_users_following_followers_counts + + ## Fix the pre-existing "likes" collections for all objects + + mix pleroma.database fix_likes_collections """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -119,4 +123,36 @@ defmodule Mix.Tasks.Pleroma.Database do ) end end + + def run(["fix_likes_collections"]) do + import Ecto.Query + + start_pleroma() + + from(object in Object, + where: fragment("(?)->>'likes' is not null", object.data), + select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)} + ) + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn objects -> + ids = + objects + |> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end) + |> Enum.map(& &1.id) + + Object + |> where([object], object.id in ^ids) + |> update([object], + set: [ + data: + fragment( + "jsonb_set(?, '{likes}', '[]'::jsonb, true)", + object.data + ) + ] + ) + |> Repo.update_all([], timeout: :infinity) + end) + |> Stream.run() + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 591227af2..65c1f9d9f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -24,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ def fix_object(object) do object + |> strip_internal_fields |> fix_actor |> fix_url |> fix_attachments @@ -32,7 +33,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_emoji |> fix_tag |> fix_content_map - |> fix_likes |> fix_addressing |> fix_summary |> fix_type @@ -150,21 +150,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end - # Check for standardisation - # This is what Peertube does - # curl -H 'Accept: application/activity+json' $likes | jq .totalItems - # Prismo returns only an integer (count) as "likes" - def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do - object - |> Map.put("likes", []) - |> Map.put("like_count", 0) - end + def fix_in_reply_to(object, options \\ []) - def fix_likes(object) do - object - end - - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, _options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -203,7 +191,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_in_reply_to(object), do: object + def fix_in_reply_to(object, _options), do: object def fix_context(object) do context = object["context"] || object["conversation"] || Utils.generate_context_id() @@ -752,7 +740,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> add_mention_tags |> add_emoji_tags |> add_attributed_to - |> add_likes |> prepare_attachments |> set_conversation |> set_reply_to_uri @@ -936,22 +923,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("attributedTo", attributed_to) end - def add_likes(%{"id" => id, "like_count" => likes} = object) do - likes = %{ - "id" => "#{id}/likes", - "first" => "#{id}/likes?page=1", - "type" => "OrderedCollection", - "totalItems" => likes - } - - object - |> Map.put("likes", likes) - end - - def add_likes(object) do - object - end - def prepare_attachments(object) do attachments = (object["attachment"] || []) @@ -967,6 +938,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp strip_internal_fields(object) do object |> Map.drop([ + "likes", "like_count", "announcements", "announcement_count", diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index 579130b05..a8f25f500 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -3,8 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.CommonAPI + use Pleroma.DataCase import Pleroma.Factory @@ -46,4 +49,37 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do assert user.info.follower_count == 0 end end + + describe "running fix_likes_collections" do + test "it turns OrderedCollection likes into empty arrays" do + [user, user2] = insert_pair(:user) + + {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"}) + {:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"}) + + CommonAPI.favorite(id, user2) + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + new_data = Map.put(object2.data, "likes", likes) + + object2 + |> Ecto.Changeset.change(%{data: new_data}) + |> Repo.update() + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert is_map(Object.get_by_id(object2.id).data["likes"]) + + assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"]) + + assert length(Object.get_by_id(object.id).data["likes"]) == 1 + assert Enum.empty?(Object.get_by_id(object2.id).data["likes"]) + end + end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5a7b1ed80..b8b998acd 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -424,6 +424,27 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert !is_nil(data["cc"]) end + test "it strips internal likes" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + object = Map.put(data["object"], "likes", likes) + data = Map.put(data, "object", object) + + {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data) + + refute Map.has_key?(object.data, "likes") + end + test "it works for incoming update activities" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() @@ -1010,14 +1031,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["context_id"]) - end - - test "it adds like collection to object" do - activity = insert(:note_activity) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - - assert modified["object"]["likes"]["type"] == "OrderedCollection" - assert modified["object"]["likes"]["totalItems"] == 0 + assert is_nil(modified["object"]["likes"]) end test "the directMessage flag is present" do From f7028ae8acc8a1a5af426725778b84c7648f5854 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 20:54:17 +0000 Subject: [PATCH 22/48] tests: mastodon account views: add missing CommonAPI alias --- test/web/mastodon_api/account_view_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index f8a354272..592a941a9 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView test "Represent a user account" do From 25c818ed6fd8fb66143a544e8e745b3d0703c51b Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 29 Jul 2019 16:17:22 +0000 Subject: [PATCH 23/48] Redirect not logged-in users to the MastoFE login page on private instances --- CHANGELOG.md | 5 +++++ lib/pleroma/web/router.ex | 2 +- .../mastodon_api/mastodon_api_controller_test.exs | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea0052c6..eb47be7cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate - Templates: properly style anchor tags - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. +- Not being able to access the Mastodon FE login page on private instances ### Added - Relays: Added a task to list relay subscriptions. @@ -35,6 +36,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [1.0.2] - 2019-07-28 ### Fixed - Not being able to pin unlisted posts +- Mastodon API: represent poll IDs as strings +- MediaProxy: fix matching filenames +- MediaProxy: fix filename encoding +- Migrations: fix a sporadic migration failure - Metadata rendering errors resulting in the entire page being inaccessible - Federation/MediaProxy not working with instances that have wrong certificate order - ActivityPub S2S: remote user deletions now work the same as local user deletions. diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ff9ed1640..3a8dda772 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -684,7 +684,7 @@ defmodule Pleroma.Web.Router do delete("/auth/sign_out", MastodonAPIController, :logout) scope [] do - pipe_through(:oauth_read_or_public) + pipe_through(:oauth_read) get("/web/*path", MastodonAPIController, :index) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 7efdc7adb..de6f044e7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2879,6 +2879,21 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert redirected_to(conn) == "/web/login" end + test "redirects not logged-in users to the login page on private instances", %{ + conn: conn, + path: path + } do + is_public = Pleroma.Config.get([:instance, :public]) + Pleroma.Config.put([:instance, :public], false) + + conn = get(conn, path) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/login" + + Pleroma.Config.put([:instance, :public], is_public) + end + test "does not redirect logged in users to the login page", %{conn: conn, path: path} do token = insert(:oauth_token) From 4d0dd046533efb94af5974e9c7ef027155aafbdc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 10 Aug 2019 21:18:26 +0000 Subject: [PATCH 24/48] MRF: ensure that subdomain_match calls are case-insensitive --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/mrf.ex | 2 +- test/web/activity_pub/mrf/mrf_test.exs | 24 +++++++++++++++++++----- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb47be7cf..0a8f1e363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Templates: properly style anchor tags - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Not being able to access the Mastodon FE login page on private instances +- MRF: ensure that subdomain_match calls are case-insensitive ### Added - Relays: Added a task to list relay subscriptions. diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index dd204b21c..caa2a3231 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do - for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$) + for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$)i end @spec subdomain_match?([Regex.t()], String.t()) :: boolean() diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index a9cdf5317..1a888e18f 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -4,8 +4,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "subdomains_regex/1" do assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ - ~r/^unsafe.tld$/, - ~r/^(.*\.)*unsafe.tld$/ + ~r/^unsafe.tld$/i, + ~r/^(.*\.)*unsafe.tld$/i ] end @@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) - assert regexes == [~r/^unsafe.tld$/, ~r/^unsafe2.tld$/] + assert regexes == [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with one subdomain" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/] + assert regexes == [~r/^(.*\.)*unsafe.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -35,12 +35,26 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "wildcard domains with two subdomains" do regexes = MRF.subdomains_regex(["*.unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/] + assert regexes == [~r/^(.*\.)*unsafe.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") refute MRF.subdomain_match?(regexes, "sub.anotherunsafe.tld") refute MRF.subdomain_match?(regexes, "sub.unsafe.tldanother") end + + test "matches are case-insensitive" do + regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) + + assert regexes == [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i] + + assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") + assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") + assert MRF.subdomain_match?(regexes, "unsafe.tld") + assert MRF.subdomain_match?(regexes, "unsafe2.tld") + + refute MRF.subdomain_match?(regexes, "EXAMPLE.COM") + refute MRF.subdomain_match?(regexes, "example.com") + end end end From 14efbcf1f91b1b758397d3445695bf72b43dcb53 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:05:50 +0000 Subject: [PATCH 25/48] add removed section to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8f1e363..70fce5882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` + +### Removed - Federation: Remove `likes` from objects. ## [1.0.4] - 2019-08-01 From 60c75d6740e5a5194cac4ae690a7b84cfa104efa Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 19 Jul 2019 16:20:23 +0000 Subject: [PATCH 26/48] #1110 fixed /api/pleroma/healthcheck --- .../controllers/util_controller.ex | 32 ++++++---- test/web/twitter_api/util_controller_test.exs | 64 ++++++++++++++++++- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 39bc2efce..b37e51a3d 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -9,7 +9,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Comeonin.Pbkdf2 alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Emoji + alias Pleroma.Healthcheck alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web @@ -22,7 +24,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do - with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do + with %User{} = user <- User.get_cached_by_nickname(nick), + avatar = User.avatar_url(user) do conn |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false}) else @@ -335,20 +338,21 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def healthcheck(conn, _params) do - info = - if Pleroma.Config.get([:instance, :healthcheck]) do - Pleroma.Healthcheck.system_info() - else - %{} - end + with true <- Config.get([:instance, :healthcheck]), + %{healthy: true} = info <- Healthcheck.system_info() do + json(conn, info) + else + %{healthy: false} = info -> + service_unavailable(conn, info) - conn = - if info[:healthy] do - conn - else - Plug.Conn.put_status(conn, :service_unavailable) - end + _ -> + service_unavailable(conn, %{}) + end + end - json(conn, info) + defp service_unavailable(conn, info) do + conn + |> put_status(:service_unavailable) + |> json(info) end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index cab9e5d90..9e4a3094c 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory + import Mock setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -227,10 +228,67 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end - test "GET /api/pleroma/healthcheck", %{conn: conn} do - conn = get(conn, "/api/pleroma/healthcheck") + describe "GET /api/pleroma/healthcheck" do + setup do + config_healthcheck = Pleroma.Config.get([:instance, :healthcheck]) - assert conn.status in [200, 503] + on_exit(fn -> + Pleroma.Config.put([:instance, :healthcheck], config_healthcheck) + end) + + :ok + end + + test "returns 503 when healthcheck disabled", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], false) + + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert response == %{} + end + + test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: true} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(200) + + assert %{ + "active" => _, + "healthy" => true, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end + + test "returns 503 when healthcheck enabled and health is false", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: false} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert %{ + "active" => _, + "healthy" => false, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end end describe "POST /api/pleroma/disable_account" do From 5af3c00072b27e4f23567e0defbc3c021ed11b2b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 11 Aug 2019 22:49:55 +0300 Subject: [PATCH 27/48] Do not fetch the reply object in `fix_type` unless the object has the `name` key and use a depth limit when fetching it --- lib/pleroma/web/activity_pub/transmogrifier.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 65c1f9d9f..af756c21e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -324,17 +324,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do - reply = Object.normalize(reply_id) + def fix_type(object, options \\ []) - if reply && (reply.data["type"] == "Question" and object["name"]) do + def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) + when is_binary(reply_id) do + reply = + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), + {:ok, object} <- get_obj_helper(reply_id) do + object + end + + if reply && reply.data["type"] == "Question" do Map.put(object, "type", "Answer") else object end end - def fix_type(object), do: object + def fix_type(object, _), do: object defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", From 54405919d86ce803bf1a17a55c382c2bc59db3b0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:12:42 +0000 Subject: [PATCH 28/48] changelog updates --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70fce5882..2a1cfccf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Not being able to access the Mastodon FE login page on private instances - MRF: ensure that subdomain_match calls are case-insensitive +- Fix internal server error when using the healthcheck API. +- `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added - Relays: Added a task to list relay subscriptions. From 29db8dc79945026d2024e2462f86b1795859c574 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 02:15:21 +0000 Subject: [PATCH 29/48] config: remove legacy activitypub accept_blocks setting Anyone who is interested in dropping blocks can write their own MRF policy at this point. This setting predated the MRF framework. Disabling the side effect (unsubscription) is still a config option per policy. --- CHANGELOG.md | 1 + config/config.exs | 1 - docs/config.md | 1 - lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++---- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1cfccf2..35dcd3401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - Federation: Remove `likes` from objects. +- ActivityPub: The `accept_blocks` configuration setting. ## [1.0.4] - 2019-08-01 ### Fixed diff --git a/config/config.exs b/config/config.exs index 872ee3b00..b145cab22 100644 --- a/config/config.exs +++ b/config/config.exs @@ -300,7 +300,6 @@ config :pleroma, :assets, default_mascot: :pleroma_fox_tan config :pleroma, :activitypub, - accept_blocks: true, unfollow_blocked: true, outgoing_blocks: true, follow_handshake_timeout: 500 diff --git a/docs/config.md b/docs/config.md index 22799af23..3b9be73ec 100644 --- a/docs/config.md +++ b/docs/config.md @@ -319,7 +319,6 @@ config :pleroma, Pleroma.Web.Endpoint, This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` ## :activitypub -* ``accept_blocks``: Whether to accept incoming block activities from other instances * ``unfollow_blocked``: Whether blocks result in people getting unfollowed * ``outgoing_blocks``: Whether to federate blocks to other instances * ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index af756c21e..ca6940026 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -678,8 +678,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "id" => id } = _data ) do - with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), - %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), + with %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do User.unblock(blocker, blocked) @@ -692,8 +691,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data ) do - with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), - %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), + with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do User.unfollow(blocker, blocked) From e13edc2d3b83bdb063a991edcf56721eef084f6d Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:26:24 +0000 Subject: [PATCH 30/48] MRF: add describe() for gathering and describing the MRF configuration --- lib/pleroma/web/activity_pub/mrf.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index caa2a3231..d43a8760b 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -35,4 +35,20 @@ defmodule Pleroma.Web.ActivityPub.MRF do def subdomain_match?(domains, host) do Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) end + + @callback describe() :: {:ok | :error, Map.t()} + + def describe(policies) do + policies + |> Enum.reduce({:ok, %{}}, fn + policy, {:ok, data} -> + {:ok, policy_data} = policy.describe() + {:ok, Map.merge(data, policy_data)} + + _, error -> + error + end) + end + + def describe(), do: get_policies() |> describe() end From b8186c26fa319eb9fa3d8417a9a3fbe387f40b9a Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:29:15 +0000 Subject: [PATCH 31/48] test: add mock MRF module for describe() testing --- test/support/mrf_module_mock.ex | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test/support/mrf_module_mock.ex diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex new file mode 100644 index 000000000..e5ae21ad8 --- /dev/null +++ b/test/support/mrf_module_mock.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule MRFModuleMock do + @behaviour Pleroma.Web.ActivityPub.MRF + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe(), do: %{"mrf_module_mock" => "some config data"} +end From 7e2bc39f3c51bc7f25238d4a349f2c39c8b9f107 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:52:54 +0000 Subject: [PATCH 32/48] MRF: add describe() to all modules, add base MRF configuration to base describe() --- lib/pleroma/web/activity_pub/mrf.ex | 32 ++++++++++++++----- .../activity_pub/mrf/anti_followbot_policy.ex | 3 ++ .../activity_pub/mrf/anti_link_spam_policy.ex | 6 ++++ .../web/activity_pub/mrf/drop_policy.ex | 3 ++ .../activity_pub/mrf/ensure_re_prepended.ex | 2 ++ .../web/activity_pub/mrf/hellthread_policy.ex | 3 ++ .../web/activity_pub/mrf/keyword_policy.ex | 32 +++++++++++++++++++ .../mrf/no_placeholder_text_policy.ex | 3 ++ .../web/activity_pub/mrf/noop_policy.ex | 3 ++ .../web/activity_pub/mrf/normalize_markup.ex | 2 ++ .../web/activity_pub/mrf/reject_non_public.ex | 3 ++ .../web/activity_pub/mrf/simple_policy.ex | 12 +++++++ .../web/activity_pub/mrf/subchain_policy.ex | 3 ++ .../web/activity_pub/mrf/tag_policy.ex | 3 ++ .../web/activity_pub/mrf/user_allowlist.ex | 9 ++++++ 15 files changed, 111 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index d43a8760b..7533552d5 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -39,15 +39,31 @@ defmodule Pleroma.Web.ActivityPub.MRF do @callback describe() :: {:ok | :error, Map.t()} def describe(policies) do - policies - |> Enum.reduce({:ok, %{}}, fn - policy, {:ok, data} -> - {:ok, policy_data} = policy.describe() - {:ok, Map.merge(data, policy_data)} + {:ok, policy_configs} = + policies + |> Enum.reduce({:ok, %{}}, fn + policy, {:ok, data} -> + {:ok, policy_data} = policy.describe() + {:ok, Map.merge(data, policy_data)} - _, error -> - error - end) + _, error -> + error + end) + + mrf_policies = + get_policies() + |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) + + exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + + base = + %{ + mrf_policies: mrf_policies, + exclusions: length(exclusions) > 0, + } + |> Map.merge(policy_configs) + + {:ok, base} end def describe(), do: get_policies() |> describe() diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 87fa514c3..ad2d9bf54 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -62,4 +62,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 2da3eac2f..d27386591 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do alias Pleroma.User + @behaviour Pleroma.Web.ActivityPub.MRF + require Logger # has the user successfully posted before? @@ -22,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do defp contains_links?(_), do: false + @impl true def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, @@ -45,4 +48,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do # in all other cases, pass through def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index b8d38aae6..dcb640b12 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -12,4 +12,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do Logger.info("REJECTING #{inspect(object)}") {:reject, object} end + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 15d8514be..259920cff 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -42,4 +42,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do end def filter(object), do: {:ok, object} + + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index a699f6a7e..deb4995dd 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -87,4 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index d5c341433..dfa1704c4 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -94,4 +94,36 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe() do + # This horror is needed to convert regex sigils to strings + mrf_keyword = + Pleroma.Config.get(:mrf_keyword, []) + |> Enum.map(fn {key, value} -> + {key, + Enum.map(value, fn + {pattern, replacement} -> + %{ + "pattern" => + if not is_binary(pattern) do + inspect(pattern) + else + pattern + end, + "replacement" => replacement + } + + pattern -> + if not is_binary(pattern) do + inspect(pattern) + else + pattern + end + end)} + end) + |> Enum.into(%{}) + + {:ok, %{mrf_keyword: mrf_keyword}} + end end diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f30fee0d5..a94cda856 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -27,4 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @impl true def filter(object), do: {:ok, object} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex index c47cb3298..19890ef0c 100644 --- a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex @@ -10,4 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do def filter(object) do {:ok, object} end + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 9c87c6963..36b0763be 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -25,4 +25,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do end def filter(object), do: {:ok, object} + + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index ea3df1b4d..52ffc4ca3 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -48,4 +48,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @impl true def filter(object), do: {:ok, object} + + @impl true + def describe(), do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2cf63d3db..07bef362f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -179,4 +179,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end def filter(object), do: {:ok, object} + + @impl true + def describe() do + exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + + mrf_simple = + Pleroma.Config.get(:mrf_simple) + |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) + |> Enum.into(%{}) + + {:ok, %{mrf_simple: mrf_simple}} + end end diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 765704389..b69410ca8 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -37,4 +37,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 6683b8d8e..de4e53cb8 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -149,4 +149,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do @impl true def filter(message), do: {:ok, message} + + @impl true + def describe(), do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index 47663414a..2b5f7d516 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -27,4 +27,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do end def filter(object), do: {:ok, object} + + @impl true + def describe() do + mrf_user_allowlist = + Config.get([:mrf_user_allowlist], []) + |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) + + {:ok, %{mrf_user_allowlist: mrf_user_allowlist}} + end end From 4ccd3410a859e0bb2365aee9967c7d8e88487591 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:57:39 +0000 Subject: [PATCH 33/48] nodeinfo: use MRF.describe() instead of hardcoded MRF transparency stuff --- .../web/nodeinfo/nodeinfo_controller.ex | 54 ++----------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index dc4858e43..066e9f5ba 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -34,64 +34,18 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do def raw_nodeinfo do stats = Stats.get_stats() - exclusions = Config.get([:instance, :mrf_transparency_exclusions]) - - mrf_simple = - Config.get(:mrf_simple) - |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) - |> Enum.into(%{}) - - # This horror is needed to convert regex sigils to strings - mrf_keyword = - Config.get(:mrf_keyword, []) - |> Enum.map(fn {key, value} -> - {key, - Enum.map(value, fn - {pattern, replacement} -> - %{ - "pattern" => - if not is_binary(pattern) do - inspect(pattern) - else - pattern - end, - "replacement" => replacement - } - - pattern -> - if not is_binary(pattern) do - inspect(pattern) - else - pattern - end - end)} - end) - |> Enum.into(%{}) - - mrf_policies = - MRF.get_policies() - |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - quarantined = Config.get([:instance, :quarantined_instances], []) staff_accounts = User.all_superusers() |> Enum.map(fn u -> u.ap_id end) - mrf_user_allowlist = - Config.get([:mrf_user_allowlist], []) - |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) - federation_response = if Config.get([:instance, :mrf_transparency]) do - %{ - mrf_policies: mrf_policies, - mrf_simple: mrf_simple, - mrf_keyword: mrf_keyword, - mrf_user_allowlist: mrf_user_allowlist, - quarantined_instances: quarantined, - exclusions: length(exclusions) > 0 - } + {:ok, data} = MRF.describe() + + data + |> Map.merge(%{quarantined_instances: quarantined}) else %{} end From c9468c9a597f768e0b236b65e5da7979df4ad42a Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:19:15 +0000 Subject: [PATCH 34/48] tests: add tests for MRF.describe() --- test/support/mrf_module_mock.ex | 2 +- test/web/activity_pub/mrf/mrf_test.exs | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex index e5ae21ad8..573eb0147 100644 --- a/test/support/mrf_module_mock.ex +++ b/test/support/mrf_module_mock.ex @@ -9,5 +9,5 @@ defmodule MRFModuleMock do def filter(message), do: {:ok, message} @impl true - def describe(), do: %{"mrf_module_mock" => "some config data"} + def describe(), do: {:ok, %{mrf_module_mock: "some config data"}} end diff --git a/test/web/activity_pub/mrf/mrf_test.exs b/test/web/activity_pub/mrf/mrf_test.exs index 1a888e18f..19e172939 100644 --- a/test/web/activity_pub/mrf/mrf_test.exs +++ b/test/web/activity_pub/mrf/mrf_test.exs @@ -57,4 +57,30 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do refute MRF.subdomain_match?(regexes, "example.com") end end + + describe "describe/0" do + test "it works as expected with noop policy" do + expected = %{ + mrf_policies: ["NoOpPolicy"], + exclusions: false + } + + {:ok, ^expected} = MRF.describe() + end + + test "it works as expected with mock policy" do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [MRFModuleMock]) + + expected = %{ + mrf_policies: ["MRFModuleMock"], + mrf_module_mock: "some config data", + exclusions: false + } + + {:ok, ^expected} = MRF.describe() + + Pleroma.Config.put([:instance, :rewrite_policy], config) + end + end end From 3af51afdd7773db566ed8c92c19d1605705f8208 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:32:40 +0000 Subject: [PATCH 35/48] tests: fix up nodeinfo tests --- test/web/node_info_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index d7f848bfa..f6147c286 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -85,6 +85,9 @@ defmodule Pleroma.Web.NodeInfoTest do end test "it shows MRF transparency data if enabled", %{conn: conn} do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + option = Pleroma.Config.get([:instance, :mrf_transparency]) Pleroma.Config.put([:instance, :mrf_transparency], true) @@ -98,11 +101,15 @@ defmodule Pleroma.Web.NodeInfoTest do assert response["metadata"]["federation"]["mrf_simple"] == simple_config + Pleroma.Config.put([:instance, :rewrite_policy], config) Pleroma.Config.put([:instance, :mrf_transparency], option) Pleroma.Config.put(:mrf_simple, %{}) end test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do + config = Pleroma.Config.get([:instance, :rewrite_policy]) + Pleroma.Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + option = Pleroma.Config.get([:instance, :mrf_transparency]) Pleroma.Config.put([:instance, :mrf_transparency], true) @@ -122,6 +129,7 @@ defmodule Pleroma.Web.NodeInfoTest do assert response["metadata"]["federation"]["mrf_simple"] == expected_config assert response["metadata"]["federation"]["exclusions"] == true + Pleroma.Config.put([:instance, :rewrite_policy], config) Pleroma.Config.put([:instance, :mrf_transparency], option) Pleroma.Config.put([:instance, :mrf_transparency_exclusions], exclusions) Pleroma.Config.put(:mrf_simple, %{}) From fb77dc50aae4fb9c8bdb7b20dc874e6a85ce322b Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:36:24 +0000 Subject: [PATCH 36/48] fix credo --- lib/pleroma/web/activity_pub/mrf.ex | 4 ++-- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex | 2 +- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 2 +- .../web/activity_pub/mrf/no_placeholder_text_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/noop_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/normalize_markup.ex | 2 +- lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 2 +- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/subchain_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 2 +- test/support/mrf_module_mock.ex | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 7533552d5..263ed11af 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -59,12 +59,12 @@ defmodule Pleroma.Web.ActivityPub.MRF do base = %{ mrf_policies: mrf_policies, - exclusions: length(exclusions) > 0, + exclusions: length(exclusions) > 0 } |> Map.merge(policy_configs) {:ok, base} end - def describe(), do: get_policies() |> describe() + def describe, do: get_policies() |> describe() end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index ad2d9bf54..de1eb4aa5 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -64,5 +64,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index d27386591..b90193ca0 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -50,5 +50,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index dcb640b12..f7831bc3e 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -14,5 +14,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do end @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 259920cff..4cc45233f 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -43,5 +43,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter(object), do: {:ok, object} - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index deb4995dd..621c9fa24 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -89,5 +89,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} + def describe, do: {:ok, %{mrf_hellthread: Pleroma.Config.get([:mrf_hellthread])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index dfa1704c4..4852355ac 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do def filter(message), do: {:ok, message} @impl true - def describe() do + def describe do # This horror is needed to convert regex sigils to strings mrf_keyword = Pleroma.Config.get(:mrf_keyword, []) diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index a94cda856..3ae81e025 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -29,5 +29,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do def filter(object), do: {:ok, object} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex index 19890ef0c..878c57925 100644 --- a/lib/pleroma/web/activity_pub/mrf/noop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/noop_policy.ex @@ -12,5 +12,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do end @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 36b0763be..62ed379f1 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -26,5 +26,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do def filter(object), do: {:ok, object} - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 52ffc4ca3..adaeb1c91 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -50,5 +50,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do def filter(object), do: {:ok, object} @impl true - def describe(), do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} + def describe, do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get([:mrf_rejectnonpublic])}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 07bef362f..56ff47304 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,7 +181,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do def filter(object), do: {:ok, object} @impl true - def describe() do + def describe do exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) mrf_simple = diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index b69410ca8..566c1e191 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -39,5 +39,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index de4e53cb8..2e2292aff 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -151,5 +151,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{}} + def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index 2b5f7d516..f77d1a3a3 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do def filter(object), do: {:ok, object} @impl true - def describe() do + def describe do mrf_user_allowlist = Config.get([:mrf_user_allowlist], []) |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex index 573eb0147..12c7e22bc 100644 --- a/test/support/mrf_module_mock.ex +++ b/test/support/mrf_module_mock.ex @@ -9,5 +9,5 @@ defmodule MRFModuleMock do def filter(message), do: {:ok, message} @impl true - def describe(), do: {:ok, %{mrf_module_mock: "some config data"}} + def describe, do: {:ok, %{mrf_module_mock: "some config data"}} end From eed36278a60e1366635343754bf0c660c49b8ad4 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:28:59 +0000 Subject: [PATCH 37/48] MRF: add vocabulary policy module --- config/config.exs | 4 +++ .../web/activity_pub/mrf/vocabulary_policy.ex | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex diff --git a/config/config.exs b/config/config.exs index b145cab22..66a8c18ea 100644 --- a/config/config.exs +++ b/config/config.exs @@ -333,6 +333,10 @@ config :pleroma, :mrf_keyword, config :pleroma, :mrf_subchain, match_actor: %{} +config :pleroma, :mrf_vocabulary, + accept: [], + reject: [] + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex new file mode 100644 index 000000000..de00b23da --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do + @moduledoc "Filter messages which belong to certain activity vocabularies" + + @behaviour Pleroma.Web.ActivityPub.MRF + + def filter(%{"type" => "Undo", "object" => child_message} = message) do + with {:ok, _} <- filter(child_message) do + {:ok, message} + else + {:reject, nil} -> + {:reject, nil} + end + end + + def filter(%{"type" => message_type} = message) do + with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), + rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), + true <- + length(accepted_vocabulary) == 0 || Enum.member?(accepted_vocabulary, message_type), + false <- + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), + {:ok, _} <- filter(message["object"]) do + {:ok, message} + else + _ -> {:reject, nil} + end + end + + def filter(message), do: {:ok, message} +end From 7a5e1d64a53f719c085cbff30228318311651a14 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:32:43 +0000 Subject: [PATCH 38/48] docs: document mrf_vocabulary module settings --- docs/config.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/config.md b/docs/config.md index 3b9be73ec..a4461521b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` @@ -266,6 +267,10 @@ config :pleroma, :mrf_subchain, * `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +## :mrf_vocabulary +* `accept`: A list of ActivityStreams terms to accept. If empty, all messages are accepted. +* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. + ## :media_proxy * `enabled`: Enables proxying of remote media to the instance’s proxy * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. From d5df62edd8297251fcbd7472ede27e76d6ded419 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 20:55:13 +0000 Subject: [PATCH 39/48] tests: add tests for mrf_vocabulary --- .../mrf/vocabulary_policy_test.exs | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 test/web/activity_pub/mrf/vocabulary_policy_test.exs diff --git a/test/web/activity_pub/mrf/vocabulary_policy_test.exs b/test/web/activity_pub/mrf/vocabulary_policy_test.exs new file mode 100644 index 000000000..c3b11d7a1 --- /dev/null +++ b/test/web/activity_pub/mrf/vocabulary_policy_test.exs @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicyTest do + use Pleroma.DataCase + + alias Pleroma.Web.ActivityPub.MRF.VocabularyPolicy + + describe "accept" do + test "it accepts based on parent activity type" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Like"]) + + message = %{ + "type" => "Like", + "object" => "whatever" + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it accepts based on child object type" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it does not accept disallowed child objects" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Create", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Article", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + + test "it does not accept disallowed parent types" do + config = Pleroma.Config.get([:mrf_vocabulary, :accept]) + Pleroma.Config.put([:mrf_vocabulary, :accept], ["Announce", "Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :accept], config) + end + end + + describe "reject" do + test "it rejects based on parent activity type" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) + + message = %{ + "type" => "Like", + "object" => "whatever" + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + + test "it rejects based on child object type" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Note"]) + + message = %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "whatever" + } + } + + {:reject, nil} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + + test "it passes through objects that aren't disallowed" do + config = Pleroma.Config.get([:mrf_vocabulary, :reject]) + Pleroma.Config.put([:mrf_vocabulary, :reject], ["Like"]) + + message = %{ + "type" => "Announce", + "object" => "whatever" + } + + {:ok, ^message} = VocabularyPolicy.filter(message) + + Pleroma.Config.put([:mrf_vocabulary, :reject], config) + end + end +end From b3afc0bb326e3b4d95e73684c71c1bf8f695d3c6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 21:00:23 +0000 Subject: [PATCH 40/48] update changelog for mrf_vocabulary --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35dcd3401..3c85bb90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Relays: Added a task to list relay subscriptions. +- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) - MRF (Simple Policy): Support for wildcard domains. - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. From c7a3a15c3de93212a131ea5886278c4030309941 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:39:26 +0000 Subject: [PATCH 41/48] mrf_vocabulary: add describe API support --- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index de00b23da..74da8d57e 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -31,4 +31,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do end def filter(message), do: {:ok, message} + + def describe, do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary)}} end From 4681747ec1e889530b0df52fe599aa236078a0b6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 13 Aug 2019 22:40:18 +0000 Subject: [PATCH 42/48] docs tweak --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index a4461521b..4699baa80 100644 --- a/docs/config.md +++ b/docs/config.md @@ -268,7 +268,7 @@ config :pleroma, :mrf_subchain, * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) ## :mrf_vocabulary -* `accept`: A list of ActivityStreams terms to accept. If empty, all messages are accepted. +* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. * `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. ## :media_proxy From 606ccbab035dce7dfc543e1d6354ed8ae3c87c3c Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:36:42 +0000 Subject: [PATCH 43/48] update changelog to cover MRF describe API. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c85bb90d..4effcf887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added +- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. + Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules. - Relays: Added a task to list relay subscriptions. - MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) - MRF (Simple Policy): Support for wildcard domains. From e652f83e5a99e7f8526952147eec39eee63914e1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 9 Jul 2019 10:39:36 +0000 Subject: [PATCH 44/48] Apply suggestion to docs/config.md --- docs/config.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/config.md b/docs/config.md index 4699baa80..37824ecac 100644 --- a/docs/config.md +++ b/docs/config.md @@ -87,6 +87,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. * `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: From 1a46a13d1523347e8310dde9f5bddf396bd11342 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 29 Jun 2019 20:04:50 +0300 Subject: [PATCH 45/48] [#161] Limited replies depth on incoming federation in order to prevent memory leaks on recursive replies fetching. --- lib/pleroma/object.ex | 24 ++-- lib/pleroma/object/fetcher.ex | 8 +- .../web/activity_pub/transmogrifier.ex | 112 +++++++++++------- lib/pleroma/web/federator/federator.ex | 6 + .../web/ostatus/handlers/note_handler.ex | 13 +- lib/pleroma/web/ostatus/ostatus.ex | 22 ++-- test/web/activity_pub/transmogrifier_test.exs | 37 +++++- test/web/ostatus/ostatus_test.exs | 28 ++++- 8 files changed, 170 insertions(+), 80 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 4b181ec59..b8647dd26 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,20 +44,20 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(_, fetch_remote \\ true) + def normalize(_, fetch_remote \\ true, options \\ []) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! - def normalize(%Object{} = object, _), do: object - def normalize(%Activity{object: %Object{} = object}, _), do: object + def normalize(%Object{} = object, _, _), do: object + def normalize(%Activity{object: %Object{} = object}, _, _), do: object # A hack for fake activities - def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do + def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do %Object{id: "pleroma:fake_object_id", data: data} end # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. - def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -67,7 +67,7 @@ defmodule Pleroma.Object do normalize(ap_id, fetch_remote) end - def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -78,10 +78,14 @@ defmodule Pleroma.Object do end # Old way, try fetching the object through cache. - def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote) - def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) - def normalize(_, _), do: nil + def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote) + def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + + def normalize(ap_id, true, options) when is_binary(ap_id) do + Fetcher.fetch_object_from_id!(ap_id, options) + end + + def normalize(_, _, _), do: nil # Owned objects can only be mutated by their owner def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}), diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 3c137836c..a1c668156 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Object.Fetcher do # TODO: # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do + def fetch_object_from_id(id, options \\ []) do if object = Object.get_cached_by_ap_id(id) do {:ok, object} else @@ -39,7 +39,7 @@ defmodule Pleroma.Object.Fetcher do "object" => data }, {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, - {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:ok, activity} <- Transmogrifier.handle_incoming(params, options), {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} @@ -69,8 +69,8 @@ defmodule Pleroma.Object.Fetcher do end end - def fetch_object_from_id!(id) do - with {:ok, object} <- fetch_object_from_id(id) do + def fetch_object_from_id!(id, options \\ []) do + with {:ok, object} <- fetch_object_from_id(id, options) do object else _e -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ca6940026..035d9486b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Federator import Ecto.Query @@ -22,20 +23,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ - def fix_object(object) do + def fix_object(object, options \\ []) do object |> strip_internal_fields |> fix_actor |> fix_url |> fix_attachments |> fix_context - |> fix_in_reply_to + |> fix_in_reply_to(options) |> fix_emoji |> fix_tag |> fix_content_map |> fix_addressing |> fix_summary - |> fix_type + |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -152,7 +153,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, _options) + def fix_in_reply_to(object, options \\ []) + + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -170,24 +173,30 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "" end - case get_obj_helper(in_reply_to_id) do - {:ok, replied_object} -> - with %Activity{} = _activity <- - Activity.get_create_by_object_ap_id(replied_object.data["id"]) do - object - |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) - |> Map.put("context", replied_object.data["context"] || object["conversation"]) - else - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object - end + object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object + if (options[:depth] || 1) <= Federator.max_replies_depth() do + case get_obj_helper(in_reply_to_id, options) do + {:ok, replied_object} -> + with %Activity{} = _activity <- + Activity.get_create_by_object_ap_id(replied_object.data["id"]) do + object + |> Map.put("inReplyTo", replied_object.data["id"]) + |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) + |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) + |> Map.put("context", replied_object.data["context"] || object["conversation"]) + else + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + else + object end end @@ -369,9 +378,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + def handle_incoming(data, options \\ []) + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. - def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do + def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data, _options) do with context <- data["context"] || Utils.generate_context_id(), content <- data["content"] || "", %User{} = actor <- User.get_cached_by_ap_id(actor), @@ -404,15 +415,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end # disallow objects with bogus IDs - def handle_incoming(%{"id" => nil}), do: :error - def handle_incoming(%{"id" => ""}), do: :error + def handle_incoming(%{"id" => nil}, _options), do: :error + def handle_incoming(%{"id" => ""}, _options), do: :error # length of https:// = 8, should validate better, but good enough for now. - def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error + def handle_incoming(%{"id" => id}, _options) when not (is_binary(id) and length(id) > 8), + do: :error # TODO: validate those with a Ecto scheme # - tags # - emoji - def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) + def handle_incoming( + %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + options + ) when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) @@ -422,7 +437,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with nil <- Activity.get_create_by_object_ap_id(object["id"]), {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do - object = fix_object(data["object"]) + options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + object = fix_object(data["object"], options) params = %{ to: data["to"], @@ -447,7 +463,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data + %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -498,7 +515,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -519,7 +537,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -543,7 +562,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -556,7 +576,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -571,7 +592,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} = - data + data, + _options ) when object_type in ["Person", "Application", "Service", "Organization"] do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do @@ -609,7 +631,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data + %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data, + _options ) do object_id = Utils.get_ap_id(object_id) @@ -640,7 +663,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Announce", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -658,7 +682,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Follow", "object" => followed}, "actor" => follower, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -676,7 +701,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Block", "object" => blocked}, "actor" => blocker, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker), @@ -689,7 +715,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data + %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, + _options ) do with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), @@ -708,7 +735,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Like", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -720,10 +748,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming(_), do: :error + def handle_incoming(_, _), do: :error - def get_obj_helper(id) do - if object = Object.normalize(id), do: {:ok, object}, else: nil + def get_obj_helper(id, options \\ []) do + if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil end def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4c9fe284..7c13ff323 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,6 +22,12 @@ defmodule Pleroma.Web.Federator do refresh_subscriptions() end + @max_replies_depth 100 + + @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def max_replies_depth, do: @max_replies_depth + # Client API def incoming_doc(doc) do diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 5b4befbb0..5dbf35155 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator alias Pleroma.Web.OStatus alias Pleroma.Web.XML @@ -88,14 +89,15 @@ defmodule Pleroma.Web.OStatus.NoteHandler do Map.put(note, "external_url", url) end - def fetch_replied_to_activity(entry, in_reply_to) do + def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do activity else _e -> - with in_reply_to_href when not is_nil(in_reply_to_href) <- + with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), - {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href) do + {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do activity else _e -> nil @@ -104,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do end # TODO: Clean this up a bit. - def handle_note(entry, doc \\ nil) do + def handle_note(entry, doc \\ nil, options \\ []) do with id <- XML.string_from_xpath("//id", entry), activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), @@ -112,7 +114,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do content_html <- OStatus.get_content(entry), cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), - in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to), + options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1), + in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options), in_reply_to_object <- (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 56975926f..331cbc0b7 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Web.OStatus do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end - def handle_incoming(xml_string) do + def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do with {:ok, actor_user} <- find_make_or_update_actor(doc), do: Pleroma.Instances.set_reachable(actor_user.ap_id) @@ -91,10 +91,12 @@ defmodule Pleroma.Web.OStatus do _ -> case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity _ -> Logger.error("Couldn't parse incoming document") @@ -366,7 +368,7 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_atom_url(url) do + def fetch_activity_from_atom_url(url, options \\ []) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( @@ -374,7 +376,7 @@ defmodule Pleroma.Web.OStatus do [{:Accept, "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") - handle_incoming(body) + handle_incoming(body, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -382,13 +384,13 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_html_url(url) do + def fetch_activity_from_html_url(url, options \\ []) do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do - fetch_activity_from_atom_url(atom_url) + fetch_activity_from_atom_url(atom_url, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -396,11 +398,11 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_url(url) do - with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url) do + def fetch_activity_from_url(url, options \\ []) do + with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do {:ok, activities} else - _e -> fetch_activity_from_html_url(url) + _e -> fetch_activity_from_html_url(url, options) end rescue e -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b8b998acd..5d8a6a3b6 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -11,12 +11,13 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription + import Mock import Pleroma.Factory import ExUnit.CaptureLog - alias Pleroma.Web.CommonAPI setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -46,12 +47,10 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data["object"] |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") - data = - data - |> Map.put("object", object) - + data = Map.put(data, "object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - returned_object = Object.normalize(returned_activity.data["object"]) + + returned_object = Object.normalize(returned_activity.data["object"], false) assert activity = Activity.get_create_by_object_ap_id( @@ -61,6 +60,32 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end + test "it does not fetch replied-to activities beyond max_replies_depth" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") + + data = Map.put(data, "object", object) + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + + returned_object = Object.normalize(returned_activity.data["object"], false) + + refute Activity.get_create_by_object_ap_id( + "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + ) + + assert returned_object.data["inReplyToAtomUri"] == + "https://shitposter.club/notice/2827873" + end + end + test "it does not crash if the object in inReplyTo can't be fetched" do data = File.read!("test/fixtures/mastodon-post-activity.json") diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 41cf188ca..9318292a6 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -11,8 +11,10 @@ defmodule Pleroma.Web.OStatusTest do alias Pleroma.User alias Pleroma.Web.OStatus alias Pleroma.Web.XML - import Pleroma.Factory + import ExUnit.CaptureLog + import Mock + import Pleroma.Factory setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -266,10 +268,13 @@ defmodule Pleroma.Web.OStatusTest do assert favorited_activity.local end - test "handle incoming replies" do + test_with_mock "handle incoming replies, fetching replied-to activities if we don't have them", + OStatus, + [:passthrough], + [] do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"], false) assert activity.data["type"] == "Create" assert object.data["type"] == "Note" @@ -282,6 +287,23 @@ defmodule Pleroma.Web.OStatusTest do assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] + + assert called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end + + test_with_mock "handle incoming replies, not fetching replied-to activities beyond max_replies_depth", + OStatus, + [:passthrough], + [] do + incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"], false) + + refute called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end end test "handle incoming follows" do From bc6d1f9ed951f690544d80b411b35c07d9fd7793 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 30 Jun 2019 15:58:50 +0300 Subject: [PATCH 46/48] [#161] Refactoring, documentation. --- CHANGELOG.md | 2 +- config/config.exs | 1 + lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++---- lib/pleroma/web/federator/federator.ex | 12 +++++++++--- lib/pleroma/web/ostatus/handlers/note_handler.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 2 +- test/web/ostatus/ostatus_test.exs | 2 +- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4effcf887..f5289ea6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Not being able to access the Mastodon FE login page on private instances - MRF: ensure that subdomain_match calls are case-insensitive - Fix internal server error when using the healthcheck API. -- `federation_incoming_replies_max_depth` option being ignored in certain cases. ### Added - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. @@ -24,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` +- Federation: Support for restricting max. reply-to depth on fetching ### Removed - Federation: Remove `likes` from objects. diff --git a/config/config.exs b/config/config.exs index 66a8c18ea..7e901a1f5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -220,6 +220,7 @@ config :pleroma, :instance, }, registrations_open: true, federating: true, + federation_incoming_replies_max_depth: 100, federation_reachability_timeout_days: 7, federation_publisher_modules: [ Pleroma.Web.ActivityPub.Publisher, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 035d9486b..2a66b2162 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -153,8 +153,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(object, options \\ []) - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = @@ -175,7 +173,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) - if (options[:depth] || 1) <= Federator.max_replies_depth() do + if Federator.allowed_incoming_reply_depth?(options[:depth]) do case get_obj_helper(in_reply_to_id, options) do {:ok, replied_object} -> with %Activity{} = _activity <- @@ -339,7 +337,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do when is_binary(reply_id) do reply = with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), - {:ok, object} <- get_obj_helper(reply_id) do + {:ok, object} <- get_obj_helper(reply_id, options) do object end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 7c13ff323..f4f9e83e0 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,11 +22,17 @@ defmodule Pleroma.Web.Federator do refresh_subscriptions() end - @max_replies_depth 100 - @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - def max_replies_depth, do: @max_replies_depth + def allowed_incoming_reply_depth?(depth) do + max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth]) + + if max_replies_depth do + (depth || 1) <= max_replies_depth + else + true + end + end # Client API diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 5dbf35155..c8c1c905e 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -94,7 +94,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do activity else _e -> - with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5d8a6a3b6..9c1584dbe 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -72,7 +72,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = Map.put(data, "object", object) with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, returned_activity} = Transmogrifier.handle_incoming(data) returned_object = Object.normalize(returned_activity.data["object"], false) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 9318292a6..ff9bb6351 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -298,7 +298,7 @@ defmodule Pleroma.Web.OStatusTest do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, [activity]} = OStatus.handle_incoming(incoming) object = Object.normalize(activity.data["object"], false) From 9fa2011c17926ee7bdfecdcd7a35649898420055 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 6 Jul 2019 07:18:26 +0000 Subject: [PATCH 47/48] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5289ea6d..843e4fc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for wildcard domains in user domain blocks setting. - Configuration: `quarantined_instances` support wildcard domains. - Mix Tasks: `mix pleroma.database fix_likes_collections` -- Federation: Support for restricting max. reply-to depth on fetching +- Configuration: `federation_incoming_replies_max_depth` option ### Removed - Federation: Remove `likes` from objects. From dcb4711e92382b88326260d7e1951eb66941ed1b Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 14 Aug 2019 01:47:26 +0000 Subject: [PATCH 48/48] mix: update version to 1.0.5 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 843e4fc21..f84815afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [1.0.5] - 2019-08-?? +## [1.0.5] - 2019-08-13 ### Fixed - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set - Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted diff --git a/mix.exs b/mix.exs index 0673e4581..58d7d3b67 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("1.0.4"), + version: version("1.0.5"), elixir: "~> 1.7", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(),