From 647087d7fd3796c628ceca237d56a2e27a2307cf Mon Sep 17 00:00:00 2001 From: Ilja Date: Thu, 24 Sep 2020 00:34:59 +0200 Subject: [PATCH 01/21] Deprectate strings for SimplePolicy When strings are detected in the simplepolicy, a warning will be given and the config will be changed to use tuples instead --- lib/pleroma/config/deprecation_warnings.ex | 63 ++++++++++++++- .../config/deprecation_warnings_test.exs | 76 +++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index fedd58a7e..dd5c81094 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -20,6 +20,66 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} ] + def check_simple_policy_tuples do + has_strings = + Config.get([:mrf_simple]) + |> Enum.any?(fn {_, v} -> Enum.any?(v, fn e -> is_binary(e) end) end) + + if has_strings do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using strings in the SimplePolicy configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :mrf_simple, + media_removal: ["instance.tld"], + media_nsfw: ["instance.tld"], + federated_timeline_removal: ["instance.tld"], + report_removal: ["instance.tld"], + reject: ["instance.tld"], + followers_only: ["instance.tld"], + accept: ["instance.tld"], + avatar_removal: ["instance.tld"], + banner_removal: ["instance.tld"], + reject_deletes: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :mrf_simple, + media_removal: [{"instance.tld", "Reason for media removal"}], + media_nsfw: [{"instance.tld", "Reason for media nsfw"}], + federated_timeline_removal: [{"instance.tld", "Reason for federated timeline removal"}], + report_removal: [{"instance.tld", "Reason for report removal"}], + reject: [{"instance.tld", "Reason for reject"}], + followers_only: [{"instance.tld", "Reason for followers only"}], + accept: [{"instance.tld", "Reason for accept"}], + avatar_removal: [{"instance.tld", "Reason for avatar removal"}], + banner_removal: [{"instance.tld", "Reason for banner removal"}], + reject_deletes: [{"instance.tld", "Reason for reject deletes"}] + ``` + """) + + new_config = + Config.get([:mrf_simple]) + |> Enum.map(fn {k, v} -> + {k, + Enum.map(v, fn + {instance, reason} -> {instance, reason} + instance -> {instance, ""} + end)} + end) + + Config.put([:mrf_simple], new_config) + + :error + else + :ok + end + end + def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do Logger.warn(""" @@ -42,7 +102,8 @@ def warn do :ok <- check_activity_expiration_config(), :ok <- check_remote_ip_plug_name(), :ok <- check_uploders_s3_public_endpoint(), - :ok <- check_old_chat_shoutbox() do + :ok <- check_old_chat_shoutbox(), + :ok <- check_simple_policy_tuples() do :ok else _ -> diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index ccf86634f..1c686ec7c 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -11,6 +11,82 @@ defmodule Pleroma.Config.DeprecationWarningsTest do alias Pleroma.Config alias Pleroma.Config.DeprecationWarnings + describe "simple policy tuples" do + test "gives warning when there are still strings" do + clear_config([:mrf_simple], + media_removal: ["some.removal"], + media_nsfw: ["some.nsfw"], + federated_timeline_removal: ["some.tl.removal"], + report_removal: ["some.report.removal"], + reject: ["some.reject"], + followers_only: ["some.followers.only"], + accept: ["some.accept"], + avatar_removal: ["some.avatar.removal"], + banner_removal: ["some.banner.removal"], + reject_deletes: ["some.reject.deletes"] + ) + + assert capture_log(fn -> DeprecationWarnings.check_simple_policy_tuples() end) =~ + """ + !!!DEPRECATION WARNING!!! + Your config is using strings in the SimplePolicy configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :mrf_simple, + media_removal: ["instance.tld"], + media_nsfw: ["instance.tld"], + federated_timeline_removal: ["instance.tld"], + report_removal: ["instance.tld"], + reject: ["instance.tld"], + followers_only: ["instance.tld"], + accept: ["instance.tld"], + avatar_removal: ["instance.tld"], + banner_removal: ["instance.tld"], + reject_deletes: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :mrf_simple, + media_removal: [{"instance.tld", "Reason for media removal"}], + media_nsfw: [{"instance.tld", "Reason for media nsfw"}], + federated_timeline_removal: [{"instance.tld", "Reason for federated timeline removal"}], + report_removal: [{"instance.tld", "Reason for report removal"}], + reject: [{"instance.tld", "Reason for reject"}], + followers_only: [{"instance.tld", "Reason for followers only"}], + accept: [{"instance.tld", "Reason for accept"}], + avatar_removal: [{"instance.tld", "Reason for avatar removal"}], + banner_removal: [{"instance.tld", "Reason for banner removal"}], + reject_deletes: [{"instance.tld", "Reason for reject deletes"}] + ``` + """ + end + + test "transforms config to tuples" do + clear_config([:mrf_simple], + media_removal: ["some.removal", {"some.other.instance", "Some reason"}] + ) + + expected_config = [ + {:media_removal, [{"some.removal", ""}, {"some.other.instance", "Some reason"}]} + ] + + capture_log(fn -> DeprecationWarnings.check_simple_policy_tuples() end) + + assert Config.get([:mrf_simple]) == expected_config + end + + test "doesn't give a warning with correct config" do + clear_config([:mrf_simple], + media_removal: [{"some.removal", ""}, {"some.other.instance", "Some reason"}] + ) + + assert capture_log(fn -> DeprecationWarnings.check_simple_policy_tuples() end) == "" + end + end + test "check_old_mrf_config/0" do clear_config([:instance, :rewrite_policy], []) clear_config([:instance, :mrf_transparency], true) From 4ba0beb60ccdc301f455c32773a3144b9448b2fb Mon Sep 17 00:00:00 2001 From: Ilja Date: Thu, 24 Sep 2020 21:01:33 +0200 Subject: [PATCH 02/21] Make mrfSimple work with tuples * Changed SimplePolicy * I also grepped in test/ for ':mrf_simple' to see what other things could be affected --- .../web/activity_pub/mrf/simple_policy.ex | 154 +++++++++--------- test/pleroma/user_test.exs | 2 +- .../activity_pub/mrf/simple_policy_test.exs | 69 ++++---- test/pleroma/web/node_info_test.exs | 10 +- 4 files changed, 123 insertions(+), 112 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 30562ac08..8ef03aa3a 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_accept(%{host: actor_host} = _actor_info, object) do accepts = - Config.get([:mrf_simple, :accept]) + instance_list(:accept) |> MRF.subdomains_regex() cond do @@ -28,7 +28,7 @@ defp check_accept(%{host: actor_host} = _actor_info, object) do defp check_reject(%{host: actor_host} = _actor_info, object) do rejects = - Config.get([:mrf_simple, :reject]) + instance_list(:reject) |> MRF.subdomains_regex() if MRF.subdomain_match?(rejects, actor_host) do @@ -44,7 +44,7 @@ defp check_media_removal( ) when length(child_attachment) > 0 do media_removal = - Config.get([:mrf_simple, :media_removal]) + instance_list(:media_removal) |> MRF.subdomains_regex() object = @@ -68,7 +68,7 @@ defp check_media_nsfw( } = object ) do media_nsfw = - Config.get([:mrf_simple, :media_nsfw]) + instance_list(:media_nsfw) |> MRF.subdomains_regex() object = @@ -85,7 +85,7 @@ defp check_media_nsfw(_actor_info, object), do: {:ok, object} defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do timeline_removal = - Config.get([:mrf_simple, :federated_timeline_removal]) + instance_list(:federated_timeline_removal) |> MRF.subdomains_regex() object = @@ -112,7 +112,7 @@ defp intersection(list1, list2) do defp check_followers_only(%{host: actor_host} = _actor_info, object) do followers_only = - Config.get([:mrf_simple, :followers_only]) + instance_list(:followers_only) |> MRF.subdomains_regex() object = @@ -137,7 +137,7 @@ defp check_followers_only(%{host: actor_host} = _actor_info, object) do defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do report_removal = - Config.get([:mrf_simple, :report_removal]) + instance_list(:report_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(report_removal, actor_host) do @@ -151,7 +151,7 @@ defp check_report_removal(_actor_info, object), do: {:ok, object} defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do avatar_removal = - Config.get([:mrf_simple, :avatar_removal]) + instance_list(:avatar_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(avatar_removal, actor_host) do @@ -165,7 +165,7 @@ defp check_avatar_removal(_actor_info, object), do: {:ok, object} defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do banner_removal = - Config.get([:mrf_simple, :banner_removal]) + instance_list(:banner_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(banner_removal, actor_host) do @@ -185,12 +185,19 @@ defp check_object(%{"object" => object} = activity) do defp check_object(object), do: {:ok, object} + defp instance_list(config_key) do + Config.get([:mrf_simple, config_key]) + |> Enum.map(fn + {instance, _} -> instance + end) + end + @impl true def filter(%{"type" => "Delete", "actor" => actor} = object) do %{host: actor_host} = URI.parse(actor) reject_deletes = - Config.get([:mrf_simple, :reject_deletes]) + instance_list(:reject_deletes) |> MRF.subdomains_regex() if MRF.subdomain_match?(reject_deletes, actor_host) do @@ -257,7 +264,7 @@ def describe do mrf_simple = Config.get(:mrf_simple) - |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) + |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {v, _} -> v in exclusions end)} end) |> Enum.into(%{}) {:ok, %{mrf_simple: mrf_simple}} @@ -270,70 +277,67 @@ def config_description do related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy", label: "MRF Simple", description: "Simple ingress policies", - children: [ - %{ - key: :media_removal, - type: {:list, :string}, - description: "List of instances to strip media attachments from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :media_nsfw, - label: "Media NSFW", - type: {:list, :string}, - description: "List of instances to tag all media as NSFW (sensitive) from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :federated_timeline_removal, - type: {:list, :string}, - description: - "List of instances to remove from the Federated (aka The Whole Known Network) Timeline", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :reject, - type: {:list, :string}, - description: "List of instances to reject activities from (except deletes)", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :accept, - type: {:list, :string}, - description: "List of instances to only accept activities from (except deletes)", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :followers_only, - type: {:list, :string}, - description: "Force posts from the given instances to be visible by followers only", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :report_removal, - type: {:list, :string}, - description: "List of instances to reject reports from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :avatar_removal, - type: {:list, :string}, - description: "List of instances to strip avatars from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :banner_removal, - type: {:list, :string}, - description: "List of instances to strip banners from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :reject_deletes, - type: {:list, :string}, - description: "List of instances to reject deletions from", - suggestions: ["example.com", "*.example.com"] - } - ] + children: + [ + %{ + key: :media_removal, + description: + "List of instances to strip media attachments from and the reason for doing so" + }, + %{ + key: :media_nsfw, + label: "Media NSFW", + description: + "List of instances to tag all media as NSFW (sensitive) from and the reason for doing so" + }, + %{ + key: :federated_timeline_removal, + description: + "List of instances to remove from the Federated (aka The Whole Known Network) Timeline and the reason for doing so" + }, + %{ + key: :reject, + description: + "List of instances to reject activities from (except deletes) and the reason for doing so" + }, + %{ + key: :accept, + description: + "List of instances to only accept activities from (except deletes) and the reason for doing so" + }, + %{ + key: :followers_only, + description: + "Force posts from the given instances to be visible by followers only and the reason for doing so" + }, + %{ + key: :report_removal, + description: "List of instances to reject reports from and the reason for doing so" + }, + %{ + key: :avatar_removal, + description: "List of instances to strip avatars from and the reason for doing so" + }, + %{ + key: :banner_removal, + description: "List of instances to strip banners from and the reason for doing so" + }, + %{ + key: :reject_deletes, + description: "List of instances to reject deletions from and the reason for doing so" + } + ] + |> Enum.map(fn setting -> + Map.merge( + setting, + %{ + type: {:list, :tuple}, + key_placeholder: "instance", + value_placeholder: "reason", + suggestions: [{"example.com", "Some reason"}, {"*.example.com", "Another reason"}] + } + ) + end) } end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 4021a565d..c2ed2c2a3 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -480,7 +480,7 @@ test "it sends a welcome chat message if it is set" do ) test "it sends a welcome chat message when Simple policy applied to local instance" do - clear_config([:mrf_simple, :media_nsfw], ["localhost"]) + clear_config([:mrf_simple, :media_nsfw], [{"localhost", ""}]) welcome_user = insert(:user) clear_config([:welcome, :chat_message, :enabled], true) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 0b0143d09..0a0f51bdb 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -33,7 +33,7 @@ test "is empty" do end test "has a matching host" do - clear_config([:mrf_simple, :media_removal], ["remote.instance"]) + clear_config([:mrf_simple, :media_removal], [{"remote.instance", "Some reason"}]) media_message = build_media_message() local_message = build_local_message() @@ -46,7 +46,7 @@ test "has a matching host" do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_removal], ["*.remote.instance"]) + clear_config([:mrf_simple, :media_removal], [{"*.remote.instance", "Whatever reason"}]) media_message = build_media_message() local_message = build_local_message() @@ -70,7 +70,7 @@ test "is empty" do end test "has a matching host" do - clear_config([:mrf_simple, :media_nsfw], ["remote.instance"]) + clear_config([:mrf_simple, :media_nsfw], [{"remote.instance", "Whetever"}]) media_message = build_media_message() local_message = build_local_message() @@ -81,7 +81,7 @@ test "has a matching host" do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_nsfw], ["*.remote.instance"]) + clear_config([:mrf_simple, :media_nsfw], [{"*.remote.instance", "yeah yeah"}]) media_message = build_media_message() local_message = build_local_message() @@ -115,7 +115,7 @@ test "is empty" do end test "has a matching host" do - clear_config([:mrf_simple, :report_removal], ["remote.instance"]) + clear_config([:mrf_simple, :report_removal], [{"remote.instance", "muh"}]) report_message = build_report_message() local_message = build_local_message() @@ -124,7 +124,7 @@ test "has a matching host" do end test "match with wildcard domain" do - clear_config([:mrf_simple, :report_removal], ["*.remote.instance"]) + clear_config([:mrf_simple, :report_removal], [{"*.remote.instance", "suya"}]) report_message = build_report_message() local_message = build_local_message() @@ -159,7 +159,7 @@ test "has a matching host" do |> URI.parse() |> Map.fetch!(:host) - clear_config([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + clear_config([:mrf_simple, :federated_timeline_removal], [{ftl_message_actor_host, "uwu"}]) local_message = build_local_message() assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) @@ -180,7 +180,10 @@ test "match with wildcard domain" do |> URI.parse() |> Map.fetch!(:host) - clear_config([:mrf_simple, :federated_timeline_removal], ["*." <> ftl_message_actor_host]) + clear_config([:mrf_simple, :federated_timeline_removal], [ + {"*." <> ftl_message_actor_host, "owo"} + ]) + local_message = build_local_message() assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) @@ -203,7 +206,9 @@ test "has a matching host but only as:Public in to" do ftl_message = Map.put(ftl_message, "cc", []) - clear_config([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + clear_config([:mrf_simple, :federated_timeline_removal], [ + {ftl_message_actor_host, "spiderwaifu goes 88w88"} + ]) assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] @@ -232,7 +237,7 @@ test "is empty" do end test "activity has a matching host" do - clear_config([:mrf_simple, :reject], ["remote.instance"]) + clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) remote_message = build_remote_message() @@ -240,7 +245,7 @@ test "activity has a matching host" do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :reject], ["*.remote.instance"]) + clear_config([:mrf_simple, :reject], [{"*.remote.instance", ""}]) remote_message = build_remote_message() @@ -248,7 +253,7 @@ test "activity matches with wildcard domain" do end test "actor has a matching host" do - clear_config([:mrf_simple, :reject], ["remote.instance"]) + clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) remote_user = build_remote_user() @@ -256,7 +261,7 @@ test "actor has a matching host" do end test "reject Announce when object would be rejected" do - clear_config([:mrf_simple, :reject], ["blocked.tld"]) + clear_config([:mrf_simple, :reject], [{"blocked.tld", ""}]) announce = %{ "type" => "Announce", @@ -268,7 +273,7 @@ test "reject Announce when object would be rejected" do end test "reject by URI object" do - clear_config([:mrf_simple, :reject], ["blocked.tld"]) + clear_config([:mrf_simple, :reject], [{"blocked.tld", ""}]) announce = %{ "type" => "Announce", @@ -322,7 +327,7 @@ test "has a matching host" do |> URI.parse() |> Map.fetch!(:host) - clear_config([:mrf_simple, :followers_only], [actor_domain]) + clear_config([:mrf_simple, :followers_only], [{actor_domain, ""}]) assert {:ok, new_activity} = SimplePolicy.filter(activity) assert actor.follower_address in new_activity["cc"] @@ -350,7 +355,7 @@ test "is empty" do end test "is not empty but activity doesn't have a matching host" do - clear_config([:mrf_simple, :accept], ["non.matching.remote"]) + clear_config([:mrf_simple, :accept], [{"non.matching.remote", ""}]) local_message = build_local_message() remote_message = build_remote_message() @@ -360,7 +365,7 @@ test "is not empty but activity doesn't have a matching host" do end test "activity has a matching host" do - clear_config([:mrf_simple, :accept], ["remote.instance"]) + clear_config([:mrf_simple, :accept], [{"remote.instance", ""}]) local_message = build_local_message() remote_message = build_remote_message() @@ -370,7 +375,7 @@ test "activity has a matching host" do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :accept], ["*.remote.instance"]) + clear_config([:mrf_simple, :accept], [{"*.remote.instance", ""}]) local_message = build_local_message() remote_message = build_remote_message() @@ -380,7 +385,7 @@ test "activity matches with wildcard domain" do end test "actor has a matching host" do - clear_config([:mrf_simple, :accept], ["remote.instance"]) + clear_config([:mrf_simple, :accept], [{"remote.instance", ""}]) remote_user = build_remote_user() @@ -398,7 +403,7 @@ test "is empty" do end test "is not empty but it doesn't have a matching host" do - clear_config([:mrf_simple, :avatar_removal], ["non.matching.remote"]) + clear_config([:mrf_simple, :avatar_removal], [{"non.matching.remote", ""}]) remote_user = build_remote_user() @@ -406,7 +411,7 @@ test "is not empty but it doesn't have a matching host" do end test "has a matching host" do - clear_config([:mrf_simple, :avatar_removal], ["remote.instance"]) + clear_config([:mrf_simple, :avatar_removal], [{"remote.instance", ""}]) remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) @@ -415,7 +420,7 @@ test "has a matching host" do end test "match with wildcard domain" do - clear_config([:mrf_simple, :avatar_removal], ["*.remote.instance"]) + clear_config([:mrf_simple, :avatar_removal], [{"*.remote.instance", ""}]) remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) @@ -434,7 +439,7 @@ test "is empty" do end test "is not empty but it doesn't have a matching host" do - clear_config([:mrf_simple, :banner_removal], ["non.matching.remote"]) + clear_config([:mrf_simple, :banner_removal], [{"non.matching.remote", ""}]) remote_user = build_remote_user() @@ -442,7 +447,7 @@ test "is not empty but it doesn't have a matching host" do end test "has a matching host" do - clear_config([:mrf_simple, :banner_removal], ["remote.instance"]) + clear_config([:mrf_simple, :banner_removal], [{"remote.instance", ""}]) remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) @@ -451,7 +456,7 @@ test "has a matching host" do end test "match with wildcard domain" do - clear_config([:mrf_simple, :banner_removal], ["*.remote.instance"]) + clear_config([:mrf_simple, :banner_removal], [{"*.remote.instance", ""}]) remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) @@ -464,7 +469,7 @@ test "match with wildcard domain" do setup do: clear_config([:mrf_simple, :reject_deletes], []) test "it accepts deletions even from rejected servers" do - clear_config([:mrf_simple, :reject], ["remote.instance"]) + clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) deletion_message = build_remote_deletion_message() @@ -472,7 +477,7 @@ test "it accepts deletions even from rejected servers" do end test "it accepts deletions even from non-whitelisted servers" do - clear_config([:mrf_simple, :accept], ["non.matching.remote"]) + clear_config([:mrf_simple, :accept], [{"non.matching.remote", ""}]) deletion_message = build_remote_deletion_message() @@ -481,10 +486,10 @@ test "it accepts deletions even from non-whitelisted servers" do end describe "when :reject_deletes is not empty but it doesn't have a matching host" do - setup do: clear_config([:mrf_simple, :reject_deletes], ["non.matching.remote"]) + setup do: clear_config([:mrf_simple, :reject_deletes], [{"non.matching.remote", ""}]) test "it accepts deletions even from rejected servers" do - clear_config([:mrf_simple, :reject], ["remote.instance"]) + clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) deletion_message = build_remote_deletion_message() @@ -492,7 +497,7 @@ test "it accepts deletions even from rejected servers" do end test "it accepts deletions even from non-whitelisted servers" do - clear_config([:mrf_simple, :accept], ["non.matching.remote"]) + clear_config([:mrf_simple, :accept], [{"non.matching.remote", ""}]) deletion_message = build_remote_deletion_message() @@ -501,7 +506,7 @@ test "it accepts deletions even from non-whitelisted servers" do end describe "when :reject_deletes has a matching host" do - setup do: clear_config([:mrf_simple, :reject_deletes], ["remote.instance"]) + setup do: clear_config([:mrf_simple, :reject_deletes], [{"remote.instance", ""}]) test "it rejects the deletion" do deletion_message = build_remote_deletion_message() @@ -511,7 +516,7 @@ test "it rejects the deletion" do end describe "when :reject_deletes match with wildcard domain" do - setup do: clear_config([:mrf_simple, :reject_deletes], ["*.remote.instance"]) + setup do: clear_config([:mrf_simple, :reject_deletes], [{"*.remote.instance", ""}]) test "it rejects the deletion" do deletion_message = build_remote_deletion_message() diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index ee6fdaae8..7731aadcc 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -154,15 +154,17 @@ test "it shows MRF transparency data if enabled", %{conn: conn} do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) clear_config([:mrf, :transparency], true) - simple_config = %{"reject" => ["example.com"]} + simple_config = %{"reject" => [{"example.com", ""}]} clear_config(:mrf_simple, simple_config) + expected_config = %{"reject" => [["example.com", ""]]} + response = conn |> get("/nodeinfo/2.1.json") |> json_response(:ok) - assert response["metadata"]["federation"]["mrf_simple"] == simple_config + assert response["metadata"]["federation"]["mrf_simple"] == expected_config end test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do @@ -170,10 +172,10 @@ test "it performs exclusions from MRF transparency data if configured", %{conn: clear_config([:mrf, :transparency], true) clear_config([:mrf, :transparency_exclusions], ["other.site"]) - simple_config = %{"reject" => ["example.com", "other.site"]} + simple_config = %{"reject" => [{"example.com", ""}, {"other.site", ""}]} clear_config(:mrf_simple, simple_config) - expected_config = %{"reject" => ["example.com"]} + expected_config = %{"reject" => [["example.com", ""]]} response = conn From dd947d9bc8320ec828df97353733bed4d4e2b5b8 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 25 Sep 2020 23:36:19 +0200 Subject: [PATCH 03/21] Add tests for setting `:instance, :quarantined_instances` No test was done for quarantined instances yet. I added a factory for followers_only notes and checked * That no followers only post is send when the target server is quarantined * That a followers only post is send when the target server is not quarantined --- .../web/activity_pub/publisher_test.exs | 74 +++++++++++++++++++ test/support/factory.ex | 32 ++++++++ 2 files changed, 106 insertions(+) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index 89f3ad411..d0bb43fb2 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -267,6 +267,80 @@ test "publish to url with with different ports" do end describe "publish/2" do + test_with_mock "doesn't publish a non-public activity to quarantined instances.", + Pleroma.Web.Federator.Publisher, + [:passthrough], + [] do + Config.put([:instance, :quarantined_instances], ["domain.com"]) + + follower = + insert(:user, %{ + local: false, + inbox: "https://domain.com/users/nick1/inbox", + ap_enabled: true + }) + + actor = insert(:user, follower_address: follower.ap_id) + + {:ok, follower, actor} = Pleroma.User.follow(follower, actor) + actor = refresh_record(actor) + + note_activity = + insert(:followers_only_note_activity, + user: actor, + recipients: [follower.ap_id] + ) + + res = Publisher.publish(actor, note_activity) + + assert res == :ok + + assert not called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain.com/users/nick1/inbox", + actor_id: actor.id, + id: note_activity.data["id"] + }) + ) + end + + test_with_mock "Publishes a non-public activity to non-quarantined instances.", + Pleroma.Web.Federator.Publisher, + [:passthrough], + [] do + Config.put([:instance, :quarantined_instances], ["somedomain.com"]) + + follower = + insert(:user, %{ + local: false, + inbox: "https://domain.com/users/nick1/inbox", + ap_enabled: true + }) + + actor = insert(:user, follower_address: follower.ap_id) + + {:ok, follower, actor} = Pleroma.User.follow(follower, actor) + actor = refresh_record(actor) + + note_activity = + insert(:followers_only_note_activity, + user: actor, + recipients: [follower.ap_id] + ) + + res = Publisher.publish(actor, note_activity) + + assert res == :ok + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain.com/users/nick1/inbox", + actor_id: actor.id, + id: note_activity.data["id"] + }) + ) + end + test_with_mock "publishes an activity with BCC to all relevant peers.", Pleroma.Web.Federator.Publisher, [:passthrough], diff --git a/test/support/factory.ex b/test/support/factory.ex index c267dba4e..f31f64a50 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -142,6 +142,11 @@ defp attachment_data(ap_id, href) do } end + def followers_only_note_factory(attrs \\ %{}) do + %Pleroma.Object{data: data} = note_factory(attrs) + %Pleroma.Object{data: Map.merge(data, %{"to" => [data["actor"] <> "/followers"]})} + end + def audio_factory(attrs \\ %{}) do text = sequence(:text, &"lain radio episode #{&1}") @@ -267,6 +272,33 @@ defp featured_collection_activity(attrs, type) do |> Map.merge(attrs) end + def followers_only_note_activity_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + note = insert(:followers_only_note, user: user) + + data_attrs = attrs[:data_attrs] || %{} + attrs = Map.drop(attrs, [:user, :note, :data_attrs]) + + data = + %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Create", + "actor" => note.data["actor"], + "to" => note.data["to"], + "object" => note.data, + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "context" => note.data["context"] + } + |> Map.merge(data_attrs) + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + |> Map.merge(attrs) + end + def note_activity_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) note = attrs[:note] || insert(:note, user: user) From 27fe7b0274cd9904d91167adade2cf7e56fd482b Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 2 Oct 2020 14:51:39 +0200 Subject: [PATCH 04/21] Make quarentine work with list of tuples instead of strings --- lib/pleroma/web/activity_pub/mrf.ex | 5 +++++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 4 +--- lib/pleroma/web/activity_pub/publisher.ex | 1 + test/pleroma/web/activity_pub/mrf_test.exs | 9 +++++++++ test/pleroma/web/activity_pub/publisher_test.exs | 4 ++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ac00fa54b..5ac4f9f20 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -100,6 +100,11 @@ def subdomain_match?(domains, host) do Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) end + @spec instance_list_from_tuples([{String.t(), String.t()}]) :: [String.t()] + def instance_list_from_tuples(list) do + Enum.map(list, fn {instance, _} -> instance end) + end + def describe(policies) do {:ok, policy_configs} = policies diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 8ef03aa3a..fe0dc874b 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -187,9 +187,7 @@ defp check_object(object), do: {:ok, object} defp instance_list(config_key) do Config.get([:mrf_simple, config_key]) - |> Enum.map(fn - {instance, _} -> instance - end) + |> MRF.instance_list_from_tuples() end @impl true diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 590beef64..4f29a4411 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -112,6 +112,7 @@ defp should_federate?(inbox, public) do quarantined_instances = Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 61d308b97..6ab27bc86 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -63,6 +63,15 @@ test "matches are case-insensitive" do end end + describe "instance_list_from_tuples/1" do + test "returns a list of instances from a list of {instance, reason} tuples" do + list = [{"some.tld", "a reason"}, {"other.tld", "another reason"}] + expected = ["some.tld", "other.tld"] + + assert MRF.instance_list_from_tuples(list) == expected + end + end + describe "describe/0" do test "it works as expected with noop policy" do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.NoOpPolicy]) diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index d0bb43fb2..b50e22bbe 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -271,7 +271,7 @@ test "publish to url with with different ports" do Pleroma.Web.Federator.Publisher, [:passthrough], [] do - Config.put([:instance, :quarantined_instances], ["domain.com"]) + Config.put([:instance, :quarantined_instances], [{"domain.com", "some reason"}]) follower = insert(:user, %{ @@ -308,7 +308,7 @@ test "publish to url with with different ports" do Pleroma.Web.Federator.Publisher, [:passthrough], [] do - Config.put([:instance, :quarantined_instances], ["somedomain.com"]) + Config.put([:instance, :quarantined_instances], [{"somedomain.com", "some reason"}]) follower = insert(:user, %{ From e0c7d7719797bad0edf7e5c5bd0d3c43cace6f36 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 2 Oct 2020 16:03:20 +0200 Subject: [PATCH 05/21] Deprecate and rewrite settings for quarentine settings * This is for the settings, not yet a DB migration --- lib/pleroma/config/deprecation_warnings.ex | 39 +++++++++++++++ .../config/deprecation_warnings_test.exs | 50 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index dd5c81094..37f783fec 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -80,6 +80,44 @@ def check_simple_policy_tuples do end end + def check_quarantined_instances_tuples do + has_strings = + Config.get([:instance, :quarantined_instances]) |> Enum.any?(fn e -> is_binary(e) end) + + if has_strings do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using strings in the quarantined_instances configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :instance, + quarantined_instances: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :instance, + quarantined_instances: [{"instance.tld", "Reason for quarantine"}] + ``` + """) + + new_config = + Config.get([:instance, :quarantined_instances]) + |> Enum.map(fn + {instance, reason} -> {instance, reason} + instance -> {instance, ""} + end) + + Config.put([:instance, :quarantined_instances], new_config) + + :error + else + :ok + end + end + def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do Logger.warn(""" @@ -103,6 +141,7 @@ def warn do :ok <- check_remote_ip_plug_name(), :ok <- check_uploders_s3_public_endpoint(), :ok <- check_old_chat_shoutbox(), + :ok <- check_quarantined_instances_tuples(), :ok <- check_simple_policy_tuples() do :ok else diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 1c686ec7c..61c835fc9 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -87,6 +87,56 @@ test "doesn't give a warning with correct config" do end end + describe "quarantined_instances tuples" do + test "gives warning when there are still strings" do + clear_config([:instance, :quarantined_instances], [ + {"domain.com", "some reason"}, + "somedomain.tld" + ]) + + assert capture_log(fn -> DeprecationWarnings.check_quarantined_instances_tuples() end) =~ + """ + !!!DEPRECATION WARNING!!! + Your config is using strings in the quarantined_instances configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :instance, + quarantined_instances: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :instance, + quarantined_instances: [{"instance.tld", "Reason for quarantine"}] + ``` + """ + end + + test "transforms config to tuples" do + clear_config([:instance, :quarantined_instances], [ + {"domain.com", "some reason"}, + "some.tld" + ]) + + expected_config = [{"domain.com", "some reason"}, {"some.tld", ""}] + + capture_log(fn -> DeprecationWarnings.check_quarantined_instances_tuples() end) + + assert Config.get([:instance, :quarantined_instances]) == expected_config + end + + test "doesn't give a warning with correct config" do + clear_config([:instance, :quarantined_instances], [ + {"domain.com", "some reason"}, + {"some.tld", ""} + ]) + + assert capture_log(fn -> DeprecationWarnings.check_quarantined_instances_tuples() end) == "" + end + end + test "check_old_mrf_config/0" do clear_config([:instance, :rewrite_policy], []) clear_config([:instance, :mrf_transparency], true) From dfeb3862da2ceaf63db300be1a916f5139250bc2 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 2 Oct 2020 19:08:04 +0200 Subject: [PATCH 06/21] config :mrf, :transparency_exclusions works with tumples now --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- test/pleroma/web/node_info_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index fe0dc874b..2f26fc3a0 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -258,7 +258,7 @@ def filter(object), do: {:ok, object} @impl true def describe do - exclusions = Config.get([:mrf, :transparency_exclusions]) + exclusions = Config.get([:mrf, :transparency_exclusions]) |> MRF.instance_list_from_tuples() mrf_simple = Config.get(:mrf_simple) diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index 7731aadcc..477c44a51 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -170,7 +170,7 @@ test "it shows MRF transparency data if enabled", %{conn: conn} do test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) clear_config([:mrf, :transparency], true) - clear_config([:mrf, :transparency_exclusions], ["other.site"]) + clear_config([:mrf, :transparency_exclusions], [{"other.site", "We don't want them to know"}]) simple_config = %{"reject" => [{"example.com", ""}, {"other.site", ""}]} clear_config(:mrf_simple, simple_config) From 3c5a497b19237c5e4f0f5d7aeb3fa1e43f13d932 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 2 Oct 2020 20:35:51 +0200 Subject: [PATCH 07/21] Deprecate transparency_exclusions * Give deprecation message * Rewrite configs --- lib/pleroma/config/deprecation_warnings.ex | 39 +++++++++++++++ .../config/deprecation_warnings_test.exs | 50 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 37f783fec..cc22b5d47 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -118,6 +118,44 @@ def check_quarantined_instances_tuples do end end + def check_transparency_exclusions_tuples do + has_strings = + Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(fn e -> is_binary(e) end) + + if has_strings do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using strings in the transparency_exclusions configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :mrf, + transparency_exclusions: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :mrf, + transparency_exclusions: [{"instance.tld", "Reason to exlude transparency"}] + ``` + """) + + new_config = + Config.get([:mrf, :transparency_exclusions]) + |> Enum.map(fn + {instance, reason} -> {instance, reason} + instance -> {instance, ""} + end) + + Config.put([:mrf, :transparency_exclusions], new_config) + + :error + else + :ok + end + end + def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do Logger.warn(""" @@ -142,6 +180,7 @@ def warn do :ok <- check_uploders_s3_public_endpoint(), :ok <- check_old_chat_shoutbox(), :ok <- check_quarantined_instances_tuples(), + :ok <- check_transparency_exclusions_tuples(), :ok <- check_simple_policy_tuples() do :ok else diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 61c835fc9..1037c4d35 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -137,6 +137,56 @@ test "doesn't give a warning with correct config" do end end + describe "transparency_exclusions tuples" do + test "gives warning when there are still strings" do + clear_config([:mrf, :transparency_exclusions], [ + {"domain.com", "some reason"}, + "somedomain.tld" + ]) + + assert capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) =~ + """ + !!!DEPRECATION WARNING!!! + Your config is using strings in the transparency_exclusions configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: + + ``` + config :pleroma, :mrf, + transparency_exclusions: ["instance.tld"] + ``` + + Is now + + + ``` + config :pleroma, :mrf, + transparency_exclusions: [{"instance.tld", "Reason to exlude transparency"}] + ``` + """ + end + + test "transforms config to tuples" do + clear_config([:mrf, :transparency_exclusions], [ + {"domain.com", "some reason"}, + "some.tld" + ]) + + expected_config = [{"domain.com", "some reason"}, {"some.tld", ""}] + + capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) + + assert Config.get([:mrf, :transparency_exclusions]) == expected_config + end + + test "doesn't give a warning with correct config" do + clear_config([:mrf, :transparency_exclusions], [ + {"domain.com", "some reason"}, + {"some.tld", ""} + ]) + + assert capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) == "" + end + end + test "check_old_mrf_config/0" do clear_config([:instance, :rewrite_policy], []) clear_config([:instance, :mrf_transparency], true) From b674ba658b8adad42b96921a4afb031b8323f8a1 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sat, 3 Oct 2020 11:55:16 +0200 Subject: [PATCH 08/21] make linter happy --- test/pleroma/config/deprecation_warnings_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 1037c4d35..84459de8f 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -183,7 +183,8 @@ test "doesn't give a warning with correct config" do {"some.tld", ""} ]) - assert capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) == "" + assert capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) == + "" end end From 64002e92adcf08564d9b8e3fa2dcf7c07c72145a Mon Sep 17 00:00:00 2001 From: Ilja Date: Sat, 3 Oct 2020 12:08:09 +0200 Subject: [PATCH 09/21] config/description.exs: Update quarantine settings to tuples --- config/description.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/description.exs b/config/description.exs index 934a62a62..252aa63d8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -687,12 +687,12 @@ }, %{ key: :quarantined_instances, - type: {:list, :string}, + type: {:list, :tuple}, description: - "List of ActivityPub instances where private (DMs, followers-only) activities will not be sent", + "List of ActivityPub instances where private (DMs, followers-only) activities will not be sent and the reason for doing so", suggestions: [ - "quarantined.com", - "*.quarantined.com" + {"quarantined.com", "Reason"}, + {"*.quarantined.com", "Reason"} ] }, %{ From c0489f9fac78701345c2902fa59bd99381bc27ab Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 5 Oct 2020 11:26:08 +0200 Subject: [PATCH 10/21] Fixed deprecation warning checks When a setting was deprecated, the code would stop checking for the rest of the possible deprications. This also meant that the settings weren't rewritten to the new settings for deprecated settings besides the first one. --- lib/pleroma/config/deprecation_warnings.ex | 35 ++++++++++--------- .../config/deprecation_warnings_test.exs | 6 ++-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index cc22b5d47..887470de9 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -170,23 +170,24 @@ def check_hellthread_threshold do end def warn do - with :ok <- check_hellthread_threshold(), - :ok <- check_old_mrf_config(), - :ok <- check_media_proxy_whitelist_config(), - :ok <- check_welcome_message_config(), - :ok <- check_gun_pool_options(), - :ok <- check_activity_expiration_config(), - :ok <- check_remote_ip_plug_name(), - :ok <- check_uploders_s3_public_endpoint(), - :ok <- check_old_chat_shoutbox(), - :ok <- check_quarantined_instances_tuples(), - :ok <- check_transparency_exclusions_tuples(), - :ok <- check_simple_policy_tuples() do - :ok - else - _ -> - :error - end + [ + check_hellthread_threshold(), + check_old_mrf_config(), + check_media_proxy_whitelist_config(), + check_welcome_message_config(), + check_gun_pool_options(), + check_activity_expiration_config(), + check_remote_ip_plug_name(), + check_uploders_s3_public_endpoint(), + check_old_chat_shoutbox(), + check_quarantined_instances_tuples(), + check_transparency_exclusions_tuples(), + check_simple_policy_tuples() + ] + |> Enum.reduce(:ok, fn + :ok, :ok -> :ok + _, _ -> :error + end) end def check_welcome_message_config do diff --git a/test/pleroma/config/deprecation_warnings_test.exs b/test/pleroma/config/deprecation_warnings_test.exs index 84459de8f..c5e2b20f4 100644 --- a/test/pleroma/config/deprecation_warnings_test.exs +++ b/test/pleroma/config/deprecation_warnings_test.exs @@ -73,7 +73,7 @@ test "transforms config to tuples" do {:media_removal, [{"some.removal", ""}, {"some.other.instance", "Some reason"}]} ] - capture_log(fn -> DeprecationWarnings.check_simple_policy_tuples() end) + capture_log(fn -> DeprecationWarnings.warn() end) assert Config.get([:mrf_simple]) == expected_config end @@ -122,7 +122,7 @@ test "transforms config to tuples" do expected_config = [{"domain.com", "some reason"}, {"some.tld", ""}] - capture_log(fn -> DeprecationWarnings.check_quarantined_instances_tuples() end) + capture_log(fn -> DeprecationWarnings.warn() end) assert Config.get([:instance, :quarantined_instances]) == expected_config end @@ -172,7 +172,7 @@ test "transforms config to tuples" do expected_config = [{"domain.com", "some reason"}, {"some.tld", ""}] - capture_log(fn -> DeprecationWarnings.check_transparency_exclusions_tuples() end) + capture_log(fn -> DeprecationWarnings.warn() end) assert Config.get([:mrf, :transparency_exclusions]) == expected_config end From 1f52246a026a81f216ebf761b2bda5a2035abbbc Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 5 Oct 2020 14:13:11 +0200 Subject: [PATCH 11/21] Add database migrations * SimplePolicy * quarentine * transparency_exclusions --- ...05123100_simple_policy_string_to_tuple.exs | 40 ++++++++++++ ...00_quarantained_policy_string_to_tuple.exs | 61 +++++++++++++++++++ ...ransparency_exclusions_string_to_tuple.exs | 61 +++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 priv/repo/migrations/20201005123100_simple_policy_string_to_tuple.exs create mode 100644 priv/repo/migrations/20201005124600_quarantained_policy_string_to_tuple.exs create mode 100644 priv/repo/migrations/20201005132900_transparency_exclusions_string_to_tuple.exs diff --git a/priv/repo/migrations/20201005123100_simple_policy_string_to_tuple.exs b/priv/repo/migrations/20201005123100_simple_policy_string_to_tuple.exs new file mode 100644 index 000000000..77a4a7311 --- /dev/null +++ b/priv/repo/migrations/20201005123100_simple_policy_string_to_tuple.exs @@ -0,0 +1,40 @@ +defmodule Pleroma.Repo.Migrations.SimplePolicyStringToTuple do + use Ecto.Migration + + alias Pleroma.ConfigDB + + def up, do: ConfigDB.get_by_params(%{group: :pleroma, key: :mrf_simple}) |> update_to_tuples + def down, do: ConfigDB.get_by_params(%{group: :pleroma, key: :mrf_simple}) |> update_to_strings + + defp update_to_tuples(%{value: value}) do + new_value = + value + |> Enum.map(fn {k, v} -> + {k, + Enum.map(v, fn + {instance, reason} -> {instance, reason} + instance -> {instance, ""} + end)} + end) + + ConfigDB.update_or_create(%{group: :pleroma, key: :mrf_simple, value: new_value}) + end + + defp update_to_tuples(nil), do: {:ok, nil} + + defp update_to_strings(%{value: value}) do + new_value = + value + |> Enum.map(fn {k, v} -> + {k, + Enum.map(v, fn + {instance, _} -> instance + instance -> instance + end)} + end) + + ConfigDB.update_or_create(%{group: :pleroma, key: :mrf_simple, value: new_value}) + end + + defp update_to_strings(nil), do: {:ok, nil} +end diff --git a/priv/repo/migrations/20201005124600_quarantained_policy_string_to_tuple.exs b/priv/repo/migrations/20201005124600_quarantained_policy_string_to_tuple.exs new file mode 100644 index 000000000..b924e4638 --- /dev/null +++ b/priv/repo/migrations/20201005124600_quarantained_policy_string_to_tuple.exs @@ -0,0 +1,61 @@ +defmodule Pleroma.Repo.Migrations.QuarantainedStringToTuple do + use Ecto.Migration + + alias Pleroma.ConfigDB + + def up, + do: + ConfigDB.get_by_params(%{group: :pleroma, key: :instance}) + |> update_quarantined_instances_to_tuples + + def down, + do: + ConfigDB.get_by_params(%{group: :pleroma, key: :instance}) + |> update_quarantined_instances_to_strings + + defp update_quarantined_instances_to_tuples(%{value: settings}) do + settings |> List.keyfind(:quarantined_instances, 0) |> update_to_tuples + end + + defp update_quarantined_instances_to_tuples(nil), do: {:ok, nil} + + defp update_to_tuples({:quarantined_instances, instance_list}) do + new_value = + instance_list + |> Enum.map(fn + {v, r} -> {v, r} + v -> {v, ""} + end) + + ConfigDB.update_or_create(%{ + group: :pleroma, + key: :instance, + value: [quarantined_instances: new_value] + }) + end + + defp update_to_tuples(nil), do: {:ok, nil} + + defp update_quarantined_instances_to_strings(%{value: settings}) do + settings |> List.keyfind(:quarantined_instances, 0) |> update_to_strings + end + + defp update_quarantined_instances_to_strings(nil), do: {:ok, nil} + + defp update_to_strings({:quarantined_instances, instance_list}) do + new_value = + instance_list + |> Enum.map(fn + {v, _} -> v + v -> v + end) + + ConfigDB.update_or_create(%{ + group: :pleroma, + key: :instance, + value: [quarantined_instances: new_value] + }) + end + + defp update_to_strings(nil), do: {:ok, nil} +end diff --git a/priv/repo/migrations/20201005132900_transparency_exclusions_string_to_tuple.exs b/priv/repo/migrations/20201005132900_transparency_exclusions_string_to_tuple.exs new file mode 100644 index 000000000..6516083a7 --- /dev/null +++ b/priv/repo/migrations/20201005132900_transparency_exclusions_string_to_tuple.exs @@ -0,0 +1,61 @@ +defmodule Pleroma.Repo.Migrations.TransparencyExclusionsStringToTuple do + use Ecto.Migration + + alias Pleroma.ConfigDB + + def up, + do: + ConfigDB.get_by_params(%{group: :pleroma, key: :mrf}) + |> update_transparency_exclusions_instances_to_tuples + + def down, + do: + ConfigDB.get_by_params(%{group: :pleroma, key: :mrf}) + |> update_transparency_exclusions_instances_to_strings + + defp update_transparency_exclusions_instances_to_tuples(%{value: settings}) do + settings |> List.keyfind(:transparency_exclusions, 0) |> update_to_tuples + end + + defp update_transparency_exclusions_instances_to_tuples(nil), do: {:ok, nil} + + defp update_to_tuples({:transparency_exclusions, instance_list}) do + new_value = + instance_list + |> Enum.map(fn + {v, r} -> {v, r} + v -> {v, ""} + end) + + ConfigDB.update_or_create(%{ + group: :pleroma, + key: :mrf, + value: [transparency_exclusions: new_value] + }) + end + + defp update_to_tuples(nil), do: {:ok, nil} + + defp update_transparency_exclusions_instances_to_strings(%{value: settings}) do + settings |> List.keyfind(:transparency_exclusions, 0) |> update_to_strings + end + + defp update_transparency_exclusions_instances_to_strings(nil), do: {:ok, nil} + + defp update_to_strings({:transparency_exclusions, instance_list}) do + new_value = + instance_list + |> Enum.map(fn + {v, _} -> v + v -> v + end) + + ConfigDB.update_or_create(%{ + group: :pleroma, + key: :mrf, + value: [transparency_exclusions: new_value] + }) + end + + defp update_to_strings(nil), do: {:ok, nil} +end From 7fdc3cde065ce20257e0e03e416ca18775b18943 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 23 Oct 2020 20:27:13 +0200 Subject: [PATCH 12/21] Return maps in node_info It's easiest (and imo most proper) to use tuples {"instance, "reason"} in BE, but for FE maps like %{"instance": "instance", "reason", "reason"} are better. I changed it so that node_info returns maps now for simple_policy and quarantined instances. --- .../web/activity_pub/mrf/simple_policy.ex | 3 +++ .../web/mastodon_api/views/instance_view.ex | 6 +++++- test/pleroma/web/node_info_test.exs | 20 +++++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2f26fc3a0..0853a05a7 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -263,6 +263,9 @@ def describe do mrf_simple = Config.get(:mrf_simple) |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {v, _} -> v in exclusions end)} end) + |> Enum.map(fn {k, v} -> + {k, Enum.map(v, fn {i, r} -> %{"instance" => i, "reason" => r} end)} + end) |> Enum.into(%{}) {:ok, %{mrf_simple: mrf_simple}} diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 3528185d5..db40ea3fa 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -95,7 +95,11 @@ def federation do {:ok, data} = MRF.describe() data - |> Map.merge(%{quarantined_instances: quarantined}) + |> Map.merge(%{ + quarantined_instances: + quarantined + |> Enum.map(fn {instance, reason} -> %{"instance" => instance, "reason" => reason} end) + }) else %{} end diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index 477c44a51..cb910d668 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -150,6 +150,22 @@ test "it shows default features flags", %{conn: conn} do ) end + test "it shows quarantined instances data if enabled", %{conn: conn} do + clear_config([:mrf, :transparency], true) + + quarantined_instances = [{"example.com", ""}] + clear_config([:instance, :quarantined_instances], quarantined_instances) + + expected_config = [%{"instance" => "example.com", "reason" => ""}] + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["quarantined_instances"] == expected_config + end + test "it shows MRF transparency data if enabled", %{conn: conn} do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) clear_config([:mrf, :transparency], true) @@ -157,7 +173,7 @@ test "it shows MRF transparency data if enabled", %{conn: conn} do simple_config = %{"reject" => [{"example.com", ""}]} clear_config(:mrf_simple, simple_config) - expected_config = %{"reject" => [["example.com", ""]]} + expected_config = %{"reject" => [%{"instance" => "example.com", "reason" => ""}]} response = conn @@ -175,7 +191,7 @@ test "it performs exclusions from MRF transparency data if configured", %{conn: simple_config = %{"reject" => [{"example.com", ""}, {"other.site", ""}]} clear_config(:mrf_simple, simple_config) - expected_config = %{"reject" => [["example.com", ""]]} + expected_config = %{"reject" => [%{"instance" => "example.com", "reason" => ""}]} response = conn From 47fc57bbccbe5df32ef00dda0ee8bdd56b38885f Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 20 Nov 2020 13:48:28 +0100 Subject: [PATCH 13/21] Change what nodeinfo returns without breaking backwards compatibility * Only for SimplePolicy for now * I added an extra mrf_simple_info key that has an object as value. The object contains only relevant extra info --- .../web/activity_pub/mrf/simple_policy.ex | 18 +++- test/pleroma/web/node_info_test.exs | 100 +++++++++++++----- 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 0853a05a7..22999ef58 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -260,15 +260,27 @@ def filter(object), do: {:ok, object} def describe do exclusions = Config.get([:mrf, :transparency_exclusions]) |> MRF.instance_list_from_tuples() - mrf_simple = + mrf_simple_excluded = Config.get(:mrf_simple) |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {v, _} -> v in exclusions end)} end) + + mrf_simple = + mrf_simple_excluded |> Enum.map(fn {k, v} -> - {k, Enum.map(v, fn {i, r} -> %{"instance" => i, "reason" => r} end)} + {k, Enum.map(v, fn {instance, _} -> instance end)} end) |> Enum.into(%{}) - {:ok, %{mrf_simple: mrf_simple}} + mrf_simple_info = + mrf_simple_excluded + |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {_, reason} -> reason == "" end)} end) + |> Enum.reject(fn {_, v} -> v == [] end) + |> Enum.map(fn {k, l} -> + {k, l |> Enum.map(fn {i, r} -> {i, %{"reason" => r}} end) |> Enum.into(%{})} + end) + |> Enum.into(%{}) + + {:ok, %{mrf_simple: mrf_simple, mrf_simple_info: mrf_simple_info}} end @impl true diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index cb910d668..eb6b8bf6c 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -166,39 +166,89 @@ test "it shows quarantined instances data if enabled", %{conn: conn} do assert response["metadata"]["federation"]["quarantined_instances"] == expected_config end - test "it shows MRF transparency data if enabled", %{conn: conn} do - clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) - clear_config([:mrf, :transparency], true) + describe "MRF SimplePolicy" do + setup do + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + clear_config([:mrf, :transparency], true) + end - simple_config = %{"reject" => [{"example.com", ""}]} - clear_config(:mrf_simple, simple_config) + test "shows MRF transparency data if enabled", %{conn: conn} do + simple_config = %{"reject" => [{"example.com", ""}]} + clear_config(:mrf_simple, simple_config) - expected_config = %{"reject" => [%{"instance" => "example.com", "reason" => ""}]} + expected_config = %{"reject" => ["example.com"]} - response = - conn - |> get("/nodeinfo/2.1.json") - |> json_response(:ok) + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) - assert response["metadata"]["federation"]["mrf_simple"] == expected_config - end + assert response["metadata"]["federation"]["mrf_simple"] == expected_config + end - test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do - clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) - clear_config([:mrf, :transparency], true) - clear_config([:mrf, :transparency_exclusions], [{"other.site", "We don't want them to know"}]) + test "performs exclusions from MRF transparency data if configured", %{conn: conn} do + clear_config([:mrf, :transparency_exclusions], [ + {"other.site", "We don't want them to know"} + ]) - simple_config = %{"reject" => [{"example.com", ""}, {"other.site", ""}]} - clear_config(:mrf_simple, simple_config) + simple_config = %{"reject" => [{"example.com", ""}, {"other.site", ""}]} + clear_config(:mrf_simple, simple_config) - expected_config = %{"reject" => [%{"instance" => "example.com", "reason" => ""}]} + expected_config = %{"reject" => ["example.com"]} - response = - conn - |> get("/nodeinfo/2.1.json") - |> json_response(:ok) + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) - assert response["metadata"]["federation"]["mrf_simple"] == expected_config - assert response["metadata"]["federation"]["exclusions"] == true + assert response["metadata"]["federation"]["mrf_simple"] == expected_config + assert response["metadata"]["federation"]["exclusions"] == true + end + + test "shows extra information in the mrf_simple_extra field for relevant entries", %{ + conn: conn + } do + simple_config = %{ + media_removal: [{"no.media", "LEEWWWDD >//<"}], + media_nsfw: [], + federated_timeline_removal: [{"no.ftl", ""}], + report_removal: [], + reject: [ + {"example.instance", "Some reason"}, + {"uwu.owo", "awoo to much"}, + {"no.reason", ""} + ], + followers_only: [], + accept: [], + avatar_removal: [], + banner_removal: [], + reject_deletes: [ + {"peak.me", "I want to peak at what they don't want me to see, eheh"} + ] + } + + clear_config(:mrf_simple, simple_config) + + clear_config([:mrf, :transparency_exclusions], [ + {"peak.me", "I don't want them to know"} + ]) + + expected_config = %{ + "media_removal" => %{ + "no.media" => %{"reason" => "LEEWWWDD >//<"} + }, + "reject" => %{ + "example.instance" => %{"reason" => "Some reason"}, + "uwu.owo" => %{"reason" => "awoo to much"} + } + } + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["mrf_simple_info"] == expected_config + end end end From 03030b47c22f4a193e7ddc582574d4a521854025 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sat, 28 Nov 2020 10:34:31 +0100 Subject: [PATCH 14/21] quarantine instances info Added a new field in the nodeinfo called quarantined_instances_info This holds an object like `"quarantined_instances_info":{"quarantined_instances":{"quar.inst":{"reason":"whatever reason"}}}}` --- .../web/mastodon_api/views/instance_view.ex | 10 ++++- test/pleroma/web/node_info_test.exs | 44 ++++++++++++++----- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index db40ea3fa..a341ca6f4 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -98,7 +98,15 @@ def federation do |> Map.merge(%{ quarantined_instances: quarantined - |> Enum.map(fn {instance, reason} -> %{"instance" => instance, "reason" => reason} end) + |> Enum.map(fn {instance, _reason} -> instance end) + }) + |> Map.merge(%{ + quarantined_instances_info: %{ + "quarantined_instances" => + quarantined + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Enum.into(%{}) + } }) else %{} diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index eb6b8bf6c..9deceb1b5 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -150,20 +150,42 @@ test "it shows default features flags", %{conn: conn} do ) end - test "it shows quarantined instances data if enabled", %{conn: conn} do - clear_config([:mrf, :transparency], true) + describe "Quarantined instances" do + setup do + clear_config([:mrf, :transparency], true) + quarantined_instances = [{"example.com", "reason to quarantine"}] + clear_config([:instance, :quarantined_instances], quarantined_instances) + end - quarantined_instances = [{"example.com", ""}] - clear_config([:instance, :quarantined_instances], quarantined_instances) + test "shows quarantined instances data if enabled", %{conn: conn} do + expected_config = ["example.com"] - expected_config = [%{"instance" => "example.com", "reason" => ""}] + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) - response = - conn - |> get("/nodeinfo/2.1.json") - |> json_response(:ok) + assert response["metadata"]["federation"]["quarantined_instances"] == expected_config + end - assert response["metadata"]["federation"]["quarantined_instances"] == expected_config + test "shows extra information in the quarantined_info field for relevant entries", %{ + conn: conn + } do + clear_config([:mrf, :transparency], true) + + expected_config = %{ + "quarantined_instances" => %{ + "example.com" => %{"reason" => "reason to quarantine"} + } + } + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["quarantined_instances_info"] == expected_config + end end describe "MRF SimplePolicy" do @@ -205,7 +227,7 @@ test "performs exclusions from MRF transparency data if configured", %{conn: con assert response["metadata"]["federation"]["exclusions"] == true end - test "shows extra information in the mrf_simple_extra field for relevant entries", %{ + test "shows extra information in the mrf_simple_info field for relevant entries", %{ conn: conn } do simple_config = %{ From 506bf16363649ac1b91a1796399eddb88ed50371 Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 14 Dec 2020 13:11:51 +0100 Subject: [PATCH 15/21] Change docs * ./configuration/mrf.md * Change example * ./configuration/cheatsheet.md * Change descriptions to include that a reason is given * CHANGELOG.md * Add as breaking change --- CHANGELOG.md | 1 + docs/configuration/cheatsheet.md | 21 +++++++++++---------- docs/configuration/mrf.md | 12 ++++++------ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc536c55..d26a83caf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Configuration: `:chat, enabled` moved to `:shout, enabled` and `:instance, chat_limit` moved to `:shout, limit` +- **Breaking** Besides only the instance, entries for SimplePolicy and QuarantinedInstances now contain a reason as well. - Support for Erlang/OTP 24 - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. - HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 5b49185dc..d3c9c5716 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -39,7 +39,7 @@ To add configuration to your config file, you can copy it from the base config. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. -* `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send. +* `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send. * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with older software for theses nicknames. @@ -135,15 +135,16 @@ To add configuration to your config file, you can copy it from the base config. Configuring MRF policies is not enough for them to take effect. You have to enable them by specifying their module in `policies` under [:mrf](#mrf) section. #### :mrf_simple -* `media_removal`: List of instances to remove media from. -* `media_nsfw`: List of instances to put media as NSFW(sensitive) from. -* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline. -* `reject`: List of instances to reject any activities from. -* `accept`: List of instances to accept any activities from. -* `followers_only`: List of instances to decrease post visibility to only the followers, including for DM mentions. -* `report_removal`: List of instances to reject reports from. -* `avatar_removal`: List of instances to strip avatars from. -* `banner_removal`: List of instances to strip banners from. +* `media_removal`: List of instances to strip media attachments from and the reason for doing so. +* `media_nsfw`: List of instances to tag all media as NSFW (sensitive) from and the reason for doing so. +* `federated_timeline_removal`: List of instances to remove from the Federated Timeline (aka The Whole Known Network) and the reason for doing so. +* `reject`: List of instances to reject activities (except deletes) from and the reason for doing so. +* `accept`: List of instances to only accept activities (except deletes) from and the reason for doing so. +* `followers_only`: Force posts from the given instances to be visible by followers only and the reason for doing so. +* `report_removal`: List of instances to reject reports from and the reason for doing so. +* `avatar_removal`: List of instances to strip avatars from and the reason for doing so. +* `banner_removal`: List of instances to strip banners from and the reason for doing so. +* `reject_deletes`: List of instances to reject deletions from and the reason for doing so. #### :mrf_subchain This policy processes messages through an alternate pipeline when a given message matches certain criteria. diff --git a/docs/configuration/mrf.md b/docs/configuration/mrf.md index 5618634a2..a31c26b9c 100644 --- a/docs/configuration/mrf.md +++ b/docs/configuration/mrf.md @@ -55,18 +55,18 @@ Servers should be configured as lists. ### Example -This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`: +This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`. We also give a reason why the moderation was done: ```elixir config :pleroma, :mrf, policies: [Pleroma.Web.ActivityPub.MRF.SimplePolicy] config :pleroma, :mrf_simple, - media_removal: ["illegalporn.biz"], - media_nsfw: ["porn.biz", "porn.business"], - reject: ["spam.com"], - federated_timeline_removal: ["spam.university"], - report_removal: ["whiny.whiner"] + media_removal: [{"illegalporn.biz", "Media can contain illegal contant"}], + media_nsfw: [{"porn.biz", "unmarked nsfw media"}, {"porn.business", "A lot of unmarked nsfw media"}], + reject: [{"spam.com", "They keep spamming our users"}], + federated_timeline_removal: [{"spam.university", "Annoying low-quality posts who otherwise fill up TWKN"}], + report_removal: [{"whiny.whiner", "Keep spamming us with irrelevant reports"}] ``` ### Use with Care From 9418424048283864bd46c2f0dae769d016895220 Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 14 Dec 2020 13:18:15 +0100 Subject: [PATCH 16/21] Add transparency_exclusions also to the breaking changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d26a83caf..76a99e5a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Configuration: `:chat, enabled` moved to `:shout, enabled` and `:instance, chat_limit` moved to `:shout, limit` -- **Breaking** Besides only the instance, entries for SimplePolicy and QuarantinedInstances now contain a reason as well. +- **Breaking** Besides only the instance, entries for `simple_policy`, `transparency_exclusions` and `quarantined_instances` now contain a reason as well. - Support for Erlang/OTP 24 - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. - HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising. From f4028c908c56736b88caa5edb71f5aad21244de1 Mon Sep 17 00:00:00 2001 From: Ilja Date: Fri, 18 Dec 2020 14:29:38 +0100 Subject: [PATCH 17/21] Add key- and valuePlaceholders for quarantined_instances and mrf_simple * I also added for keywordpolicy as well now. It was done in the admin-fe, but is better to be done here * I also added comments to explain why we did the _info keys (backwards compatibility) --- config/description.exs | 2 ++ lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 2 ++ lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 4 ++++ lib/pleroma/web/mastodon_api/views/instance_view.ex | 4 ++++ 4 files changed, 12 insertions(+) diff --git a/config/description.exs b/config/description.exs index 252aa63d8..411029c27 100644 --- a/config/description.exs +++ b/config/description.exs @@ -688,6 +688,8 @@ %{ key: :quarantined_instances, type: {:list, :tuple}, + keyPlaceholder: "instance", + valuePlaceholder: "reason", description: "List of ActivityPub instances where private (DMs, followers-only) activities will not be sent and the reason for doing so", suggestions: [ diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 646008dd9..aebf78b46 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -159,6 +159,8 @@ def config_description do %{ key: :replace, type: {:list, :tuple}, + keyPlaceholder: "pattern", + valuePlaceholder: "replacement", description: """ **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 22999ef58..00a74ead8 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -271,6 +271,10 @@ def describe do end) |> Enum.into(%{}) + # This is for backwards compatibility. We originally didn't sent + # extra info like a reason why an instance was rejected/quarantined/etc. + # Because we didn't want to break backwards compatibility it was decided + # to add an extra "info" key. mrf_simple_info = mrf_simple_excluded |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {_, reason} -> reason == "" end)} end) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index a341ca6f4..f62b52a64 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -100,6 +100,10 @@ def federation do quarantined |> Enum.map(fn {instance, _reason} -> instance end) }) + # This is for backwards compatibility. We originally didn't sent + # extra info like a reason why an instance was rejected/quarantined/etc. + # Because we didn't want to break backwards compatibility it was decided + # to add an extra "info" key. |> Map.merge(%{ quarantined_instances_info: %{ "quarantined_instances" => From b0926a71b288249f1e318493f3f67674ddb8e12c Mon Sep 17 00:00:00 2001 From: Ilja Date: Sat, 26 Dec 2020 11:35:05 +0100 Subject: [PATCH 18/21] Make transparency_exclusions use tuples in admin-fe * Make it use tuples * I also changed the keys for key_placeholder and value_placeholder to use snake_case instead of camelCase --- lib/pleroma/web/activity_pub/mrf.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 5ac4f9f20..23ea039c3 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -33,9 +33,11 @@ defmodule Pleroma.Web.ActivityPub.MRF do %{ key: :transparency_exclusions, label: "MRF transparency exclusions", - type: {:list, :string}, + type: {:list, :tuple}, + key_placeholder: "instance", + value_placeholder: "reason", description: - "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.", + "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. You can also provide a reason for excluding these instance names. The instances and reasons won't be publicly disclosed.", suggestions: [ "exclusion.com" ] diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index aebf78b46..1383fa757 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -159,8 +159,8 @@ def config_description do %{ key: :replace, type: {:list, :tuple}, - keyPlaceholder: "pattern", - valuePlaceholder: "replacement", + key_placeholder: "instance", + value_placeholder: "reason", description: """ **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. From cd706c033588d290d7726155b1b346adcf858fb1 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sun, 17 Jan 2021 00:06:04 +0100 Subject: [PATCH 19/21] improve changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a99e5a6..7a580d7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Configuration: `:chat, enabled` moved to `:shout, enabled` and `:instance, chat_limit` moved to `:shout, limit` -- **Breaking** Besides only the instance, entries for `simple_policy`, `transparency_exclusions` and `quarantined_instances` now contain a reason as well. +- **Breaking** Entries for simple_policy, transparency_exclusions and quarantined_instances now list both the instance and a reason. - Support for Erlang/OTP 24 - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. - HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising. From ee26f2c91b6335a086949334138877432944b208 Mon Sep 17 00:00:00 2001 From: Ilja Date: Sun, 17 Jan 2021 14:32:42 +0100 Subject: [PATCH 20/21] Quarantine placeholders * kePlaceholder and valuePlaceholder of quarantined_instances where in wrong case, should be snake_case * The mrf simple and transparency exclusion were already OK --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 411029c27..c72231faa 100644 --- a/config/description.exs +++ b/config/description.exs @@ -688,8 +688,8 @@ %{ key: :quarantined_instances, type: {:list, :tuple}, - keyPlaceholder: "instance", - valuePlaceholder: "reason", + key_placeholder: "instance", + value_placeholder: "reason", description: "List of ActivityPub instances where private (DMs, followers-only) activities will not be sent and the reason for doing so", suggestions: [ From ad09bdb3764e529d1f0682b5395f34bc4849bd22 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 4 Feb 2021 17:23:21 +0400 Subject: [PATCH 21/21] Improve readability --- lib/pleroma/config/deprecation_warnings.ex | 8 +++--- .../web/activity_pub/mrf/reject_non_public.ex | 2 +- .../web/activity_pub/mrf/simple_policy.ex | 27 ++++++++++++------- .../mrf/user_allow_list_policy.ex | 2 +- .../web/activity_pub/mrf/vocabulary_policy.ex | 2 +- .../web/mastodon_api/views/instance_view.ex | 21 +++++++-------- 6 files changed, 33 insertions(+), 29 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 887470de9..029ee8b65 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Config.DeprecationWarnings do def check_simple_policy_tuples do has_strings = Config.get([:mrf_simple]) - |> Enum.any?(fn {_, v} -> Enum.any?(v, fn e -> is_binary(e) end) end) + |> Enum.any?(fn {_, v} -> Enum.any?(v, &is_binary/1) end) if has_strings do Logger.warn(""" @@ -81,8 +81,7 @@ def check_simple_policy_tuples do end def check_quarantined_instances_tuples do - has_strings = - Config.get([:instance, :quarantined_instances]) |> Enum.any?(fn e -> is_binary(e) end) + has_strings = Config.get([:instance, :quarantined_instances]) |> Enum.any?(&is_binary/1) if has_strings do Logger.warn(""" @@ -119,8 +118,7 @@ def check_quarantined_instances_tuples do end def check_transparency_exclusions_tuples do - has_strings = - Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(fn e -> is_binary(e) end) + has_strings = Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(&is_binary/1) if has_strings do Logger.warn(""" diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index b9d3e52c7..dbb7ca0df 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -47,7 +47,7 @@ def filter(object), do: {:ok, object} @impl true def describe, - do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} + do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Map.new()}} @impl true def config_description do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 00a74ead8..c631cc85f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -262,14 +262,16 @@ def describe do mrf_simple_excluded = Config.get(:mrf_simple) - |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {v, _} -> v in exclusions end)} end) + |> Enum.map(fn {rule, instances} -> + {rule, Enum.reject(instances, fn {host, _} -> host in exclusions end)} + end) mrf_simple = mrf_simple_excluded - |> Enum.map(fn {k, v} -> - {k, Enum.map(v, fn {instance, _} -> instance end)} + |> Enum.map(fn {rule, instances} -> + {rule, Enum.map(instances, fn {host, _} -> host end)} end) - |> Enum.into(%{}) + |> Map.new() # This is for backwards compatibility. We originally didn't sent # extra info like a reason why an instance was rejected/quarantined/etc. @@ -277,12 +279,19 @@ def describe do # to add an extra "info" key. mrf_simple_info = mrf_simple_excluded - |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn {_, reason} -> reason == "" end)} end) - |> Enum.reject(fn {_, v} -> v == [] end) - |> Enum.map(fn {k, l} -> - {k, l |> Enum.map(fn {i, r} -> {i, %{"reason" => r}} end) |> Enum.into(%{})} + |> Enum.map(fn {rule, instances} -> + {rule, Enum.reject(instances, fn {_, reason} -> reason == "" end)} end) - |> Enum.into(%{}) + |> Enum.reject(fn {_, instances} -> instances == [] end) + |> Enum.map(fn {rule, instances} -> + instances = + instances + |> Enum.map(fn {host, reason} -> {host, %{"reason" => reason}} end) + |> Map.new() + + {rule, instances} + end) + |> Map.new() {:ok, %{mrf_simple: mrf_simple, mrf_simple_info: mrf_simple_info}} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 1bcb3688b..52fb02a84 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -37,7 +37,7 @@ def filter(object), do: {:ok, object} def describe do mrf_user_allowlist = Config.get([:mrf_user_allowlist], []) - |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) + |> Map.new(fn {k, v} -> {k, length(v)} end) {:ok, %{mrf_user_allowlist: mrf_user_allowlist}} end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 20f57f609..602e10b44 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -39,7 +39,7 @@ def filter(message), do: {:ok, message} @impl true def describe, - do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Enum.into(%{})}} + do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Map.new()}} @impl true def config_description do diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f62b52a64..ef208062b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -95,22 +95,19 @@ def federation do {:ok, data} = MRF.describe() data - |> Map.merge(%{ - quarantined_instances: - quarantined - |> Enum.map(fn {instance, _reason} -> instance end) - }) + |> Map.put( + :quarantined_instances, + Enum.map(quarantined, fn {instance, _reason} -> instance end) + ) # This is for backwards compatibility. We originally didn't sent # extra info like a reason why an instance was rejected/quarantined/etc. # Because we didn't want to break backwards compatibility it was decided # to add an extra "info" key. - |> Map.merge(%{ - quarantined_instances_info: %{ - "quarantined_instances" => - quarantined - |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) - |> Enum.into(%{}) - } + |> Map.put(:quarantined_instances_info, %{ + "quarantined_instances" => + quarantined + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() }) else %{}