From 4a8c26654eb7ca7ce049dd4c485c16672b5837a6 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 16 Nov 2019 22:54:13 +0100 Subject: [PATCH 01/11] Restrict statuses that contain user's irreversible filters --- lib/pleroma/filter.ex | 42 ++++++++++- lib/pleroma/web/activity_pub/activity_pub.ex | 22 ++++++ .../controllers/filter_controller.ex | 2 +- test/filter_test.exs | 2 +- test/support/factory.ex | 8 +++ test/web/activity_pub/activity_pub_test.exs | 69 +++++++++++++++++++ 6 files changed, 141 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 4d61b3650..91884c6b3 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -34,10 +34,18 @@ defmodule Pleroma.Filter do Repo.one(query) end - def get_filters(%User{id: user_id} = _user) do + def get_active(query) do + from(f in query, where: is_nil(f.expires_at) or f.expires_at > ^NaiveDateTime.utc_now()) + end + + def get_irreversible(query) do + from(f in query, where: f.hide) + end + + def get_by_user(query, %User{id: user_id} = _user) do query = from( - f in Pleroma.Filter, + f in query, where: f.user_id == ^user_id, order_by: [desc: :id] ) @@ -95,4 +103,34 @@ defmodule Pleroma.Filter do |> validate_required([:phrase, :context]) |> Repo.update() end + + def compose_regex(user_or_filters, format \\ :postgres) + + def compose_regex(%User{} = user, format) do + __MODULE__ + |> get_active() + |> get_irreversible() + |> get_by_user(user) + |> compose_regex(format) + end + + def compose_regex([_ | _] = filters, format) do + phrases = + filters + |> Enum.map(& &1.phrase) + |> Enum.join("|") + + case format do + :postgres -> + "\\y(#{phrases})\\y" + + :re -> + ~r/\b#{phrases}\b/i + + _ -> + nil + end + end + + def compose_regex(_, _), do: nil end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 94117202c..31353c866 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Constants alias Pleroma.Conversation alias Pleroma.Conversation.Participation + alias Pleroma.Filter alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -961,6 +962,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_instance(query, _), do: query + defp restrict_filtered(query, %{user: %User{} = user}) do + case Filter.compose_regex(user) do + nil -> + query + + regex -> + from([activity, object] in query, + where: + fragment("not(?->>'content' ~* ?)", object.data, ^regex) or + activity.actor == ^user.ap_id + ) + end + end + + defp restrict_filtered(query, %{blocking_user: %User{} = user}) do + restrict_filtered(query, %{user: user}) + end + + defp restrict_filtered(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1099,6 +1120,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted_reblogs(restrict_muted_reblogs_opts) |> restrict_instance(opts) |> restrict_announce_object_actor(opts) + |> restrict_filtered(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex index abbf0ce02..db1ff3189 100644 --- a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterController do @doc "GET /api/v1/filters" def index(%{assigns: %{user: user}} = conn, _) do - filters = Filter.get_filters(user) + filters = Filter.get_by_user(Filter, user) render(conn, "index.json", filters: filters) end diff --git a/test/filter_test.exs b/test/filter_test.exs index 63a30c736..061a95ad0 100644 --- a/test/filter_test.exs +++ b/test/filter_test.exs @@ -126,7 +126,7 @@ defmodule Pleroma.FilterTest do {:ok, filter_one} = Pleroma.Filter.create(query_one) {:ok, filter_two} = Pleroma.Filter.create(query_two) - filters = Pleroma.Filter.get_filters(user) + filters = Pleroma.Filter.get_by_user(Pleroma.Filter, user) assert filter_one in filters assert filter_two in filters end diff --git a/test/support/factory.ex b/test/support/factory.ex index af580021c..635d83650 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -428,4 +428,12 @@ defmodule Pleroma.Factory do user: build(:user) } end + + def filter_factory do + %Pleroma.Filter{ + user: build(:user), + filter_id: sequence(:filter_id, & &1), + phrase: "cofe" + } + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 575e0c5db..4968403dc 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -785,6 +785,75 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert activity == expected_activity end + describe "irreversible filters" do + setup do + user = insert(:user) + user_two = insert(:user) + + insert(:filter, user: user_two, phrase: "cofe", hide: true) + insert(:filter, user: user_two, phrase: "ok boomer", hide: true) + insert(:filter, user: user_two, phrase: "test", hide: false) + + params = %{ + "type" => ["Create", "Announce"], + "user" => user_two + } + + {:ok, %{user: user, user_two: user_two, params: params}} + end + + test "it returns statuses if they don't contain exact filter words", %{ + user: user, + params: params + } do + {:ok, _} = CommonAPI.post(user, %{"status" => "hey"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "got cofefe?"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "I am not a boomer"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "ok boomers"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "ccofee is not a word"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "this is a test"}) + + activities = ActivityPub.fetch_activities([], params) + + assert Enum.count(activities) == 6 + end + + test "it does not filter user's own statuses", %{user_two: user_two, params: params} do + {:ok, _} = CommonAPI.post(user_two, %{"status" => "Give me some cofe!"}) + {:ok, _} = CommonAPI.post(user_two, %{"status" => "ok boomer"}) + + activities = ActivityPub.fetch_activities([], params) + + assert Enum.count(activities) == 2 + end + + test "it excludes statuses with filter words", %{user: user, params: params} do + {:ok, _} = CommonAPI.post(user, %{"status" => "Give me some cofe!"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "ok boomer"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "is it a cOfE?"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "cofe is all I need"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "— ok BOOMER\n"}) + + activities = ActivityPub.fetch_activities([], params) + + assert Enum.empty?(activities) + end + + test "it returns all statuses if user does not have any filters" do + another_user = insert(:user) + {:ok, _} = CommonAPI.post(another_user, %{"status" => "got cofe?"}) + {:ok, _} = CommonAPI.post(another_user, %{"status" => "test!"}) + + activities = + ActivityPub.fetch_activities([], %{ + "type" => ["Create", "Announce"], + "user" => another_user + }) + + assert Enum.count(activities) == 2 + end + end + describe "public fetch activities" do test "doesn't retrieve unlisted activities" do user = insert(:user) From 5af1bf443dfd21a6b0be9efc1f55a73e590f6ba3 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 22 Nov 2019 19:52:50 +0100 Subject: [PATCH 02/11] Skip notifications for statuses that contain an irreversible filtered word --- lib/pleroma/notification.ex | 36 ++++++++++++- test/notification_test.exs | 101 ++++++++++++++++++++++++++++++------ 2 files changed, 119 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 2ef1a80c5..3f749cace 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -130,6 +130,7 @@ defmodule Pleroma.Notification do |> preload([n, a, o], activity: {a, object: o}) |> exclude_notification_muted(user, exclude_notification_muted_opts) |> exclude_blocked(user, exclude_blocked_opts) + |> exclude_filtered(user) |> exclude_visibility(opts) end @@ -158,6 +159,20 @@ defmodule Pleroma.Notification do |> where([n, a, o, tm], is_nil(tm.user_id)) end + defp exclude_filtered(query, user) do + case Pleroma.Filter.compose_regex(user) do + nil -> + query + + regex -> + from([_n, a, o] in query, + where: + fragment("not(?->>'content' ~* ?)", o.data, ^regex) or + fragment("?->>'actor' = ?", o.data, ^user.ap_id) + ) + end + end + @valid_visibilities ~w[direct unlisted public private] defp exclude_visibility(query, %{exclude_visibilities: visibility}) @@ -555,7 +570,8 @@ defmodule Pleroma.Notification do :follows, :non_followers, :non_follows, - :recently_followed + :recently_followed, + :filtered ] |> Enum.find(&skip?(&1, activity, user)) end @@ -624,6 +640,24 @@ defmodule Pleroma.Notification do end) end + def skip?(:filtered, activity, user) do + object = Object.normalize(activity) + + cond do + is_nil(object) -> + false + + object.data["actor"] == user.ap_id -> + false + + not is_nil(regex = Pleroma.Filter.compose_regex(user, :re)) -> + Regex.match?(regex, object.data["content"]) + + true -> + false + end + end + def skip?(_, _, _), do: false def for_user_and_activity(user, activity) do diff --git a/test/notification_test.exs b/test/notification_test.exs index 6add3f7eb..9ac6925c3 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -324,6 +324,44 @@ defmodule Pleroma.NotificationTest do {:ok, status} = CommonAPI.post(author, %{status: "hey @#{user.nickname}"}) refute Notification.create_notification(status, user) end + + test "it doesn't create notifications if content matches with an irreversible filter" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + insert(:filter, user: subscriber, phrase: "cofe", hide: true) + + {:ok, status} = CommonAPI.post(user, %{"status" => "got cofe?"}) + + assert {:ok, [nil]} == Notification.create_notifications(status) + end + + test "it creates notifications if content matches with a not irreversible filter" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + insert(:filter, user: subscriber, phrase: "cofe", hide: false) + + {:ok, status} = CommonAPI.post(user, %{"status" => "got cofe?"}) + {:ok, [notification]} = Notification.create_notifications(status) + + assert notification + end + + test "it creates notifications when someone likes user's status with a filtered word" do + user = insert(:user) + other_user = insert(:user) + insert(:filter, user: user, phrase: "tesla", hide: true) + + {:ok, activity_one} = CommonAPI.post(user, %{"status" => "wow tesla"}) + {:ok, activity_two, _} = CommonAPI.favorite(activity_one.id, other_user) + + {:ok, [notification]} = Notification.create_notifications(activity_two) + + assert notification + end end describe "follow / follow_request notifications" do @@ -990,8 +1028,13 @@ defmodule Pleroma.NotificationTest do end describe "for_user" do - test "it returns notifications for muted user without notifications" do + setup do user = insert(:user) + + {:ok, %{user: user}} + end + + test "it returns notifications for muted user without notifications", %{user: user} do muted = insert(:user) {:ok, _user_relationships} = User.mute(user, muted, false) @@ -1002,8 +1045,7 @@ defmodule Pleroma.NotificationTest do assert notification.activity.object end - test "it doesn't return notifications for muted user with notifications" do - user = insert(:user) + test "it doesn't return notifications for muted user with notifications", %{user: user} do muted = insert(:user) {:ok, _user_relationships} = User.mute(user, muted) @@ -1012,8 +1054,7 @@ defmodule Pleroma.NotificationTest do assert Notification.for_user(user) == [] end - test "it doesn't return notifications for blocked user" do - user = insert(:user) + test "it doesn't return notifications for blocked user", %{user: user} do blocked = insert(:user) {:ok, _user_relationship} = User.block(user, blocked) @@ -1022,8 +1063,7 @@ defmodule Pleroma.NotificationTest do assert Notification.for_user(user) == [] end - test "it doesn't return notifications for domain-blocked non-followed user" do - user = insert(:user) + test "it doesn't return notifications for domain-blocked non-followed user", %{user: user} do blocked = insert(:user, ap_id: "http://some-domain.com") {:ok, user} = User.block_domain(user, "some-domain.com") @@ -1044,8 +1084,7 @@ defmodule Pleroma.NotificationTest do assert length(Notification.for_user(user)) == 1 end - test "it doesn't return notifications for muted thread" do - user = insert(:user) + test "it doesn't return notifications for muted thread", %{user: user} do another_user = insert(:user) {:ok, activity} = CommonAPI.post(another_user, %{status: "hey @#{user.nickname}"}) @@ -1054,8 +1093,7 @@ defmodule Pleroma.NotificationTest do assert Notification.for_user(user) == [] end - test "it returns notifications from a muted user when with_muted is set" do - user = insert(:user) + test "it returns notifications from a muted user when with_muted is set", %{user: user} do muted = insert(:user) {:ok, _user_relationships} = User.mute(user, muted) @@ -1064,8 +1102,9 @@ defmodule Pleroma.NotificationTest do assert length(Notification.for_user(user, %{with_muted: true})) == 1 end - test "it doesn't return notifications from a blocked user when with_muted is set" do - user = insert(:user) + test "it doesn't return notifications from a blocked user when with_muted is set", %{ + user: user + } do blocked = insert(:user) {:ok, _user_relationship} = User.block(user, blocked) @@ -1075,8 +1114,8 @@ defmodule Pleroma.NotificationTest do end test "when with_muted is set, " <> - "it doesn't return notifications from a domain-blocked non-followed user" do - user = insert(:user) + "it doesn't return notifications from a domain-blocked non-followed user", + %{user: user} do blocked = insert(:user, ap_id: "http://some-domain.com") {:ok, user} = User.block_domain(user, "some-domain.com") @@ -1085,8 +1124,7 @@ defmodule Pleroma.NotificationTest do assert Enum.empty?(Notification.for_user(user, %{with_muted: true})) end - test "it returns notifications from muted threads when with_muted is set" do - user = insert(:user) + test "it returns notifications from muted threads when with_muted is set", %{user: user} do another_user = insert(:user) {:ok, activity} = CommonAPI.post(another_user, %{status: "hey @#{user.nickname}"}) @@ -1094,5 +1132,34 @@ defmodule Pleroma.NotificationTest do {:ok, _} = Pleroma.ThreadMute.add_mute(user.id, activity.data["context"]) assert length(Notification.for_user(user, %{with_muted: true})) == 1 end + + test "it doesn't return notifications about mentiones with filtered word", %{user: user} do + insert(:filter, user: user, phrase: "cofe", hide: true) + another_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(another_user, %{"status" => "@#{user.nickname} got cofe?"}) + + assert Enum.empty?(Notification.for_user(user)) + end + + test "it returns notifications about mentiones with not hidden filtered word", %{user: user} do + insert(:filter, user: user, phrase: "test", hide: false) + another_user = insert(:user) + + {:ok, _activity} = CommonAPI.post(another_user, %{"status" => "@#{user.nickname} test"}) + + assert length(Notification.for_user(user)) == 1 + end + + test "it returns notifications about favorites with filtered word", %{user: user} do + insert(:filter, user: user, phrase: "cofe", hide: true) + another_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Give me my cofe!"}) + {:ok, _, _} = CommonAPI.favorite(activity.id, another_user) + + assert length(Notification.for_user(user)) == 1 + end end end From 8277b29790dfd283d94b995539dcb28e51131150 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 25 Nov 2019 16:59:55 +0100 Subject: [PATCH 03/11] Restrict thread statuses that contain user's irreversible filters --- CHANGELOG.md | 7 ++--- lib/pleroma/web/activity_pub/activity_pub.ex | 2 ++ test/notification_test.exs | 2 +- test/web/activity_pub/activity_pub_test.exs | 28 ++++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85401809a..0d31e7928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Streaming: Repeats of a user's posts will no longer be pushed to the user's stream. - Mastodon API: Added `pleroma.metadata.fields_limits` to /api/v1/instance - Mastodon API: On deletion, returns the original post text. +- Mastodon API: Add `pleroma.unread_count` to the Marker entity.
@@ -58,8 +59,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Extended `/api/v1/instance`. - Mastodon API: Support for `include_types` in `/api/v1/notifications`. - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. -- Mastodon API: Add support for filtering replies in public and home timelines -- Mastodon API: Support for `bot` field in `/api/v1/accounts/update_credentials` +- Mastodon API: Add support for filtering replies in public and home timelines. +- Mastodon API: Support for `bot` field in `/api/v1/accounts/update_credentials`. +- Mastodon API: Support irreversible property for filters. - Admin API: endpoints for create/update/delete OAuth Apps. - Admin API: endpoint for status view. - OTP: Add command to reload emoji packs @@ -214,7 +216,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `pleroma.thread_muted` to the Status entity - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message - Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload. -- Mastodon API: Add `pleroma.unread_count` to the Marker entity - Admin API: Render whole status in grouped reports - Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise). - Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try. diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 31353c866..8abbef487 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -447,6 +447,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_set_thread_muted_field(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts[:user]) + |> restrict_filtered(opts) |> where( [activity], fragment( @@ -1112,6 +1113,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) |> restrict_muted(restrict_muted_opts) + |> restrict_filtered(opts) |> restrict_media(opts) |> restrict_visibility(opts) |> restrict_thread_visibility(opts, config) diff --git a/test/notification_test.exs b/test/notification_test.exs index 9ac6925c3..abaafd60e 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -1147,7 +1147,7 @@ defmodule Pleroma.NotificationTest do insert(:filter, user: user, phrase: "test", hide: false) another_user = insert(:user) - {:ok, _activity} = CommonAPI.post(another_user, %{"status" => "@#{user.nickname} test"}) + {:ok, _} = CommonAPI.post(another_user, %{"status" => "@#{user.nickname} test"}) assert length(Notification.for_user(user)) == 1 end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4968403dc..2190ff808 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -507,6 +507,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do activities = ActivityPub.fetch_activities_for_context("2hu", %{blocking_user: user}) assert activities == [activity_two, activity] end + + test "doesn't return activities with filtered words" do + user = insert(:user) + user_two = insert(:user) + insert(:filter, user: user, phrase: "test", hide: true) + + {:ok, %{id: id1, data: %{"context" => context}}} = CommonAPI.post(user, %{"status" => "1"}) + + {:ok, %{id: id2}} = + CommonAPI.post(user_two, %{"status" => "2", "in_reply_to_status_id" => id1}) + + {:ok, %{id: id3} = user_activity} = + CommonAPI.post(user, %{"status" => "3 test?", "in_reply_to_status_id" => id2}) + + {:ok, %{id: id4} = filtered_activity} = + CommonAPI.post(user_two, %{"status" => "4 test!", "in_reply_to_status_id" => id3}) + + {:ok, _} = CommonAPI.post(user, %{"status" => "5", "in_reply_to_status_id" => id4}) + + activities = + context + |> ActivityPub.fetch_activities_for_context(%{"user" => user}) + |> Enum.map(& &1.id) + + assert length(activities) == 4 + assert user_activity.id in activities + refute filtered_activity.id in activities + end end test "doesn't return blocked activities" do From 6558f31cda07b8472ed99823ed0f46deffa584cc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 7 Feb 2020 18:16:39 +0300 Subject: [PATCH 04/11] don't filter notifications for follow and move types --- lib/pleroma/notification.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 3f749cace..d439f51bc 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -640,6 +640,8 @@ defmodule Pleroma.Notification do end) end + def skip?(:filtered, %{data: %{"type" => type}}, _) when type in ["Follow", "Move"], do: false + def skip?(:filtered, activity, user) do object = Object.normalize(activity) From 771748db1fa01a71c52c20b890e1b80bfcf1e230 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 13:59:07 +0000 Subject: [PATCH 05/11] Apply suggestion to lib/pleroma/filter.ex --- lib/pleroma/filter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 91884c6b3..98cb575a9 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Filter do from(f in query, where: f.hide) end - def get_by_user(query, %User{id: user_id} = _user) do + def get_filters(query \\ __MODULE__, %User{id: user_id}) do query = from( f in query, From 086a260c04185623065a97e0ba5277585d4fd49a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 14:00:28 +0000 Subject: [PATCH 06/11] Apply suggestion to test/notification_test.exs --- test/notification_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index abaafd60e..8679f52a5 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -1133,7 +1133,7 @@ defmodule Pleroma.NotificationTest do assert length(Notification.for_user(user, %{with_muted: true})) == 1 end - test "it doesn't return notifications about mentiones with filtered word", %{user: user} do + test "it doesn't return notifications about mentions with filtered word", %{user: user} do insert(:filter, user: user, phrase: "cofe", hide: true) another_user = insert(:user) From 52ff75413a5a73f045c7b515a06ae40eb568dfa8 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 14:00:38 +0000 Subject: [PATCH 07/11] Apply suggestion to test/notification_test.exs --- test/notification_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 8679f52a5..3279ea61e 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -1143,7 +1143,7 @@ defmodule Pleroma.NotificationTest do assert Enum.empty?(Notification.for_user(user)) end - test "it returns notifications about mentiones with not hidden filtered word", %{user: user} do + test "it returns notifications about mentions with not hidden filtered word", %{user: user} do insert(:filter, user: user, phrase: "test", hide: false) another_user = insert(:user) From 20c27bef4083330a2415f1c0a04e4cad128b267a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 17:50:56 +0300 Subject: [PATCH 08/11] renaming back and reject nil on create --- lib/pleroma/filter.ex | 2 +- lib/pleroma/notification.ex | 1 + .../controllers/filter_controller.ex | 2 +- test/filter_test.exs | 59 ++++++++++--------- test/notification_test.exs | 2 +- 5 files changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 98cb575a9..5d6df9530 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -110,7 +110,7 @@ defmodule Pleroma.Filter do __MODULE__ |> get_active() |> get_irreversible() - |> get_by_user(user) + |> get_filters(user) |> compose_regex(format) end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index d439f51bc..fcb2144ae 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -352,6 +352,7 @@ defmodule Pleroma.Notification do end end + @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} def create_notifications(activity, options \\ []) def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do diff --git a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex index db1ff3189..abbf0ce02 100644 --- a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterController do @doc "GET /api/v1/filters" def index(%{assigns: %{user: user}} = conn, _) do - filters = Filter.get_by_user(Filter, user) + filters = Filter.get_filters(user) render(conn, "index.json", filters: filters) end diff --git a/test/filter_test.exs b/test/filter_test.exs index 061a95ad0..0a5c4426a 100644 --- a/test/filter_test.exs +++ b/test/filter_test.exs @@ -3,37 +3,39 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.FilterTest do - alias Pleroma.Repo use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.Filter + alias Pleroma.Repo + describe "creating filters" do test "creating one filter" do user = insert(:user) - query = %Pleroma.Filter{ + query = %Filter{ user_id: user.id, filter_id: 42, phrase: "knights", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter} = Pleroma.Filter.create(query) - result = Pleroma.Filter.get(filter.filter_id, user) + {:ok, %Filter{} = filter} = Filter.create(query) + result = Filter.get(filter.filter_id, user) assert query.phrase == result.phrase end test "creating one filter without a pre-defined filter_id" do user = insert(:user) - query = %Pleroma.Filter{ + query = %Filter{ user_id: user.id, phrase: "knights", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter} = Pleroma.Filter.create(query) + {:ok, %Filter{} = filter} = Filter.create(query) # Should start at 1 assert filter.filter_id == 1 end @@ -41,23 +43,23 @@ defmodule Pleroma.FilterTest do test "creating additional filters uses previous highest filter_id + 1" do user = insert(:user) - query_one = %Pleroma.Filter{ + query_one = %Filter{ user_id: user.id, filter_id: 42, phrase: "knights", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter_one} = Pleroma.Filter.create(query_one) + {:ok, %Filter{} = filter_one} = Filter.create(query_one) - query_two = %Pleroma.Filter{ + query_two = %Filter{ user_id: user.id, # No filter_id phrase: "who", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter_two} = Pleroma.Filter.create(query_two) + {:ok, %Filter{} = filter_two} = Filter.create(query_two) assert filter_two.filter_id == filter_one.filter_id + 1 end @@ -65,29 +67,29 @@ defmodule Pleroma.FilterTest do user_one = insert(:user) user_two = insert(:user) - query_one = %Pleroma.Filter{ + query_one = %Filter{ user_id: user_one.id, phrase: "knights", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter_one} = Pleroma.Filter.create(query_one) + {:ok, %Filter{} = filter_one} = Filter.create(query_one) - query_two = %Pleroma.Filter{ + query_two = %Filter{ user_id: user_two.id, phrase: "who", context: ["home"] } - {:ok, %Pleroma.Filter{} = filter_two} = Pleroma.Filter.create(query_two) + {:ok, %Filter{} = filter_two} = Filter.create(query_two) assert filter_one.filter_id == 1 assert filter_two.filter_id == 1 - result_one = Pleroma.Filter.get(filter_one.filter_id, user_one) + result_one = Filter.get(filter_one.filter_id, user_one) assert result_one.phrase == filter_one.phrase - result_two = Pleroma.Filter.get(filter_two.filter_id, user_two) + result_two = Filter.get(filter_two.filter_id, user_two) assert result_two.phrase == filter_two.phrase end end @@ -95,38 +97,38 @@ defmodule Pleroma.FilterTest do test "deleting a filter" do user = insert(:user) - query = %Pleroma.Filter{ + query = %Filter{ user_id: user.id, filter_id: 0, phrase: "knights", context: ["home"] } - {:ok, _filter} = Pleroma.Filter.create(query) - {:ok, filter} = Pleroma.Filter.delete(query) - assert is_nil(Repo.get(Pleroma.Filter, filter.filter_id)) + {:ok, _filter} = Filter.create(query) + {:ok, filter} = Filter.delete(query) + assert is_nil(Repo.get(Filter, filter.filter_id)) end test "getting all filters by an user" do user = insert(:user) - query_one = %Pleroma.Filter{ + query_one = %Filter{ user_id: user.id, filter_id: 1, phrase: "knights", context: ["home"] } - query_two = %Pleroma.Filter{ + query_two = %Filter{ user_id: user.id, filter_id: 2, phrase: "who", context: ["home"] } - {:ok, filter_one} = Pleroma.Filter.create(query_one) - {:ok, filter_two} = Pleroma.Filter.create(query_two) - filters = Pleroma.Filter.get_by_user(Pleroma.Filter, user) + {:ok, filter_one} = Filter.create(query_one) + {:ok, filter_two} = Filter.create(query_two) + filters = Filter.get_filters(user) assert filter_one in filters assert filter_two in filters end @@ -134,7 +136,7 @@ defmodule Pleroma.FilterTest do test "updating a filter" do user = insert(:user) - query_one = %Pleroma.Filter{ + query_one = %Filter{ user_id: user.id, filter_id: 1, phrase: "knights", @@ -146,8 +148,9 @@ defmodule Pleroma.FilterTest do context: ["home", "timeline"] } - {:ok, filter_one} = Pleroma.Filter.create(query_one) - {:ok, filter_two} = Pleroma.Filter.update(filter_one, changes) + {:ok, filter_one} = Filter.create(query_one) + {:ok, filter_two} = Filter.update(filter_one, changes) + assert filter_one != filter_two assert filter_two.phrase == changes.phrase assert filter_two.context == changes.context diff --git a/test/notification_test.exs b/test/notification_test.exs index 3279ea61e..898c804cb 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -334,7 +334,7 @@ defmodule Pleroma.NotificationTest do {:ok, status} = CommonAPI.post(user, %{"status" => "got cofe?"}) - assert {:ok, [nil]} == Notification.create_notifications(status) + assert {:ok, []} == Notification.create_notifications(status) end test "it creates notifications if content matches with a not irreversible filter" do From da509487b21bbb627e5fdac6815ad9b3e4e4728b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 31 Mar 2020 16:53:11 +0300 Subject: [PATCH 09/11] adding benchmarks in new format --- benchmarks/load_testing/activities.ex | 10 ++++++++ benchmarks/load_testing/fetcher.ex | 34 +++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 074ded457..f5c7bfce8 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -24,6 +24,7 @@ defmodule Pleroma.LoadTesting.Activities do @visibility ~w(public private direct unlisted) @types [ :simple, + :simple_filtered, :emoji, :mentions, :hell_thread, @@ -242,6 +243,15 @@ defmodule Pleroma.LoadTesting.Activities do insert_local_activity(visibility, group, users, "Simple status") end + defp insert_activity(:simple_filtered, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status which must be filtered") + end + + defp insert_activity(:simple_filtered, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status which must be filtered") + end + defp insert_activity(:emoji, visibility, group, users, _opts) when group in @remote_groups do insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:") diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 15fd06c3d..dfbd916be 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -32,10 +32,22 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end + defp create_filter(user) do + Pleroma.Filter.create(%Pleroma.Filter{ + user_id: user.id, + phrase: "must be filtered", + hide: true + }) + end + + defp delete_filter(filter), do: Repo.delete(filter) + defp fetch_timelines(user) do fetch_home_timeline(user) + fetch_home_timeline_with_filter(user) fetch_direct_timeline(user) fetch_public_timeline(user) + fetch_public_timeline_with_filter(user) fetch_public_timeline(user, :with_blocks) fetch_public_timeline(user, :local) fetch_public_timeline(user, :tag) @@ -61,7 +73,7 @@ defmodule Pleroma.LoadTesting.Fetcher do } end - defp fetch_home_timeline(user) do + defp fetch_home_timeline(user, title_end \\ "") do opts = opts_for_home_timeline(user) recipients = [user.ap_id | User.following(user)] @@ -84,9 +96,11 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Enum.reverse() |> List.last() + title = "home timeline " <> title_end + Benchee.run( %{ - "home timeline" => fn opts -> ActivityPub.fetch_activities(recipients, opts) end + title => fn opts -> ActivityPub.fetch_activities(recipients, opts) end }, inputs: %{ "1 page" => opts, @@ -108,6 +122,14 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end + defp fetch_home_timeline_with_filter(user) do + {:ok, filter} = create_filter(user) + + fetch_home_timeline(user, "with filters") + + delete_filter(filter) + end + defp opts_for_direct_timeline(user) do %{ visibility: "direct", @@ -210,6 +232,14 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_public_timeline(opts, "public timeline") end + defp fetch_public_timeline_with_filter(user) do + {:ok, filter} = create_filter(user) + opts = opts_for_public_timeline(user) + + fetch_public_timeline(opts, "public timeline with filters") + delete_filter(filter) + end + defp fetch_public_timeline(user, :local) do opts = opts_for_public_timeline(user, :local) From 028a241b7dc45e31161e29ca24a34be8740a4656 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 1 May 2020 09:20:54 +0300 Subject: [PATCH 10/11] tests fixes --- test/notification_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 898c804cb..366dc176c 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -356,7 +356,7 @@ defmodule Pleroma.NotificationTest do insert(:filter, user: user, phrase: "tesla", hide: true) {:ok, activity_one} = CommonAPI.post(user, %{"status" => "wow tesla"}) - {:ok, activity_two, _} = CommonAPI.favorite(activity_one.id, other_user) + {:ok, activity_two} = CommonAPI.favorite(other_user, activity_one.id) {:ok, [notification]} = Notification.create_notifications(activity_two) @@ -1157,7 +1157,7 @@ defmodule Pleroma.NotificationTest do another_user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "Give me my cofe!"}) - {:ok, _, _} = CommonAPI.favorite(activity.id, another_user) + {:ok, _} = CommonAPI.favorite(another_user, activity.id) assert length(Notification.for_user(user)) == 1 end From 818f3c2393fb428997f783e599b0d629dcd5a842 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 17 Jun 2020 12:34:27 +0300 Subject: [PATCH 11/11] test fixes --- test/notification_test.exs | 13 +++--- test/web/activity_pub/activity_pub_test.exs | 51 ++++++++++----------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index 366dc176c..13e82ab2a 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -332,7 +332,7 @@ defmodule Pleroma.NotificationTest do User.subscribe(subscriber, user) insert(:filter, user: subscriber, phrase: "cofe", hide: true) - {:ok, status} = CommonAPI.post(user, %{"status" => "got cofe?"}) + {:ok, status} = CommonAPI.post(user, %{status: "got cofe?"}) assert {:ok, []} == Notification.create_notifications(status) end @@ -344,7 +344,7 @@ defmodule Pleroma.NotificationTest do User.subscribe(subscriber, user) insert(:filter, user: subscriber, phrase: "cofe", hide: false) - {:ok, status} = CommonAPI.post(user, %{"status" => "got cofe?"}) + {:ok, status} = CommonAPI.post(user, %{status: "got cofe?"}) {:ok, [notification]} = Notification.create_notifications(status) assert notification @@ -355,7 +355,7 @@ defmodule Pleroma.NotificationTest do other_user = insert(:user) insert(:filter, user: user, phrase: "tesla", hide: true) - {:ok, activity_one} = CommonAPI.post(user, %{"status" => "wow tesla"}) + {:ok, activity_one} = CommonAPI.post(user, %{status: "wow tesla"}) {:ok, activity_two} = CommonAPI.favorite(other_user, activity_one.id) {:ok, [notification]} = Notification.create_notifications(activity_two) @@ -1137,8 +1137,7 @@ defmodule Pleroma.NotificationTest do insert(:filter, user: user, phrase: "cofe", hide: true) another_user = insert(:user) - {:ok, _activity} = - CommonAPI.post(another_user, %{"status" => "@#{user.nickname} got cofe?"}) + {:ok, _activity} = CommonAPI.post(another_user, %{status: "@#{user.nickname} got cofe?"}) assert Enum.empty?(Notification.for_user(user)) end @@ -1147,7 +1146,7 @@ defmodule Pleroma.NotificationTest do insert(:filter, user: user, phrase: "test", hide: false) another_user = insert(:user) - {:ok, _} = CommonAPI.post(another_user, %{"status" => "@#{user.nickname} test"}) + {:ok, _} = CommonAPI.post(another_user, %{status: "@#{user.nickname} test"}) assert length(Notification.for_user(user)) == 1 end @@ -1156,7 +1155,7 @@ defmodule Pleroma.NotificationTest do insert(:filter, user: user, phrase: "cofe", hide: true) another_user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{"status" => "Give me my cofe!"}) + {:ok, activity} = CommonAPI.post(user, %{status: "Give me my cofe!"}) {:ok, _} = CommonAPI.favorite(another_user, activity.id) assert length(Notification.for_user(user)) == 1 diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 2190ff808..17e12a1a7 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -513,22 +513,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user_two = insert(:user) insert(:filter, user: user, phrase: "test", hide: true) - {:ok, %{id: id1, data: %{"context" => context}}} = CommonAPI.post(user, %{"status" => "1"}) + {:ok, %{id: id1, data: %{"context" => context}}} = CommonAPI.post(user, %{status: "1"}) - {:ok, %{id: id2}} = - CommonAPI.post(user_two, %{"status" => "2", "in_reply_to_status_id" => id1}) + {:ok, %{id: id2}} = CommonAPI.post(user_two, %{status: "2", in_reply_to_status_id: id1}) {:ok, %{id: id3} = user_activity} = - CommonAPI.post(user, %{"status" => "3 test?", "in_reply_to_status_id" => id2}) + CommonAPI.post(user, %{status: "3 test?", in_reply_to_status_id: id2}) {:ok, %{id: id4} = filtered_activity} = - CommonAPI.post(user_two, %{"status" => "4 test!", "in_reply_to_status_id" => id3}) + CommonAPI.post(user_two, %{status: "4 test!", in_reply_to_status_id: id3}) - {:ok, _} = CommonAPI.post(user, %{"status" => "5", "in_reply_to_status_id" => id4}) + {:ok, _} = CommonAPI.post(user, %{status: "5", in_reply_to_status_id: id4}) activities = context - |> ActivityPub.fetch_activities_for_context(%{"user" => user}) + |> ActivityPub.fetch_activities_for_context(%{user: user}) |> Enum.map(& &1.id) assert length(activities) == 4 @@ -823,8 +822,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do insert(:filter, user: user_two, phrase: "test", hide: false) params = %{ - "type" => ["Create", "Announce"], - "user" => user_two + type: ["Create", "Announce"], + user: user_two } {:ok, %{user: user, user_two: user_two, params: params}} @@ -834,12 +833,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do user: user, params: params } do - {:ok, _} = CommonAPI.post(user, %{"status" => "hey"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "got cofefe?"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "I am not a boomer"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "ok boomers"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "ccofee is not a word"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "this is a test"}) + {:ok, _} = CommonAPI.post(user, %{status: "hey"}) + {:ok, _} = CommonAPI.post(user, %{status: "got cofefe?"}) + {:ok, _} = CommonAPI.post(user, %{status: "I am not a boomer"}) + {:ok, _} = CommonAPI.post(user, %{status: "ok boomers"}) + {:ok, _} = CommonAPI.post(user, %{status: "ccofee is not a word"}) + {:ok, _} = CommonAPI.post(user, %{status: "this is a test"}) activities = ActivityPub.fetch_activities([], params) @@ -847,8 +846,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end test "it does not filter user's own statuses", %{user_two: user_two, params: params} do - {:ok, _} = CommonAPI.post(user_two, %{"status" => "Give me some cofe!"}) - {:ok, _} = CommonAPI.post(user_two, %{"status" => "ok boomer"}) + {:ok, _} = CommonAPI.post(user_two, %{status: "Give me some cofe!"}) + {:ok, _} = CommonAPI.post(user_two, %{status: "ok boomer"}) activities = ActivityPub.fetch_activities([], params) @@ -856,11 +855,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end test "it excludes statuses with filter words", %{user: user, params: params} do - {:ok, _} = CommonAPI.post(user, %{"status" => "Give me some cofe!"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "ok boomer"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "is it a cOfE?"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "cofe is all I need"}) - {:ok, _} = CommonAPI.post(user, %{"status" => "— ok BOOMER\n"}) + {:ok, _} = CommonAPI.post(user, %{status: "Give me some cofe!"}) + {:ok, _} = CommonAPI.post(user, %{status: "ok boomer"}) + {:ok, _} = CommonAPI.post(user, %{status: "is it a cOfE?"}) + {:ok, _} = CommonAPI.post(user, %{status: "cofe is all I need"}) + {:ok, _} = CommonAPI.post(user, %{status: "— ok BOOMER\n"}) activities = ActivityPub.fetch_activities([], params) @@ -869,13 +868,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do test "it returns all statuses if user does not have any filters" do another_user = insert(:user) - {:ok, _} = CommonAPI.post(another_user, %{"status" => "got cofe?"}) - {:ok, _} = CommonAPI.post(another_user, %{"status" => "test!"}) + {:ok, _} = CommonAPI.post(another_user, %{status: "got cofe?"}) + {:ok, _} = CommonAPI.post(another_user, %{status: "test!"}) activities = ActivityPub.fetch_activities([], %{ - "type" => ["Create", "Announce"], - "user" => another_user + type: ["Create", "Announce"], + user: another_user }) assert Enum.count(activities) == 2