From 958b4cfde916c9be71d7412fe1c90750ec578fdc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 24 Mar 2019 23:45:57 +0000 Subject: [PATCH 01/12] migrations: add function to see if a thread can be satisfied --- ...4222404_add_thread_visibility_function.exs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 priv/repo/migrations/20190324222404_add_thread_visibility_function.exs diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs new file mode 100644 index 000000000..cea0322e7 --- /dev/null +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -0,0 +1,76 @@ +defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do + use Ecto.Migration + @disable_ddl_transaction true + + def up do + statement = """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + actor_user users%ROWTYPE; + author users%ROWTYPE; + author_fa varchar; + BEGIN + --- Fetch our actor. + SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + IF activity IS NULL THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author. + SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + + --- Prepare author's AS2 followers collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa; + + --- Check visibility. + IF activity.actor = actor THEN + --- activity visible + NULL; + ELSIF ARRAY[public] && activity.recipients THEN + --- activity visible + NULL; + ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN + --- activity visible + NULL; + ELSIF ARRAY[actor] && activity.recipients THEN + --- activity visible + NULL; + ELSE + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("drop function thread_visibility(actor varchar, activity_id varchar)") + end +end From 0387f5213805cdc4e0bf86f98797cefcd03ba61d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:06:02 +0000 Subject: [PATCH 02/12] activitypub: add restrict_thread_visibility() --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 233fee4fa..fec1bcd3e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -569,6 +569,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_visibility(query, _visibility), do: query + defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do + query = + from( + a in query, + where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) + ) + + Ecto.Adapters.SQL.to_sql(:all, Repo, query) + + query + end + + defp restrict_thread_visibility(query, _), do: query + def fetch_user_activities(user, reading_user, params \\ %{}) do params = params @@ -848,6 +862,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted(opts) |> restrict_media(opts) |> restrict_visibility(opts) + |> restrict_thread_visibility(opts) |> restrict_replies(opts) |> restrict_reblogs(opts) |> restrict_pinned(opts) From de114ffbb0f92d24fd370adaaf43ff301ab04b4b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:10:20 +0000 Subject: [PATCH 03/12] activitypub: remove contain_timeline() --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 -------- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 - lib/pleroma/web/twitter_api/twitter_api_controller.ex | 4 +--- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fec1bcd3e..e544d0c50 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -980,12 +980,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def contain_activity(%Activity{} = activity, %User{} = user) do contain_broken_threads(activity, user) end - - # do post-processing on a timeline - def contain_timeline(timeline, user) do - timeline - |> Enum.filter(fn activity -> - contain_activity(activity, user) - end) - end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 87e597074..66056a846 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -303,7 +303,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) |> Enum.reverse() conn diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 3c5a70be9..31e86685a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -101,9 +101,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do |> Map.put("blocking_user", user) |> Map.put("user", user) - activities = - ActivityPub.fetch_activities([user.ap_id | user.following], params) - |> ActivityPub.contain_timeline(user) + activities = ActivityPub.fetch_activities([user.ap_id | user.following], params) conn |> put_view(ActivityView) From 31db31c5879a2dedcc8dd4c671c4c9a79656355a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:38:28 +0000 Subject: [PATCH 04/12] activitypub: visibility: use SQL thread_visibility() function instead of manually walking the thread --- lib/pleroma/web/activity_pub/visibility.ex | 26 +++++++--------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index b38ee0442..46dd46575 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false @@ -38,25 +39,14 @@ defmodule Pleroma.Web.ActivityPub.Visibility do visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y)) end - # guard - def entire_thread_visible_for_user?(nil, _user), do: false + def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do + {:ok, %{rows: [[result]]}} = + Ecto.Adapters.SQL.query(Repo, "SELECT thread_visibility($1, $2)", [ + user.ap_id, + activity.data["id"] + ]) - # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop - # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - - def entire_thread_visible_for_user?( - %Activity{} = tail, - # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, - user - ) do - case Object.normalize(tail) do - %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - - _ -> - visible_for_user?(tail, user) - end + result end def get_visibility(object) do From c7644313e72520a371e4bd417b1ff852365849b6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 01:23:15 +0000 Subject: [PATCH 05/12] test: update obsolete test --- test/web/activity_pub/activity_pub_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0f90aa1ac..b41f6ab81 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -968,7 +968,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert length(activities) == 3 - activities = ActivityPub.contain_timeline(activities, user1) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) assert [public_activity, private_activity_1] == activities assert length(activities) == 2 From 75ce6adcffd2dbbc2ca2f83d7fe2d7fd659cd2f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 02:56:13 +0000 Subject: [PATCH 06/12] migration: only care about Create activities --- .../20190324222404_add_thread_visibility_function.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs index cea0322e7..11aa47e83 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -25,6 +25,11 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do RETURN true; END IF; + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + --- Normalize the child object into child. SELECT * INTO child FROM objects INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' From 0aada88b5594b6714b8d65f8bee9c325d77d6e7b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 8 May 2019 23:17:51 +0000 Subject: [PATCH 07/12] bbs: chase timeline containment patch --- lib/pleroma/bbs/handler.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 106fe5d18..f34be961f 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -95,7 +95,6 @@ defmodule Pleroma.BBS.Handler do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) Enum.each(activities, fn activity -> puts_activity(activity) From 12f45e2a8907c74c6b65d866bc3bab547b31edfa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:22:52 +0000 Subject: [PATCH 08/12] update migration --- ...n.exs => 20190515222404_add_thread_visibility_function.exs} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename priv/repo/migrations/{20190324222404_add_thread_visibility_function.exs => 20190515222404_add_thread_visibility_function.exs} (97%) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs similarity index 97% rename from priv/repo/migrations/20190324222404_add_thread_visibility_function.exs rename to priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index 11aa47e83..a3f717b89 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -21,8 +21,9 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do LOOP --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. IF activity IS NULL THEN - RETURN true; + RETURN false; END IF; --- We only care about Create activities. From f09c3afdf51eea17103d1445b31b7a269c474538 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:23:01 +0000 Subject: [PATCH 09/12] chase test failures --- lib/pleroma/filter.ex | 3 ++- test/user_test.exs | 2 -- test/web/activity_pub/activity_pub_test.exs | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 79efc29f0..90457dadf 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -38,7 +38,8 @@ defmodule Pleroma.Filter do query = from( f in Pleroma.Filter, - where: f.user_id == ^user_id + where: f.user_id == ^user_id, + order_by: [desc: :id] ) Repo.all(query) diff --git a/test/user_test.exs b/test/user_test.exs index 0b65e89e9..bb47b4958 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -873,7 +873,6 @@ defmodule Pleroma.UserTest do assert [activity] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) {:ok, _user} = User.deactivate(user) @@ -882,7 +881,6 @@ defmodule Pleroma.UserTest do assert [] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index b41f6ab81..34e23b852 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -960,18 +960,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do "in_reply_to_status_id" => private_activity_2.id }) - activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following]) + |> Enum.map(fn a -> a.id end) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + assert [public_activity.id, private_activity_1.id, private_activity_3.id] == activities assert length(activities) == 3 activities = ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) + |> Enum.map(fn a -> a.id end) - assert [public_activity, private_activity_1] == activities + assert [public_activity.id, private_activity_1.id] == activities assert length(activities) == 2 end end From 71fa7eeb6fdc7cf2087a32fb515ad11b7bf90c01 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:54:14 +0000 Subject: [PATCH 10/12] thread visibility function: significantly improve efficiency --- ...5222404_add_thread_visibility_function.exs | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a3f717b89..a4daf680b 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -10,8 +10,8 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do child objects%ROWTYPE; activity activities%ROWTYPE; actor_user users%ROWTYPE; - author users%ROWTYPE; author_fa varchar; + valid_recipients varchar[]; BEGIN --- Fetch our actor. SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; @@ -36,26 +36,17 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; - --- Fetch the author. - SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + --- Fetch the author's AS2 following collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; - --- Prepare author's AS2 followers collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa; + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + IF ARRAY[author_fa] && actor_user.following THEN + valid_recipients := valid_recipients || author_fa; + END IF; --- Check visibility. - IF activity.actor = actor THEN - --- activity visible - NULL; - ELSIF ARRAY[public] && activity.recipients THEN - --- activity visible - NULL; - ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN - --- activity visible - NULL; - ELSIF ARRAY[actor] && activity.recipients THEN - --- activity visible - NULL; - ELSE + IF NOT valid_recipients && activity.recipients THEN --- activity not visible, break out of the loop RETURN false; END IF; From a591ab6112abdf162f4d6fdfbbcdd85bbaf75058 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:56:46 +0000 Subject: [PATCH 11/12] activity pub: remove Ecto SQL query dumps --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e544d0c50..7cd5b889b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -540,8 +540,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query else Logger.error("Could not restrict visibility to #{visibility}") @@ -557,8 +555,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end @@ -576,8 +572,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end From f3971cbde3d69faec973717e1421f4a643ef947e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 17:02:40 +0000 Subject: [PATCH 12/12] thread visibility function: fix use of no longer used author variable --- .../20190515222404_add_thread_visibility_function.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a4daf680b..dc9abc998 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -37,7 +37,7 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; --- Fetch the author's AS2 following collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; --- Prepare valid recipients array. valid_recipients := ARRAY[actor, public];