reply filtering

This commit is contained in:
Alexander 2019-12-04 09:49:17 +03:00 committed by Alexander Strizhakov
parent 2430b9bf90
commit 6e625a427c
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
9 changed files with 636 additions and 18 deletions

View file

@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<summary>API Changes</summary>
- Mastodon API: Support for `include_types` in `/api/v1/notifications`.
- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
- Mastodon API: Add support for filtering replies in public and friends timelines
- Admin API: endpoints for create/update/delete OAuth Apps.
</details>

View file

@ -495,4 +495,57 @@ defp render_long_thread(user) do
formatters: formatters()
)
end
def query_replies(user) do
public_params = %{
"type" => ["Create", "Announce"],
"local_only" => false,
"blocking_user" => user,
"muting_user" => user,
"count" => 20
}
Benchee.run(%{
"Public timeline without reply filtering" => fn ->
ActivityPub.fetch_public_activities(public_params)
end,
"Public timeline with reply filtering - following" => fn ->
public_params
|> Map.put("reply_visibility", "following")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
end,
"Public timeline with reply filtering - self" => fn ->
public_params
|> Map.put("reply_visibility", "self")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
end
})
private_params = %{
"type" => ["Create", "Announce"],
"blocking_user" => user,
"muting_user" => user,
"user" => user,
"count" => 20
}
recipients = [user.ap_id | User.following(user)]
Benchee.run(%{
"Home timeline without reply filtering" => fn ->
ActivityPub.fetch_activities(recipients, private_params)
end,
"Home timeline with reply filtering - following" => fn ->
private_params = Map.put(private_params, "reply_visibility", "following")
ActivityPub.fetch_activities(recipients, private_params)
end,
"Home timeline with reply filtering - self" => fn ->
private_params = Map.put(private_params, "reply_visibility", "self")
ActivityPub.fetch_activities(recipients, private_params)
end
})
end
end

View file

@ -14,7 +14,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re
Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
Adding the parameter `reply_visibility` to the public and friends timelines quieries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
## Statuses
- `visibility`: has an additional possible value `list`

View file

@ -832,6 +832,7 @@ def set_cache({:error, err}), do: {:error, err}
def set_cache(%User{} = user) do
Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
Cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
{:ok, user}
end
@ -847,9 +848,22 @@ def update_and_set_cache(changeset) do
end
end
def get_user_friends_ap_ids(user) do
from(u in User.get_friends_query(user), select: u.ap_id)
|> Repo.all()
end
@spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
def get_cached_user_friends_ap_ids(user) do
Cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
get_user_friends_ap_ids(user)
end)
end
def invalidate_cache(user) do
Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
Cachex.del(:user_cache, "nickname:#{user.nickname}")
Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
end
@spec get_cached_by_ap_id(String.t()) :: User.t() | nil

View file

@ -54,13 +54,13 @@ defmodule Pleroma.User.Query do
select: term(),
limit: pos_integer()
}
| %{}
| map()
@ilike_criteria [:nickname, :name, :query]
@equal_criteria [:email]
@contains_criteria [:ap_id, :nickname]
@spec build(criteria()) :: Query.t()
@spec build(Query.t(), criteria()) :: Query.t()
def build(query \\ base_query(), criteria) do
prepare_query(query, criteria)
end

View file

@ -270,9 +270,9 @@ defp do_create(%{to: to, actor: actor, context: context, object: object} = param
),
{:ok, activity} <- insert(create_data, local, fake),
{:fake, false, activity} <- {:fake, fake, activity},
{:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
_ <- increase_replies_count_if_reply(create_data),
_ <- increase_poll_votes_if_vote(create_data),
{:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
{:ok, _actor} <- increase_note_count_if_public(actor, activity),
:ok <- maybe_federate(activity) do
{:ok, activity}
@ -700,12 +700,14 @@ def fetch_activities_for_context_query(context, opts) do
do: [opts["user"].ap_id | User.following(opts["user"])] ++ public,
else: public
opts = Map.put(opts, "user", opts["user"])
from(activity in Activity)
|> maybe_preload_objects(opts)
|> maybe_preload_bookmarks(opts)
|> maybe_set_thread_muted_field(opts)
|> restrict_blocked(opts)
|> restrict_recipients(recipients, opts["user"])
|> restrict_recipients(recipients, opts)
|> where(
[activity],
fragment(
@ -740,7 +742,10 @@ def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
@spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
opts = Map.drop(opts, ["user"])
opts =
opts
|> Map.put("reply_user", opts["user"])
|> Map.delete("user")
[Constants.as_public()]
|> fetch_activities_query(opts)
@ -976,13 +981,65 @@ defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
defp restrict_tag(query, _), do: query
defp restrict_recipients(query, [], _user), do: query
defp restrict_recipients(query, recipients, nil) do
from(activity in query, where: fragment("? && ?", ^recipients, activity.recipients))
defp reply_recipients(user, "following") do
[user.ap_id | User.get_cached_user_friends_ap_ids(user)]
end
defp restrict_recipients(query, recipients, user) do
defp reply_recipients(user, "self"), do: [user.ap_id]
defp restrict_recipients(query, [], _opts), do: query
defp restrict_recipients(
query,
recipients,
%{"user" => nil, "reply_user" => user, "reply_visibility" => visibility}
)
when not is_nil(user) and visibility in ["following", "self"] do
reply_recipients = reply_recipients(user, visibility)
from([activity, object] in query,
where:
fragment(
"? && ? AND (?->>'inReplyTo' IS NULL OR array_remove(?, ?) && ? OR ? = ?)",
^recipients,
activity.recipients,
object.data,
activity.recipients,
activity.actor,
^reply_recipients,
activity.actor,
^user.ap_id
)
)
end
defp restrict_recipients(query, recipients, %{"user" => nil}) do
from(activity in query,
where: fragment("? && ?", ^recipients, activity.recipients)
)
end
defp restrict_recipients(query, recipients, %{"user" => user, "reply_visibility" => visibility})
when visibility in ["following", "self"] do
reply_recipients = reply_recipients(user, visibility)
from(
[activity, object] in query,
where:
fragment(
"? && ? AND (?->>'inReplyTo' IS NULL OR array_remove(?, ?) && ?)",
^recipients,
activity.recipients,
object.data,
activity.recipients,
activity.actor,
^reply_recipients
),
or_where: activity.actor == ^user.ap_id
)
end
defp restrict_recipients(query, recipients, %{"user" => user}) do
from(
activity in query,
where: fragment("? && ?", ^recipients, activity.recipients),
@ -1254,13 +1311,15 @@ def fetch_activities_query(recipients, opts \\ %{}) do
skip_thread_containment: Config.get([:instance, :skip_thread_containment])
}
opts = Map.put(opts, "user", opts["user"])
Activity
|> maybe_preload_objects(opts)
|> maybe_preload_bookmarks(opts)
|> maybe_preload_report_notes(opts)
|> maybe_set_thread_muted_field(opts)
|> maybe_order(opts)
|> restrict_recipients(recipients, opts["user"])
|> restrict_recipients(recipients, opts)
|> restrict_tag(opts)
|> restrict_tag_reject(opts)
|> restrict_tag_all(opts)

View file

@ -84,14 +84,18 @@ defp attachments(%{params: params} = draft) do
%__MODULE__{draft | attachments: attachments}
end
defp in_reply_to(draft) do
case Map.get(draft.params, "in_reply_to_status_id") do
"" -> draft
nil -> draft
id -> %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
defp in_reply_to(%{params: %{"in_reply_to_status_id" => ""}} = draft), do: draft
defp in_reply_to(%{params: %{"in_reply_to_status_id" => id}} = draft) when is_binary(id) do
%__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
end
defp in_reply_to(%{params: %{"in_reply_to_status_id" => %Activity{} = in_reply_to}} = draft) do
%__MODULE__{draft | in_reply_to: in_reply_to}
end
defp in_reply_to(draft), do: draft
defp in_reply_to_conversation(draft) do
in_reply_to_conversation = Participation.get(draft.params["in_reply_to_conversation_id"])
%__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation}

View file

@ -100,6 +100,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
|> Map.put("local_only", local_only)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
conn

View file

@ -1910,4 +1910,490 @@ test "old user must be in the new user's `also_known_as` list" do
ActivityPub.move(old_user, new_user)
end
end
test "doesn't retrieve replies activities with exclude_replies" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "yeah"})
{:ok, _reply} =
CommonAPI.post(user, %{"status" => "yeah", "in_reply_to_status_id" => activity.id})
[result] = ActivityPub.fetch_public_activities(%{"exclude_replies" => "true"})
assert result.id == activity.id
assert length(ActivityPub.fetch_public_activities()) == 2
end
describe "replies filtering with public messages" do
setup :public_messages
test "public timeline", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 16
end
test "public timeline with reply_visibility `following`", %{
users: %{u1: user},
u1: u1,
u2: u2,
u3: u3,
u4: u4,
activities: activities
} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 14
visible_ids =
Map.values(u1) ++ Map.values(u2) ++ Map.values(u4) ++ Map.values(activities) ++ [u3[:r1]]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "public timeline with reply_visibility `self`", %{
users: %{u1: user},
u1: u1,
u2: u2,
u3: u3,
u4: u4,
activities: activities
} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert length(activities_ids) == 10
visible_ids = Map.values(u1) ++ [u2[:r1], u3[:r1], u4[:r1]] ++ Map.values(activities)
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 13
visible_ids =
Map.values(u1) ++
Map.values(u3) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u2[:r3],
u4[:r1],
u4[:r2]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline with reply_visibility `following`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "following")
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 11
visible_ids =
Map.values(u1) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u2[:r3],
u3[:r1],
u4[:r1],
u4[:r2]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
test "home timeline with reply_visibility `self`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "self")
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 9
visible_ids =
Map.values(u1) ++
[
activities[:a1],
activities[:a2],
activities[:a4],
u2[:r1],
u3[:r1],
u4[:r1]
]
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
end
describe "replies filtering with private messages" do
setup :private_messages
test "public timeline", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "public timeline with default reply_visibility `following`", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "following")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "public timeline with default reply_visibility `self`", %{users: %{u1: user}} do
activities_ids =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", false)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("reply_visibility", "self")
|> Map.put("user", user)
|> ActivityPub.fetch_public_activities()
|> Enum.map(& &1.id)
assert activities_ids == []
end
test "home timeline", %{users: %{u1: user}} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 12
end
test "home timeline with default reply_visibility `following`", %{users: %{u1: user}} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "following")
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 12
end
test "home timeline with default reply_visibility `self`", %{
users: %{u1: user},
activities: activities,
u1: u1,
u2: u2,
u3: u3,
u4: u4
} do
params =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("reply_visibility", "self")
activities_ids =
ActivityPub.fetch_activities([user.ap_id | User.following(user)], params)
|> Enum.map(& &1.id)
assert length(activities_ids) == 10
visible_ids =
Map.values(u1) ++ Map.values(u4) ++ [u2[:r1], u3[:r1]] ++ Map.values(activities)
assert Enum.all?(visible_ids, &(&1 in activities_ids))
end
end
defp public_messages(_) do
[u1, u2, u3, u4] = insert_list(4, :user)
{:ok, u1} = User.follow(u1, u2)
{:ok, u2} = User.follow(u2, u1)
{:ok, u1} = User.follow(u1, u4)
{:ok, u4} = User.follow(u4, u1)
{:ok, u2} = User.follow(u2, u3)
{:ok, u3} = User.follow(u3, u2)
{:ok, a1} = CommonAPI.post(u1, %{"status" => "Status"})
{:ok, r1_1} =
CommonAPI.post(u2, %{
"status" => "@#{u1.nickname} reply from u2 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, r1_2} =
CommonAPI.post(u3, %{
"status" => "@#{u1.nickname} reply from u3 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, r1_3} =
CommonAPI.post(u4, %{
"status" => "@#{u1.nickname} reply from u4 to u1",
"in_reply_to_status_id" => a1.id
})
{:ok, a2} = CommonAPI.post(u2, %{"status" => "Status"})
{:ok, r2_1} =
CommonAPI.post(u1, %{
"status" => "@#{u2.nickname} reply from u1 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, r2_2} =
CommonAPI.post(u3, %{
"status" => "@#{u2.nickname} reply from u3 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, r2_3} =
CommonAPI.post(u4, %{
"status" => "@#{u2.nickname} reply from u4 to u2",
"in_reply_to_status_id" => a2.id
})
{:ok, a3} = CommonAPI.post(u3, %{"status" => "Status"})
{:ok, r3_1} =
CommonAPI.post(u1, %{
"status" => "@#{u3.nickname} reply from u1 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, r3_2} =
CommonAPI.post(u2, %{
"status" => "@#{u3.nickname} reply from u2 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, r3_3} =
CommonAPI.post(u4, %{
"status" => "@#{u3.nickname} reply from u4 to u3",
"in_reply_to_status_id" => a3.id
})
{:ok, a4} = CommonAPI.post(u4, %{"status" => "Status"})
{:ok, r4_1} =
CommonAPI.post(u1, %{
"status" => "@#{u4.nickname} reply from u1 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok, r4_2} =
CommonAPI.post(u2, %{
"status" => "@#{u4.nickname} reply from u2 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok, r4_3} =
CommonAPI.post(u3, %{
"status" => "@#{u4.nickname} reply from u3 to u4",
"in_reply_to_status_id" => a4.id
})
{:ok,
users: %{u1: u1, u2: u2, u3: u3, u4: u4},
activities: %{a1: a1.id, a2: a2.id, a3: a3.id, a4: a4.id},
u1: %{r1: r1_1.id, r2: r1_2.id, r3: r1_3.id},
u2: %{r1: r2_1.id, r2: r2_2.id, r3: r2_3.id},
u3: %{r1: r3_1.id, r2: r3_2.id, r3: r3_3.id},
u4: %{r1: r4_1.id, r2: r4_2.id, r3: r4_3.id}}
end
defp private_messages(_) do
[u1, u2, u3, u4] = insert_list(4, :user)
{:ok, u1} = User.follow(u1, u2)
{:ok, u2} = User.follow(u2, u1)
{:ok, u1} = User.follow(u1, u3)
{:ok, u3} = User.follow(u3, u1)
{:ok, u1} = User.follow(u1, u4)
{:ok, u4} = User.follow(u4, u1)
{:ok, u2} = User.follow(u2, u3)
{:ok, u3} = User.follow(u3, u2)
{:ok, a1} = CommonAPI.post(u1, %{"status" => "Status", "visibility" => "private"})
{:ok, r1_1} =
CommonAPI.post(u2, %{
"status" => "@#{u1.nickname} reply from u2 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, r1_2} =
CommonAPI.post(u3, %{
"status" => "@#{u1.nickname} reply from u3 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, r1_3} =
CommonAPI.post(u4, %{
"status" => "@#{u1.nickname} reply from u4 to u1",
"in_reply_to_status_id" => a1.id,
"visibility" => "private"
})
{:ok, a2} = CommonAPI.post(u2, %{"status" => "Status", "visibility" => "private"})
{:ok, r2_1} =
CommonAPI.post(u1, %{
"status" => "@#{u2.nickname} reply from u1 to u2",
"in_reply_to_status_id" => a2.id,
"visibility" => "private"
})
{:ok, r2_2} =
CommonAPI.post(u3, %{
"status" => "@#{u2.nickname} reply from u3 to u2",
"in_reply_to_status_id" => a2.id,
"visibility" => "private"
})
{:ok, a3} = CommonAPI.post(u3, %{"status" => "Status", "visibility" => "private"})
{:ok, r3_1} =
CommonAPI.post(u1, %{
"status" => "@#{u3.nickname} reply from u1 to u3",
"in_reply_to_status_id" => a3.id,
"visibility" => "private"
})
{:ok, r3_2} =
CommonAPI.post(u2, %{
"status" => "@#{u3.nickname} reply from u2 to u3",
"in_reply_to_status_id" => a3.id,
"visibility" => "private"
})
{:ok, a4} = CommonAPI.post(u4, %{"status" => "Status", "visibility" => "private"})
{:ok, r4_1} =
CommonAPI.post(u1, %{
"status" => "@#{u4.nickname} reply from u1 to u4",
"in_reply_to_status_id" => a4.id,
"visibility" => "private"
})
{:ok,
users: %{u1: u1, u2: u2, u3: u3, u4: u4},
activities: %{a1: a1.id, a2: a2.id, a3: a3.id, a4: a4.id},
u1: %{r1: r1_1.id, r2: r1_2.id, r3: r1_3.id},
u2: %{r1: r2_1.id, r2: r2_2.id},
u3: %{r1: r3_1.id, r2: r3_2.id},
u4: %{r1: r4_1.id}}
end
end