From afa8b469ed0a71247f27efec08d6eeac24b6674f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 1 Jul 2020 21:12:59 -0500 Subject: [PATCH 1/5] Allow restricting public timeline by instance --- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++---------- .../web/api_spec/operations/timeline_operation.ex | 10 ++++++++++ .../mastodon_api/controllers/timeline_controller.ex | 1 + .../controllers/timeline_controller_test.exs | 12 ++++++++++++ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bc7b5d95a..9ce2b04dd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -927,16 +927,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp restrict_instance(query, %{instance: instance}) do - users = - from( - u in User, - select: u.ap_id, - where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}") - ) - |> Repo.all() - - from(activity in query, where: activity.actor in ^users) + defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do + from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%")) end defp restrict_instance(query, _), do: query diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 8e19bace7..83cdbad69 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do security: [%{"oAuth" => ["read:statuses"]}], parameters: [ local_param(), + instance_param(), only_media_param(), with_muted_param(), exclude_visibilities_param(), @@ -158,6 +159,15 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do ) end + defp instance_param do + Operation.parameter( + :instance, + :query, + %Schema{type: :string}, + "Show only statuses from the given domain" + ) + end + defp with_muted_param do Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users") end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index ab7b1d6aa..7dccc0005 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -110,6 +110,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do |> Map.put(:blocking_user, user) |> Map.put(:muting_user, user) |> Map.put(:reply_filtering_user, user) + |> Map.put(:instance, params[:instance]) |> ActivityPub.fetch_public_activities() conn diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index 50e0d783d..6acd512c7 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -140,6 +140,18 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do activities = json_response_and_validate_schema(res_conn, 200) [%{"id" => ^activity_id}] = activities end + + test "can be filtered by instance", %{conn: conn} do + user = insert(:user, ap_id: "https://lain.com/users/lain") + insert(:note_activity, local: false) + insert(:note_activity, local: false) + + {:ok, _} = CommonAPI.post(user, %{status: "test"}) + + conn = get(conn, "/api/v1/timelines/public?instance=lain.com") + + assert length(json_response_and_validate_schema(conn, :ok)) == 1 + end end defp local_and_remote_activities do From ad9c925efb77287316f5dbac26f6a1b16662910a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 5 Aug 2020 13:08:13 -0500 Subject: [PATCH 2/5] Speed up instance timeline query --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++++- test/web/admin_api/controllers/admin_api_controller_test.exs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9ce2b04dd..76fc9c3ee 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -928,7 +928,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do - from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%")) + from( + activity in query, + where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance) + ) end defp restrict_instance(query, _), do: query diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index da91cd552..26194eb81 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1647,8 +1647,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do describe "instances" do test "GET /instances/:instance/statuses", %{conn: conn} do - user = insert(:user, local: false, nickname: "archaeme@archae.me") - user2 = insert(:user, local: false, nickname: "test@test.com") + user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") + user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") insert_pair(:note_activity, user: user) activity = insert(:note_activity, user: user2) From 24ce9c011caf7401fb261c7df4196b2ef9ba3d90 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 5 Aug 2020 19:33:51 +0000 Subject: [PATCH 3/5] Apply 1 suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/timeline_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 83cdbad69..95720df9f 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -169,7 +169,7 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do end defp with_muted_param do - Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users") + Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users") end defp exclude_visibilities_param do From 9c672ecbb5d4477cd16d2139a2cb66d3923ac5c8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 8 Oct 2020 20:01:48 -0500 Subject: [PATCH 4/5] Remote Timeline: add Streaming support --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 6 ++++++ lib/pleroma/activity/ir/topics.ex | 13 ++++++++++++- lib/pleroma/web/streamer/streamer.ex | 9 +++++++++ test/activity/ir/topics_test.exs | 14 ++++++++++++++ test/integration/mastodon_websocket_test.exs | 1 + test/web/streamer/streamer_test.exs | 8 ++++++++ 7 files changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..0eeffb72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media` ### Changed diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 38865dc68..bb1000b0b 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -9,9 +9,13 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas ## Timelines Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users. + Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`. + Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you. +Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance). + ## Statuses - `visibility`: has an additional possible value `list` @@ -249,6 +253,8 @@ Has these additional fields under the `pleroma` object: There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field. +For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`. + ## Not implemented Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority. diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index 9e65bedad..fe2e8cb5c 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -40,7 +40,8 @@ defmodule Pleroma.Activity.Ir.Topics do end defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do - tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + tags ++ + remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) end defp item_creation_tags(tags, _, _) do @@ -55,9 +56,19 @@ defmodule Pleroma.Activity.Ir.Topics do defp hashtags_to_topics(_), do: [] + defp remote_topics(%{local: true}), do: [] + + defp remote_topics(%{actor: actor}) when is_binary(actor), + do: ["public:remote:" <> URI.parse(actor).host] + + defp remote_topics(_), do: [] + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + defp attachment_topics(_object, %{actor: actor}) when is_binary(actor), + do: ["public:media", "public:remote:media:" <> URI.parse(actor).host] + defp attachment_topics(_object, _act), do: ["public:media"] end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 5475f18a6..d774f0dd9 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -57,6 +57,15 @@ defmodule Pleroma.Web.Streamer do {:ok, "hashtag:" <> tag} end + # Allow remote instance streams. + def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do + {:ok, "public:remote:" <> instance} + end + + def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do + {:ok, "public:remote:media:" <> instance} + end + # Expand user streams. def get_topic( stream, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs index 14a6e6b71..c8dcb28cc 100644 --- a/test/activity/ir/topics_test.exs +++ b/test/activity/ir/topics_test.exs @@ -93,6 +93,13 @@ defmodule Pleroma.Activity.Ir.TopicsTest do refute Enum.member?(topics, "hashtag:2") end + + test "non-local action produces public:remote topic", %{activity: activity} do + activity = %{activity | local: false, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:remote:lain.com") + end end describe "public visibility create events with attachments" do @@ -124,6 +131,13 @@ defmodule Pleroma.Activity.Ir.TopicsTest do refute Enum.member?(topics, "public:local:media") end + + test "non-local action produces public:remote:media topic", %{activity: activity} do + activity = %{activity | local: false, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:remote:media:lain.com") + end end describe "non-public visibility" do diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 0f2e6cc2b..bb8e795b7 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -49,6 +49,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") + assert {:ok, _} = start_socket("?stream=public:remote&instance=lain.com") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 185724a9f..1495ed124 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -29,6 +29,14 @@ defmodule Pleroma.Web.StreamerTest do assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil) end + test "allows instance streams" do + assert {:ok, "public:remote:lain.com"} = + Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) + + assert {:ok, "public:remote:media:lain.com"} = + Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"}) + end + test "allows hashtag streams" do assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"}) end From cb3cd3a761d96a08b1b55f5b277795822aa7e1d7 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 4 Nov 2020 15:24:10 +0100 Subject: [PATCH 5/5] TopicsTest: Small addition. --- test/pleroma/activity/ir/topics_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 2e5655334..5e5c2f8da 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -104,6 +104,13 @@ defmodule Pleroma.Activity.Ir.TopicsTest do assert Enum.member?(topics, "public:remote:lain.com") end + + test "local action doesn't produce public:remote topic", %{activity: activity} do + activity = %{activity | local: true, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:remote:lain.com") + end end describe "public visibility create events with attachments" do