From c60a5405db7c0bdb4f837e14088a495c1b741bc6 Mon Sep 17 00:00:00 2001 From: csaurus Date: Thu, 10 May 2018 22:17:59 -0400 Subject: [PATCH 1/9] Detect and try to stream incoming "direct" messages --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 ++++++++++----- lib/pleroma/web/mastodon_api/mastodon_socket.ex | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fde6e12d7..38e3a84fb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -42,13 +42,18 @@ def insert(map, local \\ true) when is_map(map) do def stream_out(activity) do if activity.data["type"] in ["Create", "Announce"] do Pleroma.Web.Streamer.stream("user", activity) + direct? = activity.data["object"]["visibility"] == "direct" - if Enum.member?(activity.data["to"], "https://www.w3.org/ns/activitystreams#Public") do - Pleroma.Web.Streamer.stream("public", activity) + cond do + direct? -> + Pleroma.Web.Streamer.stream("direct", activity) - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end + Enum.member?(activity.data["to"], "https://www.w3.org/ns/activitystreams#Public") -> + Pleroma.Web.Streamer.stream("public", activity) + + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end end end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_socket.ex b/lib/pleroma/web/mastodon_api/mastodon_socket.ex index f3e062941..080f62b31 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_socket.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_socket.ex @@ -15,7 +15,7 @@ def connect(params, socket) do with token when not is_nil(token) <- params["access_token"], %Token{user_id: user_id} <- Repo.get_by(Token, token: token), %User{} = user <- Repo.get(User, user_id), - stream when stream in ["public", "public:local", "user"] <- params["stream"] do + stream when stream in ["public", "public:local", "user", "direct"] <- params["stream"] do socket = socket |> assign(:topic, params["stream"]) From 392bd9ef562a98bb026d49ddd83f108cbe9fdb19 Mon Sep 17 00:00:00 2001 From: csaurus Date: Thu, 10 May 2018 22:15:42 -0400 Subject: [PATCH 2/9] Stream function to handle direct messages. --- lib/pleroma/web/activity_pub/activity_pub.ex | 14 ++++++-------- lib/pleroma/web/streamer.ex | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 38e3a84fb..8c1ba1ea3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -42,18 +42,16 @@ def insert(map, local \\ true) when is_map(map) do def stream_out(activity) do if activity.data["type"] in ["Create", "Announce"] do Pleroma.Web.Streamer.stream("user", activity) - direct? = activity.data["object"]["visibility"] == "direct" - cond do - direct? -> - Pleroma.Web.Streamer.stream("direct", activity) + visibility = Pleroma.Web.MastodonAPI.StatusView.get_visibility(activity.data["object"]) - Enum.member?(activity.data["to"], "https://www.w3.org/ns/activitystreams#Public") -> + case visibility do + "public" -> Pleroma.Web.Streamer.stream("public", activity) + if activity.local, do: Pleroma.Web.Streamer.stream("public:local", activity) - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end + "direct" -> + Pleroma.Web.Streamer.stream("direct", activity) end end end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 33041ec12..6aac472dc 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -46,6 +46,19 @@ def handle_cast(%{action: :ping}, topics) do {:noreply, topics} end + def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics || [], fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(topics, user_topic, item) + end) + + {:noreply, topics} + end + def handle_cast(%{action: :stream, topic: "user", item: %Notification{} = item}, topics) do topic = "user:#{item.user_id}" @@ -137,8 +150,8 @@ def push_to_socket(topics, topic, item) do end) end - defp internal_topic("user", socket) do - "user:#{socket.assigns[:user].id}" + defp internal_topic(topic, socket) when topic in ~w[user, direct] do + "#{topic}:#{socket.assigns[:user].id}" end defp internal_topic(topic, _), do: topic From c8d418acddd72e628caad9a6b11ff6debd3386e9 Mon Sep 17 00:00:00 2001 From: csaurus Date: Thu, 10 May 2018 22:17:33 -0400 Subject: [PATCH 3/9] api/v1/timelines/direct implementation --- lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 9 ++++++++ lib/pleroma/web/router.ex | 2 ++ 3 files changed, 34 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8c1ba1ea3..d5d00d4a2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -224,6 +224,28 @@ def fetch_public_activities(opts \\ %{}) do |> Enum.reverse() end + @valid_visibilities ~w[direct unlisted public private] + + defp restrict_visibility(query, %{visibility: "direct"}) do + public = "https://www.w3.org/ns/activitystreams#Public" + + from( + activity in query, + join: sender in User, + on: sender.ap_id == activity.actor, + where: + fragment("not data->'to' \\? ?", ^public) and fragment("not data->'cc' \\? ?", ^public) and + fragment("not data->'to' \\? ?", sender.follower_address) + ) + end + + defp restrict_visibility(_query, %{visibility: visibility}) + when visibility not in @valid_visibilities do + Logger.error("Could not restrict visibility to #{visibility}") + end + + defp restrict_visibility(query, _visibility), do: query + defp restrict_since(query, %{"since_id" => since_id}) do from(activity in query, where: activity.id > ^since_id) end @@ -347,6 +369,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_recent(opts) |> restrict_blocked(opts) |> restrict_media(opts) + |> restrict_visibility(opts) end def fetch_activities(recipients, opts \\ %{}) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 9f4261143..d190cdc3f 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -227,6 +227,15 @@ def user_statuses(%{assigns: %{user: user}} = conn, params) do end end + def dm_timeline(%{assigns: %{user: user}} = conn, params) do + query = ActivityPub.fetch_activities_query([user.ap_id], %{visibility: "direct"}) + activities = Repo.all(query) + + conn + |> add_link_headers(:user_statuses, activities, user.ap_id) + |> render(StatusView, "index.json", %{activities: activities, for: user, as: :activity}) + end + def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Repo.get(Activity, id), true <- ActivityPub.visible_for_user?(activity, user) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c025dea33..dbef19d44 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -106,6 +106,8 @@ def user_fetcher(username) do get("/timelines/home", MastodonAPIController, :home_timeline) + get("/timelines/direct", MastodonAPIController, :dm_timeline) + get("/favourites", MastodonAPIController, :favourites) post("/statuses", MastodonAPIController, :post_status) From 9aabff48835d55cf284229298ad86c07fa1fce30 Mon Sep 17 00:00:00 2001 From: csaurus Date: Sun, 13 May 2018 15:33:59 -0400 Subject: [PATCH 4/9] Fix tests. --- lib/pleroma/web/activity_pub/activity_pub.ex | 21 ++++++++++++------- .../web/mastodon_api/views/status_view.ex | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d5d00d4a2..4b2ecfa3c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -40,18 +40,23 @@ def insert(map, local \\ true) when is_map(map) do end def stream_out(activity) do + public = "https://www.w3.org/ns/activitystreams#Public" + if activity.data["type"] in ["Create", "Announce"] do Pleroma.Web.Streamer.stream("user", activity) - visibility = Pleroma.Web.MastodonAPI.StatusView.get_visibility(activity.data["object"]) + if Enum.member?(activity.data["to"], public) do + Pleroma.Web.Streamer.stream("public", activity) - case visibility do - "public" -> - Pleroma.Web.Streamer.stream("public", activity) - if activity.local, do: Pleroma.Web.Streamer.stream("public:local", activity) - - "direct" -> - Pleroma.Web.Streamer.stream("direct", activity) + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end + else + if !Enum.member?(activity.data["cc"] || [], public) && + !Enum.member?( + activity.data["to"], + User.get_by_ap_id(activity.data["actor"]).follower_address + ), do: Pleroma.Web.Streamer.stream("direct", activity) end end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5c6fd05f3..301234412 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -195,6 +195,7 @@ def get_visibility(object) do cond do public in to -> "public" public in cc -> "unlisted" + # this should use the sql for the object's activity Enum.any?(to, &String.contains?(&1, "/followers")) -> "private" true -> "direct" end From 2ce48c1a42b0e61958b513df610a81bb89b4254a Mon Sep 17 00:00:00 2001 From: csaurus Date: Sun, 13 May 2018 15:36:41 -0400 Subject: [PATCH 5/9] Formatting --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- .../web/mastodon_api/views/status_view.ex | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4b2ecfa3c..f7f6d047e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -56,7 +56,8 @@ def stream_out(activity) do !Enum.member?( activity.data["to"], User.get_by_ap_id(activity.data["actor"]).follower_address - ), do: Pleroma.Web.Streamer.stream("direct", activity) + ), + do: Pleroma.Web.Streamer.stream("direct", activity) end end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 301234412..d1d48cd0a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -193,11 +193,18 @@ def get_visibility(object) do cc = object["cc"] || [] cond do - public in to -> "public" - public in cc -> "unlisted" - # this should use the sql for the object's activity - Enum.any?(to, &String.contains?(&1, "/followers")) -> "private" - true -> "direct" + public in to -> + "public" + + public in cc -> + "unlisted" + + # this should use the sql for the object's activity + Enum.any?(to, &String.contains?(&1, "/followers")) -> + "private" + + true -> + "direct" end end end From 987a52cf6f987e5178194615637753c9619e2e5d Mon Sep 17 00:00:00 2001 From: csaurus Date: Sun, 13 May 2018 21:21:42 -0400 Subject: [PATCH 6/9] Post direct status test --- .../mastodon_api_controller_test.exs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 883ebc61e..71a6eed8d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -124,6 +124,22 @@ test "posting a sensitive status", %{conn: conn} do assert Repo.get(Activity, id) end + test "posting a direct status", %{conn: conn} do + user1 = insert(:user) + user2 = insert(:user) + content = "direct cofe @#{user2.nickname}" + + conn = + conn + |> assign(:user, user1) + |> post("api/v1/statuses", %{"status" => content, + "visibility" => "direct"}) + + assert %{"content" => content, "id" => id, "visibility" => "direct"} = json_response(conn, 200) + assert activity = Repo.get(Activity, id) + assert user2.follower_address not in activity.data["to"] + end + test "replying to a status", %{conn: conn} do user = insert(:user) From 4dfb40a5467f6206b2793bdafbd82a7ae4ee04bf Mon Sep 17 00:00:00 2001 From: csaurus Date: Mon, 14 May 2018 21:46:09 -0400 Subject: [PATCH 7/9] Handle cases where a to/cc field is absent on a status --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f7f6d047e..4ce2e6052 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -239,9 +239,11 @@ defp restrict_visibility(query, %{visibility: "direct"}) do activity in query, join: sender in User, on: sender.ap_id == activity.actor, + # Are non-direct statuses with no to/cc possible? where: - fragment("not data->'to' \\? ?", ^public) and fragment("not data->'cc' \\? ?", ^public) and - fragment("not data->'to' \\? ?", sender.follower_address) + fragment("not coalesce(data->'to' \\? ?, false)", ^public) and + fragment("not coalesce(data->'cc' \\? ?, false)", ^public) and + fragment("not coalesce(data->'to' \\? ?, false)", sender.follower_address) ) end From d0ad13c12e1410e7a11d5a5f7f5b84cad5f77732 Mon Sep 17 00:00:00 2001 From: csaurus Date: Mon, 14 May 2018 21:48:06 -0400 Subject: [PATCH 8/9] Add direct_note[_activity]_factory and a couple tests --- test/support/factory.ex | 27 +++++++++++++ .../mastodon_api_controller_test.exs | 40 ++++++++++++++----- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 8e21e2562..47626cb3e 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -45,6 +45,33 @@ def note_factory do } end + def direct_note_factory do + user2 = insert(:user) + + %Pleroma.Object{data: data} = note_factory() + %Pleroma.Object{data: Map.merge(data, %{"to" => [user2.ap_id]})} + end + + def direct_note_activity_factory do + dm = insert(:direct_note) + + data = %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Create", + "actor" => dm.data["actor"], + "to" => dm.data["to"], + "object" => dm.data, + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "context" => dm.data["context"] + } + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + end + def note_activity_factory do note = insert(:note) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 71a6eed8d..94131dcb3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -125,19 +125,37 @@ test "posting a sensitive status", %{conn: conn} do end test "posting a direct status", %{conn: conn} do - user1 = insert(:user) - user2 = insert(:user) - content = "direct cofe @#{user2.nickname}" + user1 = insert(:user) + user2 = insert(:user) + content = "direct cofe @#{user2.nickname}" - conn = - conn - |> assign(:user, user1) - |> post("api/v1/statuses", %{"status" => content, - "visibility" => "direct"}) + conn = + conn + |> assign(:user, user1) + |> post("api/v1/statuses", %{"status" => content, "visibility" => "direct"}) - assert %{"content" => content, "id" => id, "visibility" => "direct"} = json_response(conn, 200) - assert activity = Repo.get(Activity, id) - assert user2.follower_address not in activity.data["to"] + assert %{"id" => id, "visibility" => "direct"} = json_response(conn, 200) + assert activity = Repo.get(Activity, id) + assert user2.follower_address not in activity.data["to"] + end + + test "direct timeline", %{conn: conn} do + dm = insert(:direct_note_activity) + reg_note = insert(:note_activity) + + recipient = User.get_by_ap_id(hd(dm.recipients)) + + conn = + conn + |> assign(:user, recipient) + |> get("api/v1/timelines/direct") + + resp = json_response(conn, 200) + first_status = hd(resp) + + assert length(resp) == 1 + assert %{"visibility" => "direct"} = first_status + assert first_status["url"] != reg_note.data["id"] end test "replying to a status", %{conn: conn} do From 841ee8e3e4d31d4236a022d46fe18f7751605c74 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 26 May 2018 16:25:32 +0200 Subject: [PATCH 9/9] Simplify DM query. Should also use indexes better. --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++-- .../mastodon_api_controller_test.exs | 43 ++++++++++++++----- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d54dc224d..4e0be5ba2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -313,9 +313,11 @@ defp restrict_visibility(query, %{visibility: "direct"}) do on: sender.ap_id == activity.actor, # Are non-direct statuses with no to/cc possible? where: - fragment("not coalesce(data->'to' \\? ?, false)", ^public) and - fragment("not coalesce(data->'cc' \\? ?, false)", ^public) and - fragment("not coalesce(data->'to' \\? ?, false)", sender.follower_address) + fragment( + "not (? && ?)", + [^public, sender.follower_address], + activity.recipients + ) ) end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 553581be4..2abcf0dfe 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -136,26 +136,47 @@ test "posting a direct status", %{conn: conn} do assert %{"id" => id, "visibility" => "direct"} = json_response(conn, 200) assert activity = Repo.get(Activity, id) - assert user2.follower_address not in activity.data["to"] + assert activity.recipients == [user2.ap_id] + assert activity.data["to"] == [user2.ap_id] + assert activity.data["cc"] == [] end test "direct timeline", %{conn: conn} do - dm = insert(:direct_note_activity) - reg_note = insert(:note_activity) + user_one = insert(:user) + user_two = insert(:user) - recipient = User.get_by_ap_id(hd(dm.recipients)) + {:ok, user_two} = User.follow(user_two, user_one) - conn = + {:ok, direct} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}!", + "visibility" => "direct" + }) + + {:ok, _follower_only} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}!", + "visibility" => "private" + }) + + # Only direct should be visible here + res_conn = conn - |> assign(:user, recipient) + |> assign(:user, user_two) |> get("api/v1/timelines/direct") - resp = json_response(conn, 200) - first_status = hd(resp) + [status] = json_response(res_conn, 200) - assert length(resp) == 1 - assert %{"visibility" => "direct"} = first_status - assert first_status["url"] != reg_note.data["id"] + assert %{"visibility" => "direct"} = status + assert status["url"] != direct.data["id"] + + # Both should be visible here + res_conn = + conn + |> assign(:user, user_two) + |> get("api/v1/timelines/home") + + [_s1, _s2] = json_response(res_conn, 200) end test "replying to a status", %{conn: conn} do