From 27c33f216ad250b60d44fe0662c3be3c4cee987e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 05:39:35 +0200 Subject: [PATCH 01/10] activity_draft: Add source field --- lib/pleroma/web/common_api/activity_draft.ex | 1 + test/web/common_api/common_api_test.exs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 9bcb9f587..f849b2e01 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -186,6 +186,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do draft.poll ) |> Map.put("emoji", emoji) + |> Map.put("source", draft.status) %__MODULE__{draft | object: object} end diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 6bd26050e..cbdd994a9 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -445,6 +445,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) assert object.data["content"] == "

2hu

alert('xss')" + assert object.data["source"] == post end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -461,6 +462,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) assert object.data["content"] == "

2hu

alert('xss')" + assert object.data["source"] == post end test "it does not allow replies to direct messages that are not direct messages themselves" do From 244655e884130df6dccabc0d2d78d33857809a36 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 07:16:24 +0200 Subject: [PATCH 02/10] MastoAPI: Show source field when deleting --- .../web/api_spec/operations/status_operation.ex | 2 +- lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++ .../mastodon_api/controllers/status_controller.ex | 15 +++++++++++---- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + test/support/factory.ex | 1 + .../controllers/status_controller_test.exs | 11 ++++++++--- test/web/mastodon_api/views/status_view_test.exs | 1 + 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 0b7fad793..5bd4619d5 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do operationId: "StatusController.delete", parameters: [id_param()], responses: %{ - 200 => empty_object_response(), + 200 => status_response(), 403 => Operation.response("Forbidden", "application/json", ApiError), 404 => Operation.response("Not Found", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 8b87cb25b..a38b5b40f 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -62,6 +62,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do } }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, + text: %Schema{ + type: :string, + description: "Original unformatted content in plain text", + nullable: true + }, created_at: %Schema{ type: :string, format: "date-time", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 468b44b67..3f4c53437 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -200,11 +200,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "DELETE /api/v1/statuses/:id" def delete(%{assigns: %{user: user}} = conn, %{id: id}) do - with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do - json(conn, %{}) + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + render <- + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_source: true + ), + {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + render else - {:error, :not_found} = e -> e - _e -> render_error(conn, :forbidden, "Can't delete this post") + _e -> {:error, :not_found} end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2c49bedb3..4df47f584 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -333,6 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblog: nil, card: card, content: content_html, + text: opts[:with_source] && object.data["source"], created_at: created_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, diff --git a/test/support/factory.ex b/test/support/factory.ex index 6e22b66a4..af580021c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -67,6 +67,7 @@ defmodule Pleroma.Factory do data = %{ "type" => "Note", "content" => text, + "source" => text, "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), "actor" => user.ap_id, "to" => ["https://www.w3.org/ns/activitystreams#Public"], diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index a98e939e8..fd2de8d80 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -760,13 +760,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do test "when you created it" do %{user: author, conn: conn} = oauth_access(["write:statuses"]) activity = insert(:note_activity, user: author) + object = Object.normalize(activity) - conn = + content = object.data["content"] + source = object.data["source"] + + result = conn |> assign(:user, author) |> delete("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) - assert %{} = json_response_and_validate_schema(conn, 200) + assert match?(%{"content" => ^content, "text" => ^source}, result) refute Activity.get_by_id(activity.id) end @@ -789,7 +794,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do conn = delete(conn, "/api/v1/statuses/#{activity.id}") - assert %{"error" => _} = json_response_and_validate_schema(conn, 403) + assert %{"error" => "Record not found"} == json_response_and_validate_schema(conn, 404) assert Activity.get_by_id(activity.id) == activity end diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index 5cbadf0fc..b6ae4d343 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -183,6 +183,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do card: nil, reblog: nil, content: HTML.filter_tags(object_data["content"]), + text: nil, created_at: created_at, reblogs_count: 0, replies_count: 0, From 3d2989278c2f97fb5247d0b58b99b77f400f3185 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 30 Jun 2020 21:26:39 +0300 Subject: [PATCH 03/10] [#1892] Excluded bot actors (applications, services) from search results. --- lib/pleroma/user/search.ex | 5 +++++ test/user_search_test.exs | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index cec59c372..0293c6ae7 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -52,6 +52,7 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_invisible_users() + |> filter_bots() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -109,6 +110,10 @@ defmodule Pleroma.User.Search do from(q in query, where: q.invisible == false) end + defp filter_bots(query) do + from(q in query, where: q.actor_type not in ["Application", "Service"]) + end + defp filter_blocked_user(query, %User{} = blocker) do query |> join(:left, [u], b in Pleroma.UserRelationship, diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 17c63322a..9a74b9764 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.UserSearchTest do describe "User.search" do setup do: clear_config([:instance, :limit_to_local_content]) - test "excluded invisible users from results" do + test "excludes invisible users from results" do user = insert(:user, %{nickname: "john t1000"}) insert(:user, %{invisible: true, nickname: "john t800"}) @@ -25,6 +25,13 @@ defmodule Pleroma.UserSearchTest do assert found_user.id == user.id end + test "excludes bots from results" do + insert(:user, actor_type: "Service", nickname: "bot1") + insert(:user, actor_type: "Application", nickname: "bot2") + + assert [] = User.search("bot") + end + test "accepts limit parameter" do Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) assert length(User.search("john", limit: 3)) == 3 From 90764670dc83c39c28cd7851f08f77f1e8bcf25a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 3 Jul 2020 11:02:15 +0300 Subject: [PATCH 04/10] [#1892] Excluded internal users (applications) from user search results, reinstated service actors in search results. --- lib/pleroma/user/search.ex | 6 +++--- test/user_search_test.exs | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 0293c6ae7..42ff1de78 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -52,7 +52,7 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_invisible_users() - |> filter_bots() + |> filter_internal_users() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -110,8 +110,8 @@ defmodule Pleroma.User.Search do from(q in query, where: q.invisible == false) end - defp filter_bots(query) do - from(q in query, where: q.actor_type not in ["Application", "Service"]) + defp filter_internal_users(query) do + from(q in query, where: q.actor_type != "Application") end defp filter_blocked_user(query, %User{} = blocker) do diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 9a74b9764..f030523d3 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -25,11 +25,13 @@ defmodule Pleroma.UserSearchTest do assert found_user.id == user.id end - test "excludes bots from results" do - insert(:user, actor_type: "Service", nickname: "bot1") - insert(:user, actor_type: "Application", nickname: "bot2") + test "excludes service actors from results" do + insert(:user, actor_type: "Application", nickname: "user1") + service = insert(:user, actor_type: "Service", nickname: "user2") + person = insert(:user, actor_type: "Person", nickname: "user3") - assert [] = User.search("bot") + assert [found_user1, found_user2] = User.search("user") + assert [found_user1.id, found_user2.id] -- [service.id, person.id] == [] end test "accepts limit parameter" do From 59b426ebefd1881181888a5b0e6abe8338b65d3f Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 11:25:12 +0200 Subject: [PATCH 05/10] Notification Backfill: Explicitly select the needed fields. Prevents a crashing migration when we change user fields. --- .../migration_helper/notification_backfill.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index b3770307a..d260e62ca 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.MigrationHelper.NotificationBackfill do - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -25,18 +24,27 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do |> type_from_activity() notification - |> Notification.changeset(%{type: type}) + |> Ecto.Changeset.change(%{type: type}) |> Repo.update() end) end + defp get_by_ap_id(ap_id) do + q = + from(u in User, + select: u.id + ) + + Repo.get_by(q, ap_id: ap_id) + end + # This is copied over from Notifications to keep this stable. defp type_from_activity(%{data: %{"type" => type}} = activity) do case type do "Follow" -> accepted_function = fn activity -> - with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]), - %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do + with %User{} = follower <- get_by_ap_id(activity.data["actor"]), + %User{} = followed <- get_by_ap_id(activity.data["object"]) do Pleroma.FollowingRelationship.following?(follower, followed) end end From 945e75c8e8f05fadd669c7aa084dd6ba7e9b5ab2 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 16:36:55 +0200 Subject: [PATCH 06/10] SearchController: Trim query. --- .../web/mastodon_api/controllers/search_controller.ex | 1 + test/web/mastodon_api/controllers/search_controller_test.exs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index e50980122..29affa7d5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do def search(conn, params), do: do_search(:v1, conn, params) defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do + query = String.trim(query) options = search_options(params, user) timeout = Keyword.get(Repo.config(), :timeout, 15_000) default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []} diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 826f37fbc..24d1959f8 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -79,6 +79,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert status["id"] == to_string(activity.id) end + @tag capture_log: true test "constructs hashtags from search query", %{conn: conn} do results = conn @@ -318,11 +319,13 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do test "search fetches remote accounts", %{conn: conn} do user = insert(:user) + query = URI.encode_query(%{q: " mike@osada.macgirvin.com ", resolve: true}) + results = conn |> assign(:user, user) |> assign(:token, insert(:oauth_token, user: user, scopes: ["read"])) - |> get("/api/v1/search?q=mike@osada.macgirvin.com&resolve=true") + |> get("/api/v1/search?#{query}") |> json_response_and_validate_schema(200) [account] = results["accounts"] From cbf2fe9649da34e78ddbc0f11c3fcc2599aa1c7a Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 16:46:11 +0200 Subject: [PATCH 07/10] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f878a76..85401809a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Emoji API: changed methods and renamed routes. - Streaming: Repeats of a user's posts will no longer be pushed to the user's stream. - Mastodon API: Added `pleroma.metadata.fields_limits` to /api/v1/instance +- Mastodon API: On deletion, returns the original post text.
From cf566556147975d45958d2d87a5ce23831eb91df Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 Jul 2020 17:11:37 +0200 Subject: [PATCH 08/10] Streamer: Don't filter out announce notifications. --- lib/pleroma/web/streamer/streamer.ex | 12 ++++++++---- test/web/streamer/streamer_test.exs | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 73ee3e1e1..d1d70e556 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -104,7 +104,9 @@ defmodule Pleroma.Web.Streamer do :ok end - def filtered_by_user?(%User{} = user, %Activity{} = item) do + def filtered_by_user?(user, item, streamed_type \\ :activity) + + def filtered_by_user?(%User{} = user, %Activity{} = item, streamed_type) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) @@ -116,7 +118,9 @@ defmodule Pleroma.Web.Streamer do true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, - true <- !(item.data["type"] == "Announce" && parent.data["actor"] == user.ap_id), + true <- + !(streamed_type == :activity && item.data["type"] == "Announce" && + parent.data["actor"] == user.ap_id), true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(parent.data["actor"] not in &1)), true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), @@ -131,8 +135,8 @@ defmodule Pleroma.Web.Streamer do end end - def filtered_by_user?(%User{} = user, %Notification{activity: activity}) do - filtered_by_user?(user, activity) + def filtered_by_user?(%User{} = user, %Notification{activity: activity}, _) do + filtered_by_user?(user, activity, :notification) end defp do_stream("direct", item) do diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index dfe341b34..d56d74464 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -128,6 +128,23 @@ defmodule Pleroma.Web.StreamerTest do assert Streamer.filtered_by_user?(user, announce) end + test "it does stream notifications announces of the user's own posts in the 'user' stream", %{ + user: user + } do + Streamer.get_topic_and_add_socket("user", user) + + other_user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) + {:ok, announce} = CommonAPI.repeat(activity.id, other_user) + + notification = + Pleroma.Notification + |> Repo.get_by(%{user_id: user.id, activity_id: announce.id}) + |> Repo.preload(:activity) + + refute Streamer.filtered_by_user?(user, notification) + end + test "it streams boosts of mastodon user in the 'user' stream", %{user: user} do Streamer.get_topic_and_add_socket("user", user) From 480dfafa831245976a5c21940adca6f2a73c1213 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 Jul 2020 08:48:20 +0300 Subject: [PATCH 09/10] don't save tesla settings into db --- lib/pleroma/config/loader.ex | 8 +++++++- test/config/holder_test.exs | 5 +---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex index 0f3ecf1ed..64e7de6df 100644 --- a/lib/pleroma/config/loader.ex +++ b/lib/pleroma/config/loader.ex @@ -12,6 +12,11 @@ defmodule Pleroma.Config.Loader do :swarm ] + @reject_groups [ + :postgrex, + :tesla + ] + if Code.ensure_loaded?(Config.Reader) do @reader Config.Reader @@ -47,7 +52,8 @@ defmodule Pleroma.Config.Loader do @spec filter_group(atom(), keyword()) :: keyword() def filter_group(group, configs) do Enum.reject(configs[group], fn {key, _v} -> - key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex + key in @reject_keys or group in @reject_groups or + (group == :phoenix and key == :serve_endpoints) end) end end diff --git a/test/config/holder_test.exs b/test/config/holder_test.exs index 15d48b5c7..abcaa27dd 100644 --- a/test/config/holder_test.exs +++ b/test/config/holder_test.exs @@ -10,7 +10,6 @@ defmodule Pleroma.Config.HolderTest do test "default_config/0" do config = Holder.default_config() assert config[:pleroma][Pleroma.Uploaders.Local][:uploads] == "test/uploads" - assert config[:tesla][:adapter] == Tesla.Mock refute config[:pleroma][Pleroma.Repo] refute config[:pleroma][Pleroma.Web.Endpoint] @@ -18,17 +17,15 @@ defmodule Pleroma.Config.HolderTest do refute config[:pleroma][:configurable_from_database] refute config[:pleroma][:database] refute config[:phoenix][:serve_endpoints] + refute config[:tesla][:adapter] end test "default_config/1" do pleroma_config = Holder.default_config(:pleroma) assert pleroma_config[Pleroma.Uploaders.Local][:uploads] == "test/uploads" - tesla_config = Holder.default_config(:tesla) - assert tesla_config[:adapter] == Tesla.Mock end test "default_config/2" do assert Holder.default_config(:pleroma, Pleroma.Uploaders.Local) == [uploads: "test/uploads"] - assert Holder.default_config(:tesla, :adapter) == Tesla.Mock end end From 465ddcfd2090abbb18afd7f1f7f1a4ee30105668 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 Jul 2020 09:12:29 +0300 Subject: [PATCH 10/10] migration to delete migrated tesla setting --- .../20200706060258_remove_tesla_from_config.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20200706060258_remove_tesla_from_config.exs diff --git a/priv/repo/migrations/20200706060258_remove_tesla_from_config.exs b/priv/repo/migrations/20200706060258_remove_tesla_from_config.exs new file mode 100644 index 000000000..798687f8a --- /dev/null +++ b/priv/repo/migrations/20200706060258_remove_tesla_from_config.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.RemoveTeslaFromConfig do + use Ecto.Migration + + def up do + execute("DELETE FROM config WHERE config.group = ':tesla'") + end + + def down do + end +end