From 6a6c4d134b7564012d00e89f1236b904261ab5db Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 12 Jul 2019 21:02:55 +0545 Subject: [PATCH 01/15] preserve the original path/filename (no encoding/decoding) for proxy --- lib/pleroma/web/media_proxy/controller.ex | 7 +------ test/media_proxy_test.exs | 16 ++++------------ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/media_proxy/controller.ex b/lib/pleroma/web/media_proxy/controller.ex index c0552d89f..ea33d7685 100644 --- a/lib/pleroma/web/media_proxy/controller.ex +++ b/lib/pleroma/web/media_proxy/controller.ex @@ -28,12 +28,7 @@ def remote(conn, %{"sig" => sig64, "url" => url64} = params) do end def filename_matches(has_filename, path, url) do - filename = - url - |> MediaProxy.filename() - |> URI.decode() - - path = URI.decode(path) + filename = url |> MediaProxy.filename() if has_filename && filename && Path.basename(path) != filename do {:wrong_filename, filename} diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 1d6d170b7..fbf200931 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -88,10 +88,10 @@ test "validates signature" do assert decode_url(sig, base64) == {:error, :invalid_signature} end - test "filename_matches matches url encoded paths" do + test "filename_matches preserves the encoded or decoded path" do assert MediaProxyController.filename_matches( true, - "/Hello%20world.jpg", + "/Hello world.jpg", "http://pleroma.social/Hello world.jpg" ) == :ok @@ -100,19 +100,11 @@ test "filename_matches matches url encoded paths" do "/Hello%20world.jpg", "http://pleroma.social/Hello%20world.jpg" ) == :ok - end - - test "filename_matches matches non-url encoded paths" do - assert MediaProxyController.filename_matches( - true, - "/Hello world.jpg", - "http://pleroma.social/Hello%20world.jpg" - ) == :ok assert MediaProxyController.filename_matches( true, - "/Hello world.jpg", - "http://pleroma.social/Hello world.jpg" + "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg", + "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg" ) == :ok end From 27ed260eed51798a20608b1c062d32bfc7b6cdc4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 18:36:07 +0300 Subject: [PATCH 02/15] AP user view: Add a test for hiding totalItems in following/followers --- .../web/activity_pub/views/user_view_test.exs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 969860c4c..86254117f 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Web.CommonAPI test "Renders a user, including the public key" do user = insert(:user) @@ -82,4 +83,28 @@ test "instance users do not expose oAuth endpoints" do refute result["endpoints"]["oauthTokenEndpoint"] end end + + describe "followers" do + test "sets totalItems to zero when followers are hidden" do + user = insert(:user) + other_user = insert(:user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) + info = Map.put(user.info, :hide_followers, true) + user = Map.put(user, :info, info) + assert %{"totalItems" => 0} = UserView.render("followers.json", %{user: user}) + end + end + + describe "following" do + test "sets totalItems to zero when follows are hidden" do + user = insert(:user) + other_user = insert(:user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) + assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) + info = Map.put(user.info, :hide_follows, true) + user = Map.put(user, :info, info) + assert %{"totalItems" => 0} = UserView.render("following.json", %{user: user}) + end + end end From 360e4cdaa2708d54903765c61afbc5ea5f1b2cdb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 12 Jul 2019 11:25:58 -0500 Subject: [PATCH 03/15] Move these to pleroma namespace in Mastodon API --- docs/api/differences_in_mastoapi_responses.md | 8 -------- docs/api/pleroma_api.md | 7 +++++++ lib/pleroma/web/router.ex | 8 ++++---- .../mastodon_api/mastodon_api_controller_test.exs | 12 ++++++------ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 2cbe1458d..3ee7115cf 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -46,14 +46,6 @@ Has these additional fields under the `pleroma` object: - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` -### Extensions for PleromaFE - -These endpoints added for controlling PleromaFE features over the Mastodon API - -- PATCH `/api/v1/accounts/update_avatar`: Set/clear user avatar image -- PATCH `/api/v1/accounts/update_banner`: Set/clear user banner image -- PATCH `/api/v1/accounts/update_background`: Set/clear user background image - ### Source Has these additional fields under the `pleroma` object: diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index edc62727a..d83ebd734 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -238,6 +238,13 @@ See [Admin-API](Admin-API.md) ] ``` +## `/api/v1/pleroma/accounts/update_*` +### Set and clear account avatar, banner, and background + +- PATCH `/api/v1/pleroma/accounts/update_avatar`: Set/clear user avatar image +- PATCH `/api/v1/pleroma/accounts/update_banner`: Set/clear user banner image +- PATCH `/api/v1/pleroma/accounts/update_background`: Set/clear user background image + ## `/api/v1/pleroma/mascot` ### Gets user mascot image * Method `GET` diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d53fa8a35..a3b6ea366 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -322,10 +322,6 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", MastodonAPIController, :update_credentials) - patch("/accounts/update_avatar", MastodonAPIController, :update_avatar) - patch("/accounts/update_banner", MastodonAPIController, :update_banner) - patch("/accounts/update_background", MastodonAPIController, :update_background) - post("/statuses", MastodonAPIController, :post_status) delete("/statuses/:id", MastodonAPIController, :delete_status) @@ -360,6 +356,10 @@ defmodule Pleroma.Web.Router do put("/filters/:id", MastodonAPIController, :update_filter) delete("/filters/:id", MastodonAPIController, :delete_filter) + patch("/pleroma/accounts/update_avatar", MastodonAPIController, :update_avatar) + patch("/pleroma/accounts/update_banner", MastodonAPIController, :update_banner) + patch("/pleroma/accounts/update_background", MastodonAPIController, :update_background) + get("/pleroma/mascot", MastodonAPIController, :get_mascot) put("/pleroma/mascot", MastodonAPIController, :set_mascot) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 8afb1497b..b7c050dbf 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -593,7 +593,7 @@ test "user avatar can be set", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_avatar", %{img: avatar_image}) + |> patch("/api/v1/pleroma/accounts/update_avatar", %{img: avatar_image}) user = refresh_record(user) @@ -618,7 +618,7 @@ test "user avatar can be reset", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_avatar", %{img: ""}) + |> patch("/api/v1/pleroma/accounts/update_avatar", %{img: ""}) user = User.get_cached_by_id(user.id) @@ -633,7 +633,7 @@ test "can set profile banner", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_banner", %{"banner" => @image}) + |> patch("/api/v1/pleroma/accounts/update_banner", %{"banner" => @image}) user = refresh_record(user) assert user.info.banner["type"] == "Image" @@ -647,7 +647,7 @@ test "can reset profile banner", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_banner", %{"banner" => ""}) + |> patch("/api/v1/pleroma/accounts/update_banner", %{"banner" => ""}) user = refresh_record(user) assert user.info.banner == %{} @@ -661,7 +661,7 @@ test "background image can be set", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_background", %{"img" => @image}) + |> patch("/api/v1/pleroma/accounts/update_background", %{"img" => @image}) user = refresh_record(user) assert user.info.background["type"] == "Image" @@ -674,7 +674,7 @@ test "background image can be reset", %{conn: conn} do conn = conn |> assign(:user, user) - |> patch("/api/v1/accounts/update_background", %{"img" => ""}) + |> patch("/api/v1/pleroma/accounts/update_background", %{"img" => ""}) user = refresh_record(user) assert user.info.background == %{} From 1f6ac7680d1ae07be7c7dfd81a8cec2ba52f1c82 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 19:41:05 +0300 Subject: [PATCH 04/15] ActivityPub User view: Following/Followers refactoring - Render the collection items if the user requesting == the user rendered - Do not render the first page if hide_{followers,follows} is set, just give the URI to it --- .../web/activity_pub/views/user_view.ex | 39 +++++++++++++------ .../activity_pub_controller_test.exs | 10 ++--- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 327e0e05b..d9c1bcb2c 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -98,29 +98,31 @@ def render("user.json", %{user: user}) do |> Map.merge(Utils.make_json_ld_header()) end - def render("following.json", %{user: user, page: page}) do + def render("following.json", %{user: user, page: page} = opts) do + showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) total = - if !user.info.hide_follows do + if showing do length(following) else 0 end - collection(following, "#{user.ap_id}/following", page, !user.info.hide_follows, total) + collection(following, "#{user.ap_id}/following", page, showing, total) |> Map.merge(Utils.make_json_ld_header()) end - def render("following.json", %{user: user}) do + def render("following.json", %{user: user} = opts) do + showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) total = - if !user.info.hide_follows do + if showing do length(following) else 0 @@ -130,34 +132,43 @@ def render("following.json", %{user: user}) do "id" => "#{user.ap_id}/following", "type" => "OrderedCollection", "totalItems" => total, - "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows) + "first" => + if showing do + collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows) + else + "#{user.ap_id}/following?page=1" + end } |> Map.merge(Utils.make_json_ld_header()) end - def render("followers.json", %{user: user, page: page}) do + def render("followers.json", %{user: user, page: page} = opts) do + showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) total = - if !user.info.hide_followers do + if showing do length(followers) else 0 end - collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_followers, total) + collection(followers, "#{user.ap_id}/followers", page, showing, total) |> Map.merge(Utils.make_json_ld_header()) end - def render("followers.json", %{user: user}) do + def render("followers.json", %{user: user} = opts) do + showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) total = - if !user.info.hide_followers do + if showing do length(followers) else 0 @@ -168,7 +179,11 @@ def render("followers.json", %{user: user}) do "type" => "OrderedCollection", "totalItems" => total, "first" => - collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers, total) + if showing do + collection(followers, "#{user.ap_id}/followers", 1, showing, total) + else + "#{user.ap_id}/followers?page=1" + end } |> Map.merge(Utils.make_json_ld_header()) end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 1f8eb9d71..cbc25bcdb 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -551,7 +551,7 @@ test "it returns the followers in a collection", %{conn: conn} do assert result["first"]["orderedItems"] == [user.ap_id] end - test "it returns returns empty if the user has 'hide_followers' set", %{conn: conn} do + test "it returns returns a uri if the user has 'hide_followers' set", %{conn: conn} do user = insert(:user) user_two = insert(:user, %{info: %{hide_followers: true}}) User.follow(user, user_two) @@ -561,8 +561,7 @@ test "it returns returns empty if the user has 'hide_followers' set", %{conn: co |> get("/users/#{user_two.nickname}/followers") |> json_response(200) - assert result["first"]["orderedItems"] == [] - assert result["totalItems"] == 0 + assert is_binary(result["first"]) end test "it works for more than 10 users", %{conn: conn} do @@ -606,7 +605,7 @@ test "it returns the following in a collection", %{conn: conn} do assert result["first"]["orderedItems"] == [user_two.ap_id] end - test "it returns returns empty if the user has 'hide_follows' set", %{conn: conn} do + test "it returns a uri if the user has 'hide_follows' set", %{conn: conn} do user = insert(:user, %{info: %{hide_follows: true}}) user_two = insert(:user) User.follow(user, user_two) @@ -616,8 +615,7 @@ test "it returns returns empty if the user has 'hide_follows' set", %{conn: conn |> get("/users/#{user.nickname}/following") |> json_response(200) - assert result["first"]["orderedItems"] == [] - assert result["totalItems"] == 0 + assert is_binary(result["first"]) end test "it works for more than 10 users", %{conn: conn} do From 92055941bd55cb75ab7b5a26d03918390e64b754 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 12 Jul 2019 16:42:54 +0000 Subject: [PATCH 05/15] Pleroma.Web.Metadata - tests --- lib/pleroma/web/metadata/opengraph.ex | 17 +--- lib/pleroma/web/metadata/twitter_card.ex | 47 ++++----- lib/pleroma/web/metadata/utils.ex | 7 ++ test/web/metadata/player_view_test.exs | 33 ++++++ test/web/metadata/twitter_card_test.exs | 123 +++++++++++++++++++++++ 5 files changed, 185 insertions(+), 42 deletions(-) create mode 100644 test/web/metadata/player_view_test.exs create mode 100644 test/web/metadata/twitter_card_test.exs diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 4033ec38f..e7fa7f408 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Utils @behaviour Provider + @media_types ["image", "audio", "video"] @impl Provider def build_tags(%{ @@ -81,26 +82,19 @@ defp build_attachments(%{data: %{"attachment" => attachments}}) do Enum.reduce(attachments, [], fn attachment, acc -> rendered_tags = Enum.reduce(attachment["url"], [], fn url, acc -> - media_type = - Enum.find(["image", "audio", "video"], fn media_type -> - String.starts_with?(url["mediaType"], media_type) - end) - # TODO: Add additional properties to objects when we have the data available. # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image # object when a Video or GIF is attached it will display that in Whatsapp Rich Preview. - case media_type do + case Utils.fetch_media_type(@media_types, url["mediaType"]) do "audio" -> [ - {:meta, - [property: "og:" <> media_type, content: Utils.attachment_url(url["href"])], []} + {:meta, [property: "og:audio", content: Utils.attachment_url(url["href"])], []} | acc ] "image" -> [ - {:meta, - [property: "og:" <> media_type, content: Utils.attachment_url(url["href"])], []}, + {:meta, [property: "og:image", content: Utils.attachment_url(url["href"])], []}, {:meta, [property: "og:image:width", content: 150], []}, {:meta, [property: "og:image:height", content: 150], []} | acc @@ -108,8 +102,7 @@ defp build_attachments(%{data: %{"attachment" => attachments}}) do "video" -> [ - {:meta, - [property: "og:" <> media_type, content: Utils.attachment_url(url["href"])], []} + {:meta, [property: "og:video", content: Utils.attachment_url(url["href"])], []} | acc ] diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 8dd01e0d5..d6a6049b3 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -1,4 +1,5 @@ # Pleroma: A lightweight social networking server + # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only @@ -9,13 +10,10 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do alias Pleroma.Web.Metadata.Utils @behaviour Provider + @media_types ["image", "audio", "video"] @impl Provider - def build_tags(%{ - activity_id: id, - object: object, - user: user - }) do + def build_tags(%{activity_id: id, object: object, user: user}) do attachments = build_attachments(id, object) scrubbed_content = Utils.scrub_html_and_truncate(object) # Zero width space @@ -27,21 +25,12 @@ def build_tags(%{ end [ - {:meta, - [ - property: "twitter:title", - content: Utils.user_name_string(user) - ], []}, - {:meta, - [ - property: "twitter:description", - content: content - ], []} + title_tag(user), + {:meta, [property: "twitter:description", content: content], []} ] ++ if attachments == [] or Metadata.activity_nsfw?(object) do [ - {:meta, - [property: "twitter:image", content: Utils.attachment_url(User.avatar_url(user))], []}, + image_tag(user), {:meta, [property: "twitter:card", content: "summary_large_image"], []} ] else @@ -53,30 +42,28 @@ def build_tags(%{ def build_tags(%{user: user}) do with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do [ - {:meta, - [ - property: "twitter:title", - content: Utils.user_name_string(user) - ], []}, + title_tag(user), {:meta, [property: "twitter:description", content: truncated_bio], []}, - {:meta, [property: "twitter:image", content: Utils.attachment_url(User.avatar_url(user))], - []}, + image_tag(user), {:meta, [property: "twitter:card", content: "summary"], []} ] end end + defp title_tag(user) do + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []} + end + + def image_tag(user) do + {:meta, [property: "twitter:image", content: Utils.attachment_url(User.avatar_url(user))], []} + end + defp build_attachments(id, %{data: %{"attachment" => attachments}}) do Enum.reduce(attachments, [], fn attachment, acc -> rendered_tags = Enum.reduce(attachment["url"], [], fn url, acc -> - media_type = - Enum.find(["image", "audio", "video"], fn media_type -> - String.starts_with?(url["mediaType"], media_type) - end) - # TODO: Add additional properties to objects when we have the data available. - case media_type do + case Utils.fetch_media_type(@media_types, url["mediaType"]) do "audio" -> [ {:meta, [property: "twitter:card", content: "player"], []}, diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 58385a3d1..720bd4519 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -39,4 +39,11 @@ def user_name_string(user) do "(@#{user.nickname})" end end + + @spec fetch_media_type(list(String.t()), String.t()) :: String.t() | nil + def fetch_media_type(supported_types, media_type) do + Enum.find(supported_types, fn support_type -> + String.starts_with?(media_type, support_type) + end) + end end diff --git a/test/web/metadata/player_view_test.exs b/test/web/metadata/player_view_test.exs new file mode 100644 index 000000000..742b0ed8b --- /dev/null +++ b/test/web/metadata/player_view_test.exs @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.PlayerViewTest do + use Pleroma.DataCase + + alias Pleroma.Web.Metadata.PlayerView + + test "it renders audio tag" do + res = + PlayerView.render( + "player.html", + %{"mediaType" => "audio", "href" => "test-href"} + ) + |> Phoenix.HTML.safe_to_string() + + assert res == + "" + end + + test "it renders videos tag" do + res = + PlayerView.render( + "player.html", + %{"mediaType" => "video", "href" => "test-href"} + ) + |> Phoenix.HTML.safe_to_string() + + assert res == + "" + end +end diff --git a/test/web/metadata/twitter_card_test.exs b/test/web/metadata/twitter_card_test.exs new file mode 100644 index 000000000..0814006d2 --- /dev/null +++ b/test/web/metadata/twitter_card_test.exs @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.User + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Endpoint + alias Pleroma.Web.Metadata.Providers.TwitterCard + alias Pleroma.Web.Metadata.Utils + alias Pleroma.Web.Router + + test "it renders twitter card for user info" do + user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") + avatar_url = Utils.attachment_url(User.avatar_url(user)) + res = TwitterCard.build_tags(%{user: user}) + + assert res == [ + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []}, + {:meta, [property: "twitter:description", content: "born 19 March 1994"], []}, + {:meta, [property: "twitter:image", content: avatar_url], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + + test "it does not render attachments if post is nsfw" do + Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false) + user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") + {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"}) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "tag" => [], + "id" => "https://pleroma.gov/objects/whatever", + "content" => "pleroma in a nutshell", + "sensitive" => true, + "attachment" => [ + %{ + "url" => [%{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"}] + }, + %{ + "url" => [ + %{ + "mediaType" => "application/octet-stream", + "href" => "https://pleroma.gov/fqa/badapple.sfc" + } + ] + }, + %{ + "url" => [ + %{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"} + ] + } + ] + } + }) + + result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id}) + + assert [ + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []}, + {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []}, + {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"], + []}, + {:meta, [property: "twitter:card", content: "summary_large_image"], []} + ] == result + end + + test "it renders supported types of attachments and skips unknown types" do + user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") + {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"}) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "tag" => [], + "id" => "https://pleroma.gov/objects/whatever", + "content" => "pleroma in a nutshell", + "attachment" => [ + %{ + "url" => [%{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"}] + }, + %{ + "url" => [ + %{ + "mediaType" => "application/octet-stream", + "href" => "https://pleroma.gov/fqa/badapple.sfc" + } + ] + }, + %{ + "url" => [ + %{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"} + ] + } + ] + } + }) + + result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id}) + + assert [ + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []}, + {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []}, + {:meta, [property: "twitter:card", content: "summary_large_image"], []}, + {:meta, [property: "twitter:player", content: "https://pleroma.gov/tenshi.png"], []}, + {:meta, [property: "twitter:card", content: "player"], []}, + {:meta, + [ + property: "twitter:player", + content: Router.Helpers.o_status_url(Endpoint, :notice_player, activity.id) + ], []}, + {:meta, [property: "twitter:player:width", content: "480"], []}, + {:meta, [property: "twitter:player:height", content: "480"], []} + ] == result + end +end From 97b79efbcd4ad829a575019f842e7dcd7548266a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 20:54:20 +0300 Subject: [PATCH 06/15] ActivityPub Controller: Actually pass for_user to following/followers views and give 403 errors when trying to request hidden follower pages when unauthenticated --- .../activity_pub/activity_pub_controller.ex | 51 +++++++++++++---- lib/pleroma/web/router.ex | 8 ++- .../activity_pub_controller_test.exs | 57 +++++++++++++++++++ 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index cf5176201..e2af4ad1a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -103,43 +103,57 @@ def activity(conn, %{"uuid" => uuid}) do end end - def following(conn, %{"nickname" => nickname, "page" => page}) do + def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- User.ensure_keys_present(user) do + {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user), + {:show_follows, true} <- + {:show_follows, (for_user && for_user == user) || !user.info.hide_follows} do {page, _} = Integer.parse(page) conn |> put_resp_header("content-type", "application/activity+json") - |> json(UserView.render("following.json", %{user: user, page: page})) + |> json(UserView.render("following.json", %{user: user, page: page, for: for_user})) + else + {:show_follows, _} -> + conn + |> put_resp_header("content-type", "application/activity+json") + |> send_resp(403, "") end end - def following(conn, %{"nickname" => nickname}) do + def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- User.ensure_keys_present(user) do + {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do conn |> put_resp_header("content-type", "application/activity+json") - |> json(UserView.render("following.json", %{user: user})) + |> json(UserView.render("following.json", %{user: user, for: for_user})) end end - def followers(conn, %{"nickname" => nickname, "page" => page}) do + def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- User.ensure_keys_present(user) do + {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user), + {:show_followers, true} <- + {:show_followers, (for_user && for_user == user) || !user.info.hide_followers} do {page, _} = Integer.parse(page) conn |> put_resp_header("content-type", "application/activity+json") - |> json(UserView.render("followers.json", %{user: user, page: page})) + |> json(UserView.render("followers.json", %{user: user, page: page, for: for_user})) + else + {:show_followers, _} -> + conn + |> put_resp_header("content-type", "application/activity+json") + |> send_resp(403, "") end end - def followers(conn, %{"nickname" => nickname}) do + def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- User.ensure_keys_present(user) do + {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do conn |> put_resp_header("content-type", "application/activity+json") - |> json(UserView.render("followers.json", %{user: user})) + |> json(UserView.render("followers.json", %{user: user, for: for_user})) end end @@ -325,4 +339,17 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end + + defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do + {:ok, new_user} = User.ensure_keys_present(user) + + for_user = + if new_user != user and match?(%User{}, for_user) do + User.get_cached_by_nickname(for_user.nickname) + else + for_user + end + + {new_user, for_user} + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d53fa8a35..e03a3a2e5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -623,8 +623,6 @@ defmodule Pleroma.Web.Router do # XXX: not really ostatus pipe_through(:ostatus) - get("/users/:nickname/followers", ActivityPubController, :followers) - get("/users/:nickname/following", ActivityPubController, :following) get("/users/:nickname/outbox", ActivityPubController, :outbox) get("/objects/:uuid/likes", ActivityPubController, :object_likes) end @@ -656,6 +654,12 @@ defmodule Pleroma.Web.Router do pipe_through(:oauth_write) post("/users/:nickname/outbox", ActivityPubController, :update_outbox) end + + scope [] do + pipe_through(:oauth_read_or_public) + get("/users/:nickname/followers", ActivityPubController, :followers) + get("/users/:nickname/following", ActivityPubController, :following) + end end scope "/relay", Pleroma.Web.ActivityPub do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index cbc25bcdb..452172bb4 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.CommonAPI setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -564,6 +565,34 @@ test "it returns returns a uri if the user has 'hide_followers' set", %{conn: co assert is_binary(result["first"]) end + test "it returns a 403 error on pages, if the user has 'hide_followers' set and the request is not authenticated", + %{conn: conn} do + user = insert(:user, %{info: %{hide_followers: true}}) + + result = + conn + |> get("/users/#{user.nickname}/followers?page=1") + + assert result.status == 403 + assert result.resp_body == "" + end + + test "it renders the page, if the user has 'hide_followers' set and the request is authenticated with the same user", + %{conn: conn} do + user = insert(:user, %{info: %{hide_followers: true}}) + other_user = insert(:user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + result = + conn + |> assign(:user, user) + |> get("/users/#{user.nickname}/followers?page=1") + |> json_response(200) + + assert result["totalItems"] == 1 + assert result["orderedItems"] == [other_user.ap_id] + end + test "it works for more than 10 users", %{conn: conn} do user = insert(:user) @@ -618,6 +647,34 @@ test "it returns a uri if the user has 'hide_follows' set", %{conn: conn} do assert is_binary(result["first"]) end + test "it returns a 403 error on pages, if the user has 'hide_follows' set and the request is not authenticated", + %{conn: conn} do + user = insert(:user, %{info: %{hide_follows: true}}) + + result = + conn + |> get("/users/#{user.nickname}/following?page=1") + + assert result.status == 403 + assert result.resp_body == "" + end + + test "it renders the page, if the user has 'hide_follows' set and the request is authenticated with the same user", + %{conn: conn} do + user = insert(:user, %{info: %{hide_follows: true}}) + other_user = insert(:user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) + + result = + conn + |> assign(:user, user) + |> get("/users/#{user.nickname}/following?page=1") + |> json_response(200) + + assert result["totalItems"] == 1 + assert result["orderedItems"] == [other_user.ap_id] + end + test "it works for more than 10 users", %{conn: conn} do user = insert(:user) From f40004e7463bafd07ba70046561dbf937bbf8646 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 21:49:16 +0300 Subject: [PATCH 07/15] Add changelog entries for follower/following collection behaviour changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 942733ab6..405b7680c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - Configuration: OpenGraph and TwitterCard providers enabled by default - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text +- Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option ### Fixed @@ -16,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) +- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) From b001b8891a0ae9d8c7291f8148eb68a354cd319f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 23:52:26 +0300 Subject: [PATCH 08/15] Merge the default options with custom ones in ReverseProxy and Pleroma.HTTP --- lib/pleroma/http/connection.ex | 2 +- lib/pleroma/http/http.ex | 5 +---- lib/pleroma/reverse_proxy/reverse_proxy.ex | 5 +++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index c216cdcb1..a1460d303 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -29,7 +29,7 @@ def new(opts \\ []) do # fetch Hackney options # - defp hackney_options(opts) do + def hackney_options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index c96ee7353..dec24458a 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,10 +65,7 @@ defp process_sni_options(options, url) do end def process_request_options(options) do - case Pleroma.Config.get([:http, :proxy_url]) do - nil -> options - proxy -> options ++ [proxy: proxy] - end + Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) end @doc """ diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index bf31e9cba..1f98f215c 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @default_hackney_options [] + @default_hackney_options [pool: :media] @inline_content_types [ "image/gif", @@ -94,7 +94,8 @@ def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do hackney_opts = - @default_hackney_options + Pleroma.HTTP.Connection.hackney_options([]) + |> Keyword.merge(@default_hackney_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() From fa7e0c4262f8844bb6224c200f7d41720607fcac Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 12 Jul 2019 23:53:21 +0300 Subject: [PATCH 09/15] Workaround for remote server certificate chain issues --- config/config.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.exs b/config/config.exs index 99b500993..eb663f3ec 100644 --- a/config/config.exs +++ b/config/config.exs @@ -194,6 +194,8 @@ send_user_agent: true, adapter: [ ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, # We don't support TLS v1.3 yet versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"] ] From 29ffe81c2e2235ed723516e74a50b025af688b9b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 13 Jul 2019 02:04:26 +0300 Subject: [PATCH 10/15] Add a changelog entry for tolerating incorrect chain order --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 942733ab6..1ee845031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts - Metadata rendering errors resulting in the entire page being inaccessible +- Federation/MediaProxy not working with instances that have wrong certificate order - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) From 369e9bb42fc907f2e3f92e7e44dc52d6940dc046 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 14:49:39 +0300 Subject: [PATCH 11/15] [#1041] Rate-limited status actions (per user and per user+status). --- CHANGELOG.md | 3 +- config/config.exs | 4 +- lib/pleroma/plugs/rate_limiter.ex | 67 ++++++++++++---- .../mastodon_api/mastodon_api_controller.ex | 22 ++++- test/plugs/rate_limiter_test.exs | 80 ++++++++++++++++--- 5 files changed, 149 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 405b7680c..c6c1ba1f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) -Configuration: `federation_incoming_replies_max_depth` option +- Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header - Mastodon API, extension: Ability to reset avatar, profile banner, and background @@ -32,6 +32,7 @@ Configuration: `federation_incoming_replies_max_depth` option - Added synchronization of following/followers counters for external users - Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`. - Mastodon API: Add support for categories for custom emojis by reusing the group feature. +- Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/config/config.exs b/config/config.exs index 99b500993..3d48a5584 100644 --- a/config/config.exs +++ b/config/config.exs @@ -519,7 +519,9 @@ config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}], - app_account_creation: {1_800_000, 25} + app_account_creation: {1_800_000, 25}, + statuses_actions: {10_000, 15}, + status_id_action: {60_000, 3} # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex index c5e0957e8..31388f574 100644 --- a/lib/pleroma/plugs/rate_limiter.ex +++ b/lib/pleroma/plugs/rate_limiter.ex @@ -31,12 +31,28 @@ defmodule Pleroma.Plugs.RateLimiter do ## Usage + AllowedSyntax: + + plug(Pleroma.Plugs.RateLimiter, :limiter_name) + plug(Pleroma.Plugs.RateLimiter, {:limiter_name, options}) + + Allowed options: + + * `bucket_name` overrides bucket name (e.g. to have a separate limit for a set of actions) + * `params` appends values of specified request params (e.g. ["id"]) to bucket name + Inside a controller: plug(Pleroma.Plugs.RateLimiter, :one when action == :one) plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) - or inside a router pipiline: + plug( + Pleroma.Plugs.RateLimiter, + {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} + when action in ~w(fav_status unfav_status)a + ) + + or inside a router pipeline: pipeline :api do ... @@ -49,33 +65,56 @@ defmodule Pleroma.Plugs.RateLimiter do alias Pleroma.User - def init(limiter_name) do + def init(limiter_name) when is_atom(limiter_name) do + init({limiter_name, []}) + end + + def init({limiter_name, opts}) do case Pleroma.Config.get([:rate_limit, limiter_name]) do nil -> nil - config -> {limiter_name, config} + config -> {limiter_name, config, opts} end end - # do not limit if there is no limiter configuration + # Do not limit if there is no limiter configuration def call(conn, nil), do: conn - def call(conn, opts) do - case check_rate(conn, opts) do - {:ok, _count} -> conn - {:error, _count} -> render_throttled_error(conn) + def call(conn, settings) do + case check_rate(conn, settings) do + {:ok, _count} -> + conn + + {:error, _count} -> + render_throttled_error(conn) end end - defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do - ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit) + defp bucket_name(conn, limiter_name, opts) do + bucket_name = opts[:bucket_name] || limiter_name + + if params_names = opts[:params] do + params_values = for p <- Enum.sort(params_names), do: conn.params[p] + Enum.join([bucket_name] ++ params_values, ":") + else + bucket_name + end end - defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do - ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit) + defp check_rate( + %{assigns: %{user: %User{id: user_id}}} = conn, + {limiter_name, [_, {scale, limit}], opts} + ) do + bucket_name = bucket_name(conn, limiter_name, opts) + ExRated.check_rate("#{bucket_name}:#{user_id}", scale, limit) end - defp check_rate(conn, {limiter_name, {scale, limit}}) do - check_rate(conn, {limiter_name, [{scale, limit}]}) + defp check_rate(conn, {limiter_name, [{scale, limit} | _], opts}) do + bucket_name = bucket_name(conn, limiter_name, opts) + ExRated.check_rate("#{bucket_name}:#{ip(conn)}", scale, limit) + end + + defp check_rate(conn, {limiter_name, {scale, limit}, opts}) do + check_rate(conn, {limiter_name, [{scale, limit}, {scale, limit}], opts}) end def ip(%{remote_ip: remote_ip}) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 8c2033c3a..76648b9f7 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Pagination + alias Pleroma.Plugs.RateLimiter alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.Stats @@ -46,8 +47,25 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register) - plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) + @rate_limited_status_crud_actions ~w(post_status delete_status)a + @rate_limited_status_reactions ~w(reblog_status unreblog_status fav_status unfav_status)a + @rate_limited_status_actions @rate_limited_status_crud_actions ++ @rate_limited_status_reactions + + plug( + RateLimiter, + {:status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]} + when action in ~w(reblog_status unreblog_status)a + ) + + plug( + RateLimiter, + {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} + when action in ~w(fav_status unfav_status)a + ) + + plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions) + plug(RateLimiter, :app_account_creation when action == :account_register) + plug(RateLimiter, :search when action in [:search, :search2, :account_search]) @local_mastodon_name "Mastodon-Local" diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index f8251b5c7..395095079 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -10,12 +10,13 @@ defmodule Pleroma.Plugs.RateLimiterTest do import Pleroma.Factory - @limiter_name :testing + # Note: each example must work with separate buckets in order to prevent concurrency issues test "init/1" do - Pleroma.Config.put([:rate_limit, @limiter_name], {1, 1}) + limiter_name = :test_init + Pleroma.Config.put([:rate_limit, limiter_name], {1, 1}) - assert {@limiter_name, {1, 1}} == RateLimiter.init(@limiter_name) + assert {limiter_name, {1, 1}, []} == RateLimiter.init(limiter_name) assert nil == RateLimiter.init(:foo) end @@ -24,14 +25,15 @@ test "ip/1" do end test "it restricts by opts" do + limiter_name = :test_opts scale = 1000 limit = 5 - Pleroma.Config.put([:rate_limit, @limiter_name], {scale, limit}) + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) - opts = RateLimiter.init(@limiter_name) + opts = RateLimiter.init(limiter_name) conn = conn(:get, "/") - bucket_name = "#{@limiter_name}:#{RateLimiter.ip(conn)}" + bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" conn = RateLimiter.call(conn, opts) assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) @@ -65,18 +67,78 @@ test "it restricts by opts" do refute conn.halted end + test "`bucket_name` option overrides default bucket name" do + limiter_name = :test_bucket_name + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name}) + + conn = conn(:get, "/") + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + customized_bucket_name = "#{base_bucket_name}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(customized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + + test "`params` option appends specified params' values to bucket name" do + limiter_name = :test_params + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + opts = RateLimiter.init({limiter_name, params: ["id"]}) + id = "1" + + conn = conn(:get, "/?id=#{id}") + conn = Plug.Conn.fetch_query_params(conn) + + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + parametrized_bucket_name = "#{limiter_name}:#{id}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + + test "it supports combination of options modifying bucket name" do + limiter_name = :test_options_combo + scale = 1000 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name, params: ["id"]}) + id = "100" + + conn = conn(:get, "/?id=#{id}") + conn = Plug.Conn.fetch_query_params(conn) + + default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + parametrized_bucket_name = "#{base_bucket_name}:#{id}:#{RateLimiter.ip(conn)}" + + RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) + assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) + end + test "optional limits for authenticated users" do + limiter_name = :test_authenticated Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) scale = 1000 limit = 5 - Pleroma.Config.put([:rate_limit, @limiter_name], [{1, 10}, {scale, limit}]) + Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}]) - opts = RateLimiter.init(@limiter_name) + opts = RateLimiter.init(limiter_name) user = insert(:user) conn = conn(:get, "/") |> assign(:user, user) - bucket_name = "#{@limiter_name}:#{user.id}" + bucket_name = "#{limiter_name}:#{user.id}" conn = RateLimiter.call(conn, opts) assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) From b74d11e20ab214d533f746bee81fde589d319f64 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 15:13:26 +0300 Subject: [PATCH 12/15] [#1041] Added documentation on existing rate limiters. --- docs/config.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/config.md b/docs/config.md index 140789d87..3ec13cff2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -640,3 +640,10 @@ A keyword list of rate limiters where a key is a limiter name and value is the l It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. + +Supported rate limiters: + +* `:search` for the search requests (account & status search etc.) +* `:app_account_creation` for registering user accounts from the same IP address +* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses +* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user From d72876c57dd8f519f63f7bb14abcfaceedf41410 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 13 Jul 2019 15:21:50 +0300 Subject: [PATCH 13/15] [#1041] Minor refactoring. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 76648b9f7..8a7b75025 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -47,9 +47,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - @rate_limited_status_crud_actions ~w(post_status delete_status)a - @rate_limited_status_reactions ~w(reblog_status unreblog_status fav_status unfav_status)a - @rate_limited_status_actions @rate_limited_status_crud_actions ++ @rate_limited_status_reactions + @rate_limited_status_actions ~w(reblog_status unreblog_status fav_status unfav_status + post_status delete_status)a plug( RateLimiter, From 80c46d6d8b84d77d86efc32c1d2af225c1eada33 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 13 Jul 2019 18:30:45 +0000 Subject: [PATCH 14/15] nodeinfo: implement MRF transparency exclusions --- CHANGELOG.md | 1 + config/config.exs | 1 + docs/config.md | 1 + .../web/nodeinfo/nodeinfo_controller.ex | 6 ++- test/web/node_info_test.exs | 43 +++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index add8a0348..bebb438b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +- MRF: Support for excluding specific domains from Transparency. - Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header diff --git a/config/config.exs b/config/config.exs index 889238f0f..2ffa8c621 100644 --- a/config/config.exs +++ b/config/config.exs @@ -240,6 +240,7 @@ "text/bbcode" ], mrf_transparency: true, + mrf_transparency_exclusions: [], autofollowed_nicknames: [], max_pinned_statuses: 1, no_attachment_links: false, diff --git a/docs/config.md b/docs/config.md index 3ec13cff2..639c5689c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -106,6 +106,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) * `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). +* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. * `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. * `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: * "email": Copy and preprend re:, as in email. diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index cd9a4f4a8..a1d7fcc7d 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -34,8 +34,11 @@ def schemas(conn, _params) do def raw_nodeinfo do stats = Stats.get_stats() + exclusions = Config.get([:instance, :mrf_transparency_exclusions]) + mrf_simple = Config.get(:mrf_simple) + |> Enum.map(fn {k, v} -> {k, Enum.reject(v, fn v -> v in exclusions end)} end) |> Enum.into(%{}) # This horror is needed to convert regex sigils to strings @@ -86,7 +89,8 @@ def raw_nodeinfo do mrf_simple: mrf_simple, mrf_keyword: mrf_keyword, mrf_user_allowlist: mrf_user_allowlist, - quarantined_instances: quarantined + quarantined_instances: quarantined, + exclusions: length(exclusions) > 0 } else %{} diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index be1173513..d7f848bfa 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -83,4 +83,47 @@ test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do Pleroma.Config.put([:instance, :safe_dm_mentions], option) end + + test "it shows MRF transparency data if enabled", %{conn: conn} do + option = Pleroma.Config.get([:instance, :mrf_transparency]) + Pleroma.Config.put([:instance, :mrf_transparency], true) + + simple_config = %{"reject" => ["example.com"]} + Pleroma.Config.put(:mrf_simple, simple_config) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["mrf_simple"] == simple_config + + Pleroma.Config.put([:instance, :mrf_transparency], option) + Pleroma.Config.put(:mrf_simple, %{}) + end + + test "it performs exclusions from MRF transparency data if configured", %{conn: conn} do + option = Pleroma.Config.get([:instance, :mrf_transparency]) + Pleroma.Config.put([:instance, :mrf_transparency], true) + + exclusions = Pleroma.Config.get([:instance, :mrf_transparency_exclusions]) + Pleroma.Config.put([:instance, :mrf_transparency_exclusions], ["other.site"]) + + simple_config = %{"reject" => ["example.com", "other.site"]} + expected_config = %{"reject" => ["example.com"]} + + Pleroma.Config.put(:mrf_simple, simple_config) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["mrf_simple"] == expected_config + assert response["metadata"]["federation"]["exclusions"] == true + + Pleroma.Config.put([:instance, :mrf_transparency], option) + Pleroma.Config.put([:instance, :mrf_transparency_exclusions], exclusions) + Pleroma.Config.put(:mrf_simple, %{}) + end end From 0cc638b96874312bce29e477a0ce6e46a92142bf Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 13 Jul 2019 19:00:03 +0000 Subject: [PATCH 15/15] docs: note that exclusions usage will be included in the transparency metrics if used --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 639c5689c..a65b7a560 100644 --- a/docs/config.md +++ b/docs/config.md @@ -106,7 +106,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) * `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). -* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. +* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. * `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. * `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: * "email": Copy and preprend re:, as in email.