From e7edfd9fec88af24869c3805a404f2b0a20914de Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 12:20:18 -0500 Subject: [PATCH 01/48] Permit fetching statuses from API with nickname or id --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 2110027c3..bc75ab35a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -330,7 +330,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do end def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do - with %User{} = user <- User.get_cached_by_id(params["id"]) do + with %User{} = user <- User.get_cached_by_nickname_or_id(params["id"]) do activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn From e912f81c828cc7e1d2c0dff8daed3ad52f407a61 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 12:22:14 -0500 Subject: [PATCH 02/48] Update docs to reflect we accept nickname for id for both of these endpoints --- docs/api/differences_in_mastoapi_responses.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 36b47608e..f952696f7 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -32,7 +32,10 @@ Has these additional fields under the `pleroma` object: ## Accounts -- `/api/v1/accounts/:id`: The `id` parameter can also be the `nickname` of the user. This only works in this endpoint, not the deeper nested ones for following etc. +The `id` parameter can also be the `nickname` of the user. This only works in these endpoints, not the deeper nested ones for following etc. + +- `/api/v1/accounts/:id` +- `/api/v1/accounts/:id/statuses` Has these additional fields under the `pleroma` object: From 46c7c53fbbfd9919d429ba15988af0da9e22d444 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 15 Jul 2019 17:10:33 -0500 Subject: [PATCH 03/48] Add changelog entry for Mastodon API change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c90da0b..599ea0a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API, extension: Ability to reset avatar, profile banner, and background - Mastodon API: Add support for categories for custom emojis by reusing the group feature. - Mastodon API: Add support for muting/unmuting notifications +- Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From 70439494af587484550510e51ad1a31c05f66d38 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 16 Jul 2019 14:56:07 +0700 Subject: [PATCH 04/48] Fix typo --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 18145e45f..c505223f7 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -131,7 +131,7 @@ def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bc %User{ap_id: ap_id} = Enum.find(recipients, fn %{info: %{source_data: data}} -> data["inbox"] == inbox end) - # Get all the recipients on the same host and add them to cc. Otherwise it a remote + # Get all the recipients on the same host and add them to cc. Otherwise, a remote # instance would only accept a first message for the first recipient and ignore the rest. cc = get_cc_ap_ids(ap_id, recipients) From c4ca142e14444a03250f246e189c78e70aafe9d7 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Tue, 16 Jul 2019 11:04:11 +0000 Subject: [PATCH 05/48] Add the `blocked_by` attribute to the relationship API (`GET /api/v1/accounts/relationships`) --- CHANGELOG.md | 1 + .../web/mastodon_api/views/account_view.ex | 1 + test/web/mastodon_api/account_view_test.exs | 102 ++++++++++++++---- 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c90da0b..f121d0fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API, extension: Ability to reset avatar, profile banner, and background - Mastodon API: Add support for categories for custom emojis by reusing the group feature. - Mastodon API: Add support for muting/unmuting notifications +- Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 65bab4062..de1eef9d2 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -51,6 +51,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target following: User.following?(user, target), followed_by: User.following?(target, user), blocking: User.blocks?(user, target), + blocked_by: User.blocks?(target, user), muting: User.mutes?(user, target), muting_notifications: User.muted_notifications?(user, target), subscribing: User.subscribed_to?(user, target), diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index de6aeec72..83b9db071 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.User + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView test "Represent a user account" do @@ -165,28 +166,90 @@ test "Represent a smaller mention" do assert expected == AccountView.render("mention.json", %{user: user}) end - test "represent a relationship" do - user = insert(:user) - other_user = insert(:user) + describe "relationship" do + test "represent a relationship for the following and followed user" do + user = insert(:user) + other_user = insert(:user) - {:ok, user} = User.follow(user, other_user) - {:ok, user} = User.block(user, other_user) + {:ok, user} = User.follow(user, other_user) + {:ok, other_user} = User.follow(other_user, user) + {:ok, other_user} = User.subscribe(user, other_user) + {:ok, user} = User.mute(user, other_user, true) + {:ok, user} = CommonAPI.hide_reblogs(user, other_user) - expected = %{ - id: to_string(other_user.id), - following: false, - followed_by: false, - blocking: true, - muting: false, - muting_notifications: false, - subscribing: false, - requested: false, - domain_blocking: false, - showing_reblogs: true, - endorsed: false - } + expected = %{ + id: to_string(other_user.id), + following: true, + followed_by: true, + blocking: false, + blocked_by: false, + muting: true, + muting_notifications: true, + subscribing: true, + requested: false, + domain_blocking: false, + showing_reblogs: false, + endorsed: false + } - assert expected == AccountView.render("relationship.json", %{user: user, target: other_user}) + assert expected == + AccountView.render("relationship.json", %{user: user, target: other_user}) + end + + test "represent a relationship for the blocking and blocked user" do + user = insert(:user) + other_user = insert(:user) + + {:ok, user} = User.follow(user, other_user) + {:ok, other_user} = User.subscribe(user, other_user) + {:ok, user} = User.block(user, other_user) + {:ok, other_user} = User.block(other_user, user) + + expected = %{ + id: to_string(other_user.id), + following: false, + followed_by: false, + blocking: true, + blocked_by: true, + muting: false, + muting_notifications: false, + subscribing: false, + requested: false, + domain_blocking: false, + showing_reblogs: true, + endorsed: false + } + + assert expected == + AccountView.render("relationship.json", %{user: user, target: other_user}) + end + + test "represent a relationship for the user with a pending follow request" do + user = insert(:user) + other_user = insert(:user, %{info: %User.Info{locked: true}}) + + {:ok, user, other_user, _} = CommonAPI.follow(user, other_user) + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) + + expected = %{ + id: to_string(other_user.id), + following: false, + followed_by: false, + blocking: false, + blocked_by: false, + muting: false, + muting_notifications: false, + subscribing: false, + requested: true, + domain_blocking: false, + showing_reblogs: true, + endorsed: false + } + + assert expected == + AccountView.render("relationship.json", %{user: user, target: other_user}) + end end test "represent an embedded relationship" do @@ -240,6 +303,7 @@ test "represent an embedded relationship" do following: false, followed_by: false, blocking: true, + blocked_by: false, subscribing: false, muting: false, muting_notifications: false, From 520ee6c59162c26caf032b2d16ca975c99b5beb5 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Tue, 16 Jul 2019 11:14:46 +0000 Subject: [PATCH 06/48] Add `pleroma.deactivated` to the Account entity (Mastodon API) --- CHANGELOG.md | 1 + docs/api/differences_in_mastoapi_responses.md | 1 + lib/pleroma/web/mastodon_api/views/account_view.ex | 7 +++++++ test/web/mastodon_api/account_view_test.exs | 7 +++++++ 4 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f121d0fbf..f3630a1c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add support for categories for custom emojis by reusing the group feature. - Mastodon API: Add support for muting/unmuting notifications - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). +- Mastodon API: Add `pleroma.deactivated` to the Account entity - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index d2e9bcc4b..c65b11872 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -47,6 +47,7 @@ Has these additional fields under the `pleroma` object: - `hide_follows`: boolean, true when the user has follow hiding enabled - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` +- `deactivated`: boolean, true when the user is deactivated ### Source diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index de1eef9d2..befb35c26 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -137,6 +137,7 @@ defp do_render("account.json", %{user: user} = opts) do |> maybe_put_notification_settings(user, opts[:for]) |> maybe_put_settings_store(user, opts[:for], opts) |> maybe_put_chat_token(user, opts[:for], opts) + |> maybe_put_activation_status(user, opts[:for]) end defp username_from_nickname(string) when is_binary(string) do @@ -197,6 +198,12 @@ defp maybe_put_notification_settings(data, %User{id: user_id} = user, %User{id: defp maybe_put_notification_settings(data, _, _), do: data + defp maybe_put_activation_status(data, user, %User{info: %{is_admin: true}}) do + Kernel.put_in(data, [:pleroma, :deactivated], user.info.deactivated) + end + + defp maybe_put_activation_status(data, _, _), do: data + defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil end diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index 83b9db071..fa44d35cc 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -153,6 +153,13 @@ test "Represent a Service(bot) account" do assert expected == AccountView.render("account.json", %{user: user}) end + test "Represent a deactivated user for an admin" do + admin = insert(:user, %{info: %{is_admin: true}}) + deactivated_user = insert(:user, %{info: %{deactivated: true}}) + represented = AccountView.render("account.json", %{user: deactivated_user, for: admin}) + assert represented[:pleroma][:deactivated] == true + end + test "Represent a smaller mention" do user = insert(:user) From 889dc17abd95bd1f414646e54d7e3cdadd9afbc9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 16 Jul 2019 19:18:30 +0300 Subject: [PATCH 07/48] [#1094] Rate-limited follow & unfollow actions. --- config/config.exs | 2 ++ docs/config.md | 2 ++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/config/config.exs b/config/config.exs index 7d539f994..03e0341c8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -528,6 +528,8 @@ config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}], app_account_creation: {1_800_000, 25}, + relations_actions: {10_000, 10}, + relation_id_action: {60_000, 2}, statuses_actions: {10_000, 15}, status_id_action: {60_000, 3} diff --git a/docs/config.md b/docs/config.md index 9a64f0ed7..f595caba5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -647,5 +647,7 @@ Supported rate limiters: * `:search` for the search requests (account & status search etc.) * `:app_account_creation` for registering user accounts from the same IP address +* `:relations_actions` for actions on relations with other users (follow, unfollow) +* `:relation_id_action` for actions on relation with specific another user (follow, unfollow) * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index f4aa576f7..a732a6990 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -47,6 +47,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger + @rate_limited_relations_actions ~w(follow unfollow)a + @rate_limited_status_actions ~w(reblog_status unreblog_status fav_status unfav_status post_status delete_status)a @@ -62,6 +64,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do when action in ~w(fav_status unfav_status)a ) + plug( + RateLimiter, + {:relations_id_action, params: ["id", "uri"]} when action in @rate_limited_relations_actions + ) + + plug(RateLimiter, :relations_actions when action in @rate_limited_relations_actions) plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions) plug(RateLimiter, :app_account_creation when action == :account_register) plug(RateLimiter, :search when action in [:search, :search2, :account_search]) From 2ba07b63f4557554cd4acc63dc8e0424612554a0 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 16 Jul 2019 16:59:02 +0000 Subject: [PATCH 08/48] Apply suggestion to docs/config.md --- docs/config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index f595caba5..346b8cda2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -647,7 +647,7 @@ Supported rate limiters: * `:search` for the search requests (account & status search etc.) * `:app_account_creation` for registering user accounts from the same IP address -* `:relations_actions` for actions on relations with other users (follow, unfollow) -* `:relation_id_action` for actions on relation with specific another user (follow, unfollow) +* `:relations_actions` for actions on relations with all users (follow, unfollow) +* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user From 18234cc44e6bc989e3e3cf15714c54b4fa05b9dd Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Tue, 16 Jul 2019 22:37:36 +0545 Subject: [PATCH 09/48] add the rich media ttl based on image exp time --- CHANGELOG.md | 1 + config/config.exs | 3 +- ..._set_richmedia_cache_ttl_based_on_image.md | 32 +++++++++++ lib/pleroma/web/rich_media/parser.ex | 41 ++++++++++++++ .../rich_media/parsers/ttl/aws_signed_url.ex | 54 +++++++++++++++++++ test/fixtures/rich_media/amz.html | 5 ++ test/web/rich_media/aws_signed_url_test.exs | 37 +++++++++++++ 7 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 docs/config/howto_set_richmedia_cache_ttl_based_on_image.md create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex create mode 100644 test/fixtures/rich_media/amz.html create mode 100644 test/web/rich_media/aws_signed_url_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3630a1c5..4e58b0a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text - Admin API: changed json structure for saving config settings. - RichMedia: parsers and their order are configured in `rich_media` config. +- RichMedia: add the rich media ttl based on image expiration time. ## [1.0.1] - 2019-07-14 ### Security diff --git a/config/config.exs b/config/config.exs index 7d539f994..aa5bd0da9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -344,7 +344,8 @@ Pleroma.Web.RichMedia.Parsers.TwitterCard, Pleroma.Web.RichMedia.Parsers.OGP, Pleroma.Web.RichMedia.Parsers.OEmbed - ] + ], + ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl] config :pleroma, :media_proxy, enabled: false, diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md new file mode 100644 index 000000000..489f9ece8 --- /dev/null +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -0,0 +1,32 @@ +# How to set rich media cache ttl based on image ttl +## Explanation + +Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. +In such cases the old image url (expired) is returned from the media cache. + +So to avoid such situation we can define a moddule that will set ttl based no image. + +The module must have a `run` function and it should be registered in the config. + +### Example + +```exs +defmodule MyModule do + def run(data, url) do + image_url = Map.get(data, :image) + # do some parsing in the url and get the ttl of the image + # ttl is unix time + ttl = parse_ttl_from_url(image_url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end +end +``` + +And update the config + +```exs +config :pleroma, :rich_media, + ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl, MyModule] +``` + +> For reference there is a parser for AWS signed URL `Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl`, it's enabled by default. diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 0d2523338..ba8dc6f2a 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -24,6 +24,7 @@ def parse(url) do Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) + |> set_ttl_based_on_image(url) rescue e -> {:error, "Cachex error: #{inspect(e)}"} @@ -31,6 +32,46 @@ def parse(url) do end end + @doc """ + Set the rich media cache based on the expiration time of image. + + Define a module that has `run` function + + ## Example + + defmodule MyModule do + def run(data, url) do + image_url = Map.get(data, :image) + # do some parsing in the url and get the ttl of the image + # ttl is unix time + ttl = parse_ttl_from_url(image_url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end + end + + Define the module in the config + + config :pleroma, :rich_media, + ttl_setters: [MyModule] + """ + def set_ttl_based_on_image({:ok, data}, url) do + case Cachex.ttl(:rich_media_cache, url) do + {:ok, nil} -> + modules = Pleroma.Config.get([:rich_media, :ttl_setters]) + + if Enum.count(modules) > 0 do + Enum.each(modules, & &1.run(data, url)) + end + + {:ok, data} + + _ -> + {:ok, data} + end + end + + def set_ttl_based_on_image(data, _url), do: data + defp parse_url(url) do try do {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex new file mode 100644 index 000000000..d57107939 --- /dev/null +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -0,0 +1,54 @@ +defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do + def run(data, url) do + image = Map.get(data, :image) + + if is_aws_signed_url(image) do + image + |> parse_query_params() + |> format_query_params() + |> get_expiration_timestamp() + |> set_ttl(url) + end + end + + defp is_aws_signed_url(""), do: nil + defp is_aws_signed_url(nil), do: nil + + defp is_aws_signed_url(image) when is_binary(image) do + %URI{host: host, query: query} = URI.parse(image) + + if String.contains?(host, "amazonaws.com") and + String.contains?(query, "X-Amz-Expires") do + image + else + nil + end + end + + defp is_aws_signed_url(_), do: nil + + defp parse_query_params(image) do + %URI{query: query} = URI.parse(image) + query + end + + defp format_query_params(query) do + query + |> String.split(~r/&|=/) + |> Enum.chunk_every(2) + |> Map.new(fn [k, v] -> {k, v} end) + end + + defp get_expiration_timestamp(params) when is_map(params) do + {:ok, date} = + params + |> Map.get("X-Amz-Date") + |> Timex.parse("{ISO:Basic:Z}") + + Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) + end + + defp set_ttl(ttl, url) do + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + end +end diff --git a/test/fixtures/rich_media/amz.html b/test/fixtures/rich_media/amz.html new file mode 100644 index 000000000..d4f8bd1a3 --- /dev/null +++ b/test/fixtures/rich_media/amz.html @@ -0,0 +1,5 @@ + + + + + diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs new file mode 100644 index 000000000..75bf6c6df --- /dev/null +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do + use ExUnit.Case, async: true + + test "amazon signed url is parsed and correct ttl is set for rich media" do + url = "https://pleroma.social/amz" + + {:ok, timestamp} = + Timex.now() + |> DateTime.truncate(:second) + |> Timex.format("{ISO:Basic:Z}") + + # in seconds + valid_till = 30 + + data = %{ + image: + "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ + timestamp + }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host", + locale: "en_US", + site_name: "Pleroma", + title: "PLeroma", + url: url + } + + Cachex.put(:rich_media_cache, url, data) + assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url) + {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) + + # as there is delay in setting and pulling the data from cache we ignore 1 second + assert_in_delta(valid_till * 1000, cache_ttl, 1000) + end +end From 21e3f9ac69e258d7c46e0f2acdcf011f74cba8e3 Mon Sep 17 00:00:00 2001 From: Maksim Date: Tue, 16 Jul 2019 21:35:43 +0000 Subject: [PATCH 10/48] added tests for Pleroma.Upload.Filter --- lib/pleroma/upload/filter/dedupe.ex | 15 ++++++-- lib/pleroma/upload/filter/mogrifun.ex | 24 ++----------- lib/pleroma/upload/filter/mogrify.ex | 13 ++++--- test/upload/filter/dedupe_test.exs | 31 ++++++++++++++++ test/upload/filter/mogrifun_test.exs | 44 +++++++++++++++++++++++ test/upload/filter/mogrify_test.exs | 51 +++++++++++++++++++++++++++ test/upload/filter_test.exs | 39 ++++++++++++++++++++ 7 files changed, 187 insertions(+), 30 deletions(-) create mode 100644 test/upload/filter/dedupe_test.exs create mode 100644 test/upload/filter/mogrifun_test.exs create mode 100644 test/upload/filter/mogrify_test.exs create mode 100644 test/upload/filter_test.exs diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex index e4c225833..14928c355 100644 --- a/lib/pleroma/upload/filter/dedupe.ex +++ b/lib/pleroma/upload/filter/dedupe.ex @@ -6,10 +6,19 @@ defmodule Pleroma.Upload.Filter.Dedupe do @behaviour Pleroma.Upload.Filter alias Pleroma.Upload - def filter(%Upload{name: name} = upload) do - extension = String.split(name, ".") |> List.last() - shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower) + def filter(%Upload{name: name, tempfile: tempfile} = upload) do + extension = + name + |> String.split(".") + |> List.last() + + shasum = + :crypto.hash(:sha256, File.read!(tempfile)) + |> Base.encode16(case: :lower) + filename = shasum <> "." <> extension {:ok, %Upload{upload | id: shasum, path: filename}} end + + def filter(_), do: :ok end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 35a5a1381..fee49fb51 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Upload.Filter.Mogrifun do @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload.Filter @filters [ {"implode", "1"}, @@ -34,31 +35,10 @@ defmodule Pleroma.Upload.Filter.Mogrifun do ] def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - filter = Enum.random(@filters) - - file - |> Mogrify.open() - |> mogrify_filter(filter) - |> Mogrify.save(in_place: true) + Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) :ok end def filter(_), do: :ok - - defp mogrify_filter(mogrify, [filter | rest]) do - mogrify - |> mogrify_filter(filter) - |> mogrify_filter(rest) - end - - defp mogrify_filter(mogrify, []), do: mogrify - - defp mogrify_filter(mogrify, {action, options}) do - Mogrify.custom(mogrify, action, options) - end - - defp mogrify_filter(mogrify, string) when is_binary(string) do - Mogrify.custom(mogrify, string) - end end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index f459eeecb..91bfdd4f5 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -11,16 +11,19 @@ defmodule Pleroma.Upload.Filter.Mogrify do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do filters = Pleroma.Config.get!([__MODULE__, :args]) - file - |> Mogrify.open() - |> mogrify_filter(filters) - |> Mogrify.save(in_place: true) - + do_filter(file, filters) :ok end def filter(_), do: :ok + def do_filter(file, filters) do + file + |> Mogrify.open() + |> mogrify_filter(filters) + |> Mogrify.save(in_place: true) + end + defp mogrify_filter(mogrify, nil), do: mogrify defp mogrify_filter(mogrify, [filter | rest]) do diff --git a/test/upload/filter/dedupe_test.exs b/test/upload/filter/dedupe_test.exs new file mode 100644 index 000000000..fddd594dc --- /dev/null +++ b/test/upload/filter/dedupe_test.exs @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.DedupeTest do + use Pleroma.DataCase + + alias Pleroma.Upload + alias Pleroma.Upload.Filter.Dedupe + + @shasum "e30397b58d226d6583ab5b8b3c5defb0c682bda5c31ef07a9f57c1c4986e3781" + + test "adds shasum" do + File.cp!( + "test/fixtures/image.jpg", + "test/fixtures/image_tmp.jpg" + ) + + upload = %Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + assert { + :ok, + %Pleroma.Upload{id: @shasum, path: "#{@shasum}.jpg"} + } = Dedupe.filter(upload) + end +end diff --git a/test/upload/filter/mogrifun_test.exs b/test/upload/filter/mogrifun_test.exs new file mode 100644 index 000000000..d5a8751cc --- /dev/null +++ b/test/upload/filter/mogrifun_test.exs @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.MogrifunTest do + use Pleroma.DataCase + import Mock + + alias Pleroma.Upload + alias Pleroma.Upload.Filter + + test "apply mogrify filter" do + File.cp!( + "test/fixtures/image.jpg", + "test/fixtures/image_tmp.jpg" + ) + + upload = %Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + task = + Task.async(fn -> + assert_receive {:apply_filter, {}}, 4_000 + end) + + with_mocks([ + {Mogrify, [], + [ + open: fn _f -> %Mogrify.Image{} end, + custom: fn _m, _a -> send(task.pid, {:apply_filter, {}}) end, + custom: fn _m, _a, _o -> send(task.pid, {:apply_filter, {}}) end, + save: fn _f, _o -> :ok end + ]} + ]) do + assert Filter.Mogrifun.filter(upload) == :ok + end + + Task.await(task) + end +end diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs new file mode 100644 index 000000000..c301440fd --- /dev/null +++ b/test/upload/filter/mogrify_test.exs @@ -0,0 +1,51 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.MogrifyTest do + use Pleroma.DataCase + import Mock + + alias Pleroma.Config + alias Pleroma.Upload + alias Pleroma.Upload.Filter + + setup do + filter = Config.get([Filter.Mogrify, :args]) + + on_exit(fn -> + Config.put([Filter.Mogrify, :args], filter) + end) + end + + test "apply mogrify filter" do + Config.put([Filter.Mogrify, :args], [{"tint", "40"}]) + + File.cp!( + "test/fixtures/image.jpg", + "test/fixtures/image_tmp.jpg" + ) + + upload = %Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + task = + Task.async(fn -> + assert_receive {:apply_filter, {_, "tint", "40"}}, 4_000 + end) + + with_mock Mogrify, + open: fn _f -> %Mogrify.Image{} end, + custom: fn _m, _a -> :ok end, + custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end, + save: fn _f, _o -> :ok end do + assert Filter.Mogrify.filter(upload) == :ok + end + + Task.await(task) + end +end diff --git a/test/upload/filter_test.exs b/test/upload/filter_test.exs new file mode 100644 index 000000000..640cd7107 --- /dev/null +++ b/test/upload/filter_test.exs @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.FilterTest do + use Pleroma.DataCase + + alias Pleroma.Config + alias Pleroma.Upload.Filter + + setup do + custom_filename = Config.get([Pleroma.Upload.Filter.AnonymizeFilename, :text]) + + on_exit(fn -> + Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], custom_filename) + end) + end + + test "applies filters" do + Config.put([Pleroma.Upload.Filter.AnonymizeFilename, :text], "custom-file.png") + + File.cp!( + "test/fixtures/image.jpg", + "test/fixtures/image_tmp.jpg" + ) + + upload = %Pleroma.Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + assert Filter.filter([], upload) == {:ok, upload} + + assert {:ok, upload} = Filter.filter([Pleroma.Upload.Filter.AnonymizeFilename], upload) + assert upload.name == "custom-file.png" + end +end From 10f82c88b88fa4d26f6fa57f9cf36439012b8d0c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 16 Jul 2019 21:44:50 +0000 Subject: [PATCH 11/48] mastoapi password reset added rate limit to password reset configure rate limit in runtime --- CHANGELOG.md | 2 + config/config.exs | 3 +- config/test.exs | 3 +- .../mastodon_api/mastodon_api_controller.ex | 17 ++++++ lib/pleroma/web/router.ex | 2 + .../web/twitter_api/twitter_api_controller.ex | 7 +++ .../mastodon_api_controller_test.exs | 52 +++++++++++++++++++ .../twitter_api_controller_test.exs | 10 ++-- 8 files changed, 90 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3630a1c5..c5632f273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add support for muting/unmuting notifications - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). - Mastodon API: Add `pleroma.deactivated` to the Account entity +- Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID @@ -40,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`. - Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options. - Addressable lists +- Twitter API: added rate limit for `/api/account/password_reset` endpoint. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/config/config.exs b/config/config.exs index 03e0341c8..38780eef7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -531,7 +531,8 @@ relations_actions: {10_000, 10}, relation_id_action: {60_000, 2}, statuses_actions: {10_000, 15}, - status_id_action: {60_000, 3} + status_id_action: {60_000, 3}, + password_reset: {1_800_000, 5} # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 96ecf3592..0af62aa14 100644 --- a/config/test.exs +++ b/config/test.exs @@ -67,7 +67,8 @@ config :pleroma, :rate_limit, search: [{1000, 30}, {1000, 30}], - app_account_creation: {10_000, 5} + app_account_creation: {10_000, 5}, + password_reset: {1000, 30} config :pleroma, :http_security, report_uri: "https://endpoint.com" diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index a732a6990..aff76e2ea 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions) plug(RateLimiter, :app_account_creation when action == :account_register) plug(RateLimiter, :search when action in [:search, :search2, :account_search]) + plug(RateLimiter, :password_reset when action == :password_reset) @local_mastodon_name "Mastodon-Local" @@ -1816,6 +1817,22 @@ def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_ end end + def password_reset(conn, params) do + nickname_or_email = params["email"] || params["nickname"] + + with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do + conn + |> put_status(:no_content) + |> json("") + else + {:error, "unknown user"} -> + put_status(conn, :not_found) + + {:error, _} -> + put_status(conn, :bad_request) + end + end + def try_render(conn, target, params) when is_binary(target) do case render(conn, target, params) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3e5142e8a..52b8dc0bf 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -691,6 +691,8 @@ defmodule Pleroma.Web.Router do get("/web/login", MastodonAPIController, :login) delete("/auth/sign_out", MastodonAPIController, :logout) + post("/auth/password", MastodonAPIController, :password_reset) + scope [] do pipe_through(:oauth_read_or_public) get("/web/*path", MastodonAPIController, :index) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 0313560a8..8cb703501 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do require Logger + plug(Pleroma.Plugs.RateLimiter, :password_reset when action == :password_reset) plug(:only_if_public_instance when action in [:public_timeline, :public_and_external_timeline]) action_fallback(:errors) @@ -437,6 +438,12 @@ def password_reset(conn, params) do with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do json_response(conn, :no_content, "") + else + {:error, "unknown user"} -> + put_status(conn, :not_found) + + {:error, _} -> + put_status(conn, :bad_request) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 85b4ad024..d9d8dafdb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -23,6 +23,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do import Pleroma.Factory import ExUnit.CaptureLog import Tesla.Mock + import Swoosh.TestAssertions @image "data:image/gif;base64,R0lGODlhEAAQAMQAAORHHOVSKudfOulrSOp3WOyDZu6QdvCchPGolfO0o/XBs/fNwfjZ0frl3/zy7////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAkAABAALAAAAAAQABAAAAVVICSOZGlCQAosJ6mu7fiyZeKqNKToQGDsM8hBADgUXoGAiqhSvp5QAnQKGIgUhwFUYLCVDFCrKUE1lBavAViFIDlTImbKC5Gm2hB0SlBCBMQiB0UjIQA7" @@ -3807,4 +3808,55 @@ test "returns empty array when status has not been reblogged yet", %{ assert Enum.empty?(response) end end + + describe "POST /auth/password, with valid parameters" do + setup %{conn: conn} do + user = insert(:user) + conn = post(conn, "/auth/password?email=#{user.email}") + %{conn: conn, user: user} + end + + test "it returns 204", %{conn: conn} do + assert json_response(conn, :no_content) + end + + test "it creates a PasswordResetToken record for user", %{user: user} do + token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) + assert token_record + end + + test "it sends an email to user", %{user: user} do + token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) + + email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token) + notify_email = Pleroma.Config.get([:instance, :notify_email]) + instance_name = Pleroma.Config.get([:instance, :name]) + + assert_email_sent( + from: {instance_name, notify_email}, + to: {user.name, user.email}, + html_body: email.html_body + ) + end + end + + describe "POST /auth/password, with invalid parameters" do + setup do + user = insert(:user) + {:ok, user: user} + end + + test "it returns 404 when user is not found", %{conn: conn, user: user} do + conn = post(conn, "/auth/password?email=nonexisting_#{user.email}") + assert conn.status == 404 + refute conn.resp_body + end + + test "it returns 400 when user is not local", %{conn: conn, user: user} do + {:ok, user} = Repo.update(Changeset.change(user, local: false)) + conn = post(conn, "/auth/password?email=#{user.email}") + assert conn.status == 400 + refute conn.resp_body + end + end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index de6177575..622bf510e 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1116,15 +1116,17 @@ test "it sends an email to user", %{user: user} do describe "POST /api/account/password_reset, with invalid parameters" do setup [:valid_user] - test "it returns 500 when user is not found", %{conn: conn, user: user} do + test "it returns 404 when user is not found", %{conn: conn, user: user} do conn = post(conn, "/api/account/password_reset?email=nonexisting_#{user.email}") - assert json_response(conn, :internal_server_error) + assert conn.status == 404 + refute conn.resp_body end - test "it returns 500 when user is not local", %{conn: conn, user: user} do + test "it returns 400 when user is not local", %{conn: conn, user: user} do {:ok, user} = Repo.update(Changeset.change(user, local: false)) conn = post(conn, "/api/account/password_reset?email=#{user.email}") - assert json_response(conn, :internal_server_error) + assert conn.status == 400 + refute conn.resp_body end end From 96a2890a9ecca3a6392edfaaaed4487303a920d7 Mon Sep 17 00:00:00 2001 From: RX14 Date: Wed, 17 Jul 2019 14:55:47 +0100 Subject: [PATCH 12/48] Add MRF MentionPolicy for dropping posts which mention specific actors --- CHANGELOG.md | 1 + docs/config.md | 4 + .../web/activity_pub/mrf/mention_policy.ex | 24 +++++ .../activity_pub/mrf/mention_policy_test.exs | 92 +++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/mention_policy.ex create mode 100644 test/web/activity_pub/mrf/mention_policy_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3630a1c5..0bc72a3fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - MRF: Support for excluding specific domains from Transparency. +- MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`) - Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header diff --git a/docs/config.md b/docs/config.md index 9a64f0ed7..5f69f8daf 100644 --- a/docs/config.md +++ b/docs/config.md @@ -101,6 +101,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. + * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` @@ -271,6 +272,9 @@ config :pleroma, :mrf_subchain, * `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) * `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +## :mrf_mention +* `actors`: A list of actors, for which to drop any posts mentioning. + ## :media_proxy * `enabled`: Enables proxying of remote media to the instance’s proxy * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex new file mode 100644 index 000000000..1842e1aeb --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do + @moduledoc "Block messages which mention a user" + + @behaviour Pleroma.Web.ActivityPub.MRF + + @impl true + def filter(%{"type" => "Create"} = message) do + reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) + recipients = (message["to"] || []) ++ (message["cc"] || []) + + if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do + {:reject, nil} + else + {:ok, message} + end + end + + @impl true + def filter(message), do: {:ok, message} +end diff --git a/test/web/activity_pub/mrf/mention_policy_test.exs b/test/web/activity_pub/mrf/mention_policy_test.exs new file mode 100644 index 000000000..9fd9c31df --- /dev/null +++ b/test/web/activity_pub/mrf/mention_policy_test.exs @@ -0,0 +1,92 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicyTest do + use Pleroma.DataCase + + alias Pleroma.Web.ActivityPub.MRF.MentionPolicy + + test "pass filter if allow list is empty" do + Pleroma.Config.delete([:mrf_mention]) + + message = %{ + "type" => "Create", + "to" => ["https://example.com/ok"], + "cc" => ["https://example.com/blocked"] + } + + assert MentionPolicy.filter(message) == {:ok, message} + end + + describe "allow" do + test "empty" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create" + } + + assert MentionPolicy.filter(message) == {:ok, message} + end + + test "to" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create", + "to" => ["https://example.com/ok"] + } + + assert MentionPolicy.filter(message) == {:ok, message} + end + + test "cc" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create", + "cc" => ["https://example.com/ok"] + } + + assert MentionPolicy.filter(message) == {:ok, message} + end + + test "both" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create", + "to" => ["https://example.com/ok"], + "cc" => ["https://example.com/ok2"] + } + + assert MentionPolicy.filter(message) == {:ok, message} + end + end + + describe "deny" do + test "to" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create", + "to" => ["https://example.com/blocked"] + } + + assert MentionPolicy.filter(message) == {:reject, nil} + end + + test "cc" do + Pleroma.Config.put([:mrf_mention], %{actors: ["https://example.com/blocked"]}) + + message = %{ + "type" => "Create", + "to" => ["https://example.com/ok"], + "cc" => ["https://example.com/blocked"] + } + + assert MentionPolicy.filter(message) == {:reject, nil} + end + end +end From 4885473be2704d4d370fdb96e1473bc4eb9368f4 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 15:48:51 +0000 Subject: [PATCH 13/48] user: refactor get_or_create_instance_user() into get_or_create_service_actor_by_id() --- lib/pleroma/user.ex | 13 ++++++------- lib/pleroma/web/activity_pub/relay.ex | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ffba3f390..463bb9ad4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1157,19 +1157,18 @@ def get_or_fetch_by_ap_id(ap_id) do end end - def get_or_create_instance_user do - relay_uri = "#{Pleroma.Web.Endpoint.url()}/relay" - - if user = get_cached_by_ap_id(relay_uri) do + @doc "Creates an internal service actor by URI if missing. Optionally takes nickname for addressing." + def get_or_create_service_actor_by_ap_id(uri, nickname \\ nil) do + if user = get_cached_by_ap_id(uri) do user else changes = %User{info: %User.Info{}} |> cast(%{}, [:ap_id, :nickname, :local]) - |> put_change(:ap_id, relay_uri) - |> put_change(:nickname, nil) + |> put_change(:ap_id, uri) + |> put_change(:nickname, nickname) |> put_change(:local, true) - |> put_change(:follower_address, relay_uri <> "/followers") + |> put_change(:follower_address, uri <> "/followers") {:ok, user} = Repo.insert(changes) user diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 93808517b..1ebfcdd86 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -10,7 +10,8 @@ defmodule Pleroma.Web.ActivityPub.Relay do require Logger def get_actor do - User.get_or_create_instance_user() + "#{Pleroma.Web.Endpoint.url()}/relay" + |> User.get_or_create_service_actor_by_ap_id() end def follow(target_instance) do From a9d6a12bb3e71bafc11fe7cf5e44ad5bc9981cf9 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 16:22:57 +0000 Subject: [PATCH 14/48] activitypub: controller: rework the way the relay actor is presented so the code can be reused --- .../web/activity_pub/activity_pub_controller.ex | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e2af4ad1a..dc9ef066d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -206,9 +206,8 @@ def inbox(conn, params) do json(conn, dgettext("errors", "error")) end - def relay(conn, _params) do - with %User{} = user <- Relay.get_actor(), - {:ok, user} <- User.ensure_keys_present(user) do + defp represent_service_actor(%User{} = user, conn) do + with {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) @@ -217,6 +216,13 @@ def relay(conn, _params) do end end + defp represent_service_actor(nil, _), do: {:error, :not_found} + + def relay(conn, _params) do + Relay.get_actor() + |> represent_service_actor(conn) + end + def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do conn |> put_resp_header("content-type", "application/activity+json") From 0a6f6e1b5bf863fc23a90e6ef358129364bcacce Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 16:59:29 +0000 Subject: [PATCH 15/48] webfinger: allow resolution of usernames with dots in them (internal actors) --- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 3fca72de8..fa34c7ced 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -32,7 +32,7 @@ def host_meta do def webfinger(resource, fmt) when fmt in ["XML", "JSON"] do host = Pleroma.Web.Endpoint.host() - regex = ~r/(acct:)?(?\w+)@#{host}/ + regex = ~r/(acct:)?(?[a-z0-9A-Z_\.-]+)@#{host}/ with %{"username" => username} <- Regex.named_captures(regex, resource), %User{} = user <- User.get_cached_by_nickname(username) do From 62e5ff624e1984b48eecaf2a6c14ad092013a13e Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 17:12:42 +0000 Subject: [PATCH 16/48] user: add is_internal_user? helper function --- lib/pleroma/user.ex | 4 ++++ test/user_test.exs | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 463bb9ad4..c91fbb68a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1410,4 +1410,8 @@ defp put_password_hash( end defp put_password_hash(changeset), do: changeset + + def is_internal_user?(%User{nickname: nil}), do: true + def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true + def is_internal_user?(_), do: false end diff --git a/test/user_test.exs b/test/user_test.exs index 264b7a40e..908f72a0e 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1310,4 +1310,21 @@ test "without args", %{user: user} do assert following == 0 end end + + describe "is_internal_user?/1" do + test "non-internal user returns false" do + user = insert(:user) + refute User.is_internal_user?(user) + end + + test "user with no nickname returns true" do + user = insert(:user, %{nickname: nil}) + assert User.is_internal_user?(user) + end + + test "user with internal-prefixed nickname returns true" do + user = insert(:user, %{nickname: "internal.test"}) + assert User.is_internal_user?(user) + end + end end From d930e5d5c322a7c4b3a080dfa3e318d97994aaf1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 17:14:08 +0000 Subject: [PATCH 17/48] activitypub: introduce internal fetch service actor --- lib/pleroma/application.ex | 5 +++++ .../web/activity_pub/internal_fetch_actor.ex | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/internal_fetch_actor.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ba4cf8486..035331491 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -140,6 +140,11 @@ def start(_type, _args) do id: :federator_init, start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, restart: :temporary + }, + %{ + id: :internal_fetch_init, + start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, + restart: :temporary } ] ++ streamer_child() ++ diff --git a/lib/pleroma/web/activity_pub/internal_fetch_actor.ex b/lib/pleroma/web/activity_pub/internal_fetch_actor.ex new file mode 100644 index 000000000..9213ddde7 --- /dev/null +++ b/lib/pleroma/web/activity_pub/internal_fetch_actor.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.InternalFetchActor do + alias Pleroma.User + + require Logger + + def init do + # Wait for everything to settle. + Process.sleep(1000 * 5) + get_actor() + end + + def get_actor do + "#{Pleroma.Web.Endpoint.url()}/internal/fetch" + |> User.get_or_create_service_actor_by_ap_id("internal.fetch") + end +end From cf9cb953d5c341bb13e4b86272ff4ef405aeab92 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 17:34:57 +0000 Subject: [PATCH 18/48] activitypub: represent internal fetch actor --- .../web/activity_pub/activity_pub_controller.ex | 6 ++++++ lib/pleroma/web/activity_pub/views/user_view.ex | 13 ++++++++++--- lib/pleroma/web/router.ex | 13 +++++++++++-- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dc9ef066d..133a726c5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do alias Pleroma.Object.Fetcher alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier @@ -223,6 +224,11 @@ def relay(conn, _params) do |> represent_service_actor(conn) end + def internal_fetch(conn, _params) do + InternalFetchActor.get_actor() + |> represent_service_actor(conn) + end + def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do conn |> put_resp_header("content-type", "application/activity+json") diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index d9c1bcb2c..639519e0a 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -31,8 +31,7 @@ def render("endpoints.json", %{user: %User{local: true} = _user}) do def render("endpoints.json", _), do: %{} - # the instance itself is not a Person, but instead an Application - def render("user.json", %{user: %{nickname: nil} = user}) do + def render("service.json", %{user: user}) do {:ok, user} = User.ensure_keys_present(user) {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) @@ -47,7 +46,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", "name" => "Pleroma", - "summary" => "Virtual actor for Pleroma relay", + "summary" => + "An internal service actor for this Pleroma instance. No user-serviceable parts inside.", "url" => user.ap_id, "manuallyApprovesFollowers" => false, "publicKey" => %{ @@ -60,6 +60,13 @@ def render("user.json", %{user: %{nickname: nil} = user}) do |> Map.merge(Utils.make_json_ld_header()) end + # the instance itself is not a Person, but instead an Application + def render("user.json", %{user: %User{nickname: nil} = user}), + do: render("service.json", %{user: user}) + + def render("user.json", %{user: %User{nickname: "internal." <> _} = user}), + do: render("service.json", %{user: user}) + def render("user.json", %{user: user}) do {:ok, user} = User.ensure_keys_present(user) {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 52b8dc0bf..8095ac4b1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -586,7 +586,7 @@ defmodule Pleroma.Web.Router do end end - pipeline :ap_relay do + pipeline :ap_service_actor do plug(:accepts, ["activity+json", "json"]) end @@ -663,8 +663,17 @@ defmodule Pleroma.Web.Router do end scope "/relay", Pleroma.Web.ActivityPub do - pipe_through(:ap_relay) + pipe_through(:ap_service_actor) + get("/", ActivityPubController, :relay) + post("/inbox", ActivityPubController, :inbox) + end + + scope "/internal/fetch", Pleroma.Web.ActivityPub do + pipe_through(:ap_service_actor) + + get("/", ActivityPubController, :internal_fetch) + post("/inbox", ActivityPubController, :inbox) end scope "/", Pleroma.Web.ActivityPub do From 3d23a12d75fc159d3ec25424245847fe703b7bd6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 17:48:08 +0000 Subject: [PATCH 19/48] tests: add test for fetching the internal fetch actor --- .../web/activity_pub/activity_pub_controller_test.exs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 452172bb4..40344f17e 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -48,6 +48,17 @@ test "with the relay disabled, it returns 404", %{conn: conn} do end end + describe "/internal/fetch" do + test "it returns the internal fetch user", %{conn: conn} do + res = + conn + |> get(activity_pub_path(conn, :internal_fetch)) + |> json_response(200) + + assert res["id"] =~ "/fetch" + end + end + describe "/users/:nickname" do test "it returns a json representation of the user with accept application/json", %{ conn: conn From c1198351022ead54b8de3bc535df32e333eb9b4d Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 17:49:21 +0000 Subject: [PATCH 20/48] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d91ad817..654d208dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options. - Addressable lists - Twitter API: added rate limit for `/api/account/password_reset` endpoint. +- ActivityPub: Add an internal service actor for fetching ActivityPub objects. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text From 4bf2bb9cff2d263e1b022f5c40128ffcbd372746 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Wed, 17 Jul 2019 18:09:31 +0000 Subject: [PATCH 21/48] Fix password reset for non-test env Fixes `Plug.Conn.NotSentError` that causes a 5xx error in response instead of 404 and 400. Fixes pattern matching error caused by different response format in test and non-test env: `Pleroma.Emails.Mailer.deliver_async` returns :ok when PleromaJobQueue is enabled and `{:ok, _}` when it's disabled. In tests, it's disabled. --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- lib/pleroma/web/twitter_api/twitter_api.ex | 2 ++ lib/pleroma/web/twitter_api/twitter_api_controller.ex | 4 ++-- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 ++-- test/web/twitter_api/twitter_api_controller_test.exs | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index aff76e2ea..877430a1d 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1826,10 +1826,10 @@ def password_reset(conn, params) do |> json("") else {:error, "unknown user"} -> - put_status(conn, :not_found) + send_resp(conn, :not_found, "") {:error, _} -> - put_status(conn, :bad_request) + send_resp(conn, :bad_request, "") end end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 41e1c2877..bb5dda204 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -221,6 +221,8 @@ def password_reset(nickname_or_email) do user |> UserEmail.password_reset_email(token_record.token) |> Mailer.deliver_async() + + {:ok, :enqueued} else false -> {:error, "bad user identifier"} diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 8cb703501..5dfab6a6c 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -440,10 +440,10 @@ def password_reset(conn, params) do json_response(conn, :no_content, "") else {:error, "unknown user"} -> - put_status(conn, :not_found) + send_resp(conn, :not_found, "") {:error, _} -> - put_status(conn, :bad_request) + send_resp(conn, :bad_request, "") end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index d9d8dafdb..b4b1dd785 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3849,14 +3849,14 @@ test "it sends an email to user", %{user: user} do test "it returns 404 when user is not found", %{conn: conn, user: user} do conn = post(conn, "/auth/password?email=nonexisting_#{user.email}") assert conn.status == 404 - refute conn.resp_body + assert conn.resp_body == "" end test "it returns 400 when user is not local", %{conn: conn, user: user} do {:ok, user} = Repo.update(Changeset.change(user, local: false)) conn = post(conn, "/auth/password?email=#{user.email}") assert conn.status == 400 - refute conn.resp_body + assert conn.resp_body == "" end end end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 622bf510e..8bb8aa36d 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1119,14 +1119,14 @@ test "it sends an email to user", %{user: user} do test "it returns 404 when user is not found", %{conn: conn, user: user} do conn = post(conn, "/api/account/password_reset?email=nonexisting_#{user.email}") assert conn.status == 404 - refute conn.resp_body + assert conn.resp_body == "" end test "it returns 400 when user is not local", %{conn: conn, user: user} do {:ok, user} = Repo.update(Changeset.change(user, local: false)) conn = post(conn, "/api/account/password_reset?email=#{user.email}") assert conn.status == 400 - refute conn.resp_body + assert conn.resp_body == "" end end From 1e3aff6ef18c774783c4fc7eb46c245e7d8fb9b2 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 19:00:26 +0000 Subject: [PATCH 22/48] mix: bump http_signatures dependency to a2a5982fa167fb1352fbd518ce6b606ba233a989 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index a26bb6202..e4ea19bb6 100644 --- a/mix.exs +++ b/mix.exs @@ -138,7 +138,7 @@ defp deps do ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", - ref: "9789401987096ead65646b52b5a2ca6bf52fc531"}, + ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"}, {:pleroma_job_queue, "~> 0.2.0"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index dcbf80f01..6477c1ed5 100644 --- a/mix.lock +++ b/mix.lock @@ -38,7 +38,7 @@ "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, - "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "9789401987096ead65646b52b5a2ca6bf52fc531", [ref: "9789401987096ead65646b52b5a2ca6bf52fc531"]}, + "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "a2a5982fa167fb1352fbd518ce6b606ba233a989", [ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"]}, "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, From f84fb340b7358df195734f2db199e76a819e45bf Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 19:18:19 +0000 Subject: [PATCH 23/48] http signatures: derive actor ID from key ID. Almost all AP servers return their key ID as the actor URI with #main-key added. Hubzilla, which doesn't, uses a URL which refers to the actor anyway, so worst case, Hubzilla users get refetched. --- lib/pleroma/signature.ex | 13 ++++++++++--- test/signature_test.exs | 16 ++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 1a4d54c62..a45c70a9d 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -8,10 +8,16 @@ defmodule Pleroma.Signature do alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Utils + + defp key_id_to_actor_id(key_id) do + URI.parse(key_id) + |> Map.put(:fragment, nil) + |> URI.to_string() + end def fetch_public_key(conn) do - with actor_id <- Utils.get_ap_id(conn.params["actor"]), + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + actor_id <- key_id_to_actor_id(kid), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -21,7 +27,8 @@ def fetch_public_key(conn) do end def refetch_public_key(conn) do - with actor_id <- Utils.get_ap_id(conn.params["actor"]), + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + actor_id <- key_id_to_actor_id(kid), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} diff --git a/test/signature_test.exs b/test/signature_test.exs index 4920196c7..840987cd6 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -31,25 +31,29 @@ defmodule Pleroma.SignatureTest do 65_537 } + defp make_fake_signature(key_id), do: "keyId=\"#{key_id}\"" + + defp make_fake_conn(key_id), + do: %Plug.Conn{req_headers: %{"signature" => make_fake_signature(key_id <> "#main-key")}} + describe "fetch_public_key/1" do test "it returns key" do expected_result = {:ok, @rsa_public_key} user = insert(:user, %{info: %{source_data: %{"publicKey" => @public_key}}}) - assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => user.ap_id}}) == - expected_result + assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == expected_result end test "it returns error when not found user" do - assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => "test-ap_id"}}) == + assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == {:error, :error} end test "it returns error if public key is empty" do user = insert(:user, %{info: %{source_data: %{"publicKey" => %{}}}}) - assert Signature.fetch_public_key(%Plug.Conn{params: %{"actor" => user.ap_id}}) == + assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == {:error, :error} end end @@ -58,12 +62,12 @@ test "it returns error if public key is empty" do test "it returns key" do ap_id = "https://mastodon.social/users/lambadalambda" - assert Signature.refetch_public_key(%Plug.Conn{params: %{"actor" => ap_id}}) == + assert Signature.refetch_public_key(make_fake_conn(ap_id)) == {:ok, @rsa_public_key} end test "it returns error when not found user" do - assert Signature.refetch_public_key(%Plug.Conn{params: %{"actor" => "test-ap_id"}}) == + assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == {:error, {:error, :ok}} end end From b2a8ccf37fac6e40e55ce9b224c66ef1fd655614 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 21:38:06 +0000 Subject: [PATCH 24/48] config: add sign_object_fetches option --- config/config.exs | 3 ++- docs/config.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 38780eef7..bda92e45d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -305,7 +305,8 @@ accept_blocks: true, unfollow_blocked: true, outgoing_blocks: true, - follow_handshake_timeout: 500 + follow_handshake_timeout: 500, + sign_object_fetches: true config :pleroma, :user, deny_follow_blocked: true diff --git a/docs/config.md b/docs/config.md index 42556a0be..02f86dc16 100644 --- a/docs/config.md +++ b/docs/config.md @@ -332,6 +332,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``unfollow_blocked``: Whether blocks result in people getting unfollowed * ``outgoing_blocks``: Whether to federate blocks to other instances * ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question +* ``sign_object_fetches``: Sign object fetches with HTTP signatures ## :http_security * ``enabled``: Whether the managed content security policy is enabled From cdecef2a084eca629d2c6f72d083362a2113dbd2 Mon Sep 17 00:00:00 2001 From: "Bradley D. Thornton" Date: Wed, 17 Jul 2019 22:35:16 +0000 Subject: [PATCH 25/48] Update otp_en.md - Added creation of initial admin user --- docs/installation/otp_en.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 9b851e395..5b50e1838 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -242,6 +242,14 @@ So for example, if the task is `mix pleroma.user set admin --admin`, you should ```sh su pleroma -s $SHELL -lc "./bin/pleroma_ctl user set admin --admin" ``` + +## Create your first user and set as admin +```sh +cd /opt/pleroma/bin +su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin" +``` +This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password. + ### Updating Generally, doing the following is enough: ```sh From 399acd4c42ae3f1c072c097ff9fabc9761c3d3c1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 22:41:42 +0000 Subject: [PATCH 26/48] fetcher: sign object fetches if configured --- lib/pleroma/object/fetcher.ex | 49 +++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 96b34ae9f..305ce8357 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Object.Containment + alias Pleroma.Signature + alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.OStatus @@ -82,15 +84,52 @@ def fetch_object_from_id!(id, options \\ []) do end end + defp make_signature(id, date) do + uri = URI.parse(id) + + signature = + InternalFetchActor.get_actor() + |> Signature.sign(%{ + "(request-target)": "get #{uri.path}", + host: uri.host, + date: date + }) + + [{:Signature, signature}] + end + + defp sign_fetch(headers, id, date) do + if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do + headers ++ make_signature(id, date) + else + headers + end + end + + defp maybe_date_fetch(headers, date) do + if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do + headers ++ [{:Date, date}] + else + headers + end + end + def fetch_and_contain_remote_object_from_id(id) do Logger.info("Fetching object #{id} via AP") + date = + NaiveDateTime.utc_now() + |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") + + headers = + [{:Accept, "application/activity+json"}] + |> maybe_date_fetch(date) + |> sign_fetch(id, date) + + Logger.debug("Fetch headers: #{inspect(headers)}") + with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status: code}} when code in 200..299 <- - HTTP.get( - id, - [{:Accept, "application/activity+json"}] - ), + {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers), {:ok, data} <- Jason.decode(body), :ok <- Containment.contain_origin_from_id(id, data) do {:ok, data} From cffe8229a570ac883d6966dae97c97dba19412f0 Mon Sep 17 00:00:00 2001 From: tallship Date: Wed, 17 Jul 2019 23:03:46 +0000 Subject: [PATCH 27/48] Update clients.md - added Homepage for Fedilab Android app --- docs/clients.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/clients.md b/docs/clients.md index 30358c210..91096970e 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -31,6 +31,7 @@ Feel free to contact us to be added to this list! - Features: No Streaming ### Fedilab +- Homepage: - Source Code: - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) - Platforms: Android From 1345e0c2bf51c7eb8add41a25683e121c83c1ff8 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 22:58:52 +0000 Subject: [PATCH 28/48] tests: add tests for signed object fetches --- config/test.exs | 2 ++ test/object/fetcher_test.exs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/config/test.exs b/config/test.exs index 0af62aa14..92dca18bc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -31,6 +31,8 @@ skip_thread_containment: false, federating: false +config :pleroma, :activitypub, sign_object_fetches: false + # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 56a9d775f..482252cff 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -150,4 +150,34 @@ test "it can refetch pruned objects" do assert object.id != object_two.id end end + + describe "signed fetches" do + test_with_mock "it signs fetches when configured to do so", + Pleroma.Signature, + [:passthrough], + [] do + option = Pleroma.Config.get([:activitypub, :sign_object_fetches]) + Pleroma.Config.put([:activitypub, :sign_object_fetches], true) + + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert called(Pleroma.Signature.sign(:_, :_)) + + Pleroma.Config.put([:activitypub, :sign_object_fetches], option) + end + + test_with_mock "it doesn't sign fetches when not configured to do so", + Pleroma.Signature, + [:passthrough], + [] do + option = Pleroma.Config.get([:activitypub, :sign_object_fetches]) + Pleroma.Config.put([:activitypub, :sign_object_fetches], false) + + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + refute called(Pleroma.Signature.sign(:_, :_)) + + Pleroma.Config.put([:activitypub, :sign_object_fetches], option) + end + end end From 7bb50bfcccdc08f72113fda5732b87876eb01df8 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Wed, 17 Jul 2019 23:07:53 +0000 Subject: [PATCH 29/48] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 654d208dd..6f268d110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Addressable lists - Twitter API: added rate limit for `/api/account/password_reset` endpoint. - ActivityPub: Add an internal service actor for fetching ActivityPub objects. +- ActivityPub: Optional signing of ActivityPub object fetches. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text From 129078eddc1422e3654514477fe438b56dcff562 Mon Sep 17 00:00:00 2001 From: aries Date: Thu, 18 Jul 2019 21:08:31 +0900 Subject: [PATCH 30/48] Fix redirect setting not working --- docs/config/howto_mediaproxy.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/config/howto_mediaproxy.md b/docs/config/howto_mediaproxy.md index fb731112b..ed70c3ed4 100644 --- a/docs/config/howto_mediaproxy.md +++ b/docs/config/howto_mediaproxy.md @@ -24,7 +24,9 @@ If you came here from one of the installation guides, take a look at the example ``` config :pleroma, :media_proxy, enabled: true, - redirect_on_failure: true + proxy_opts: [ + redirect_on_failure: true + ] #base_url: "https://cache.pleroma.social" ``` If you want to use a subdomain to serve the files, uncomment `base_url`, change the url and add a comma after `true` in the previous line. From b6b748d3e7f3383303a2fcccb17b7b0f7054100f Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 18 Jul 2019 12:30:18 +0000 Subject: [PATCH 31/48] tests for Uploader with webhook --- lib/pleroma/uploaders/uploader.ex | 9 ++- lib/pleroma/web/uploader_controller.ex | 4 -- test/support/data_case.ex | 17 +++-- test/upload_test.exs | 89 +++++++++++++++++++++++++- test/web/uploader_controller_test.exs | 43 +++++++++++++ 5 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 test/web/uploader_controller_test.exs diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index 0af76bc59..c0b22c28a 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -68,7 +68,14 @@ defp handle_callback(uploader, upload) do {:error, error} end after - 30_000 -> {:error, dgettext("errors", "Uploader callback timeout")} + callback_timeout() -> {:error, dgettext("errors", "Uploader callback timeout")} + end + end + + defp callback_timeout do + case Mix.env() do + :test -> 1_000 + _ -> 30_000 end end end diff --git a/lib/pleroma/web/uploader_controller.ex b/lib/pleroma/web/uploader_controller.ex index bf09775e6..0cc172698 100644 --- a/lib/pleroma/web/uploader_controller.ex +++ b/lib/pleroma/web/uploader_controller.ex @@ -11,10 +11,6 @@ def callback(conn, %{"upload_path" => upload_path} = params) do process_callback(conn, :global.whereis_name({Uploader, upload_path}), params) end - def callbacks(conn, _) do - render_error(conn, :bad_request, "bad request") - end - defp process_callback(conn, pid, params) when is_pid(pid) do send(pid, {Uploader, self(), conn, params}) diff --git a/test/support/data_case.ex b/test/support/data_case.ex index df260bd3f..f3d98e7e3 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -42,19 +42,18 @@ defmodule Pleroma.DataCase do :ok end - def ensure_local_uploader(_context) do + def ensure_local_uploader(context) do + test_uploader = Map.get(context, :uploader, Pleroma.Uploaders.Local) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) filters = Pleroma.Config.get([Pleroma.Upload, :filters]) - unless uploader == Pleroma.Uploaders.Local || filters != [] do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - Pleroma.Config.put([Pleroma.Upload, :filters], []) + Pleroma.Config.put([Pleroma.Upload, :uploader], test_uploader) + Pleroma.Config.put([Pleroma.Upload, :filters], []) - on_exit(fn -> - Pleroma.Config.put([Pleroma.Upload, :uploader], uploader) - Pleroma.Config.put([Pleroma.Upload, :filters], filters) - end) - end + on_exit(fn -> + Pleroma.Config.put([Pleroma.Upload, :uploader], uploader) + Pleroma.Config.put([Pleroma.Upload, :filters], filters) + end) :ok end diff --git a/test/upload_test.exs b/test/upload_test.exs index 946ebcb5a..f7b1893ad 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -3,9 +3,96 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.UploadTest do - alias Pleroma.Upload use Pleroma.DataCase + alias Pleroma.Upload + alias Pleroma.Uploaders.Uploader + + @upload_file %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + filename: "image.jpg" + } + + defmodule TestUploaderBase do + def put_file(%{path: path} = _upload, module_name) do + task_pid = + Task.async(fn -> + :timer.sleep(10) + + {Uploader, path} + |> :global.whereis_name() + |> send({Uploader, self(), {:test}, %{}}) + + assert_receive {Uploader, {:test}}, 4_000 + end) + + Agent.start(fn -> task_pid end, name: module_name) + + :wait_callback + end + end + + describe "Tried storing a file when http callback response success result" do + defmodule TestUploaderSuccess do + def http_callback(conn, _params), + do: {:ok, conn, {:file, "post-process-file.jpg"}} + + def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) + end + + setup do: [uploader: TestUploaderSuccess] + setup [:ensure_local_uploader] + + test "it returns file" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + + assert Upload.store(@upload_file) == + {:ok, + %{ + "name" => "image.jpg", + "type" => "Document", + "url" => [ + %{ + "href" => "http://localhost:4001/media/post-process-file.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + }} + + Task.await(Agent.get(TestUploaderSuccess, fn task_pid -> task_pid end)) + end + end + + describe "Tried storing a file when http callback response error" do + defmodule TestUploaderError do + def http_callback(conn, _params), do: {:error, conn, "Errors"} + + def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) + end + + setup do: [uploader: TestUploaderError] + setup [:ensure_local_uploader] + + test "it returns error" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + assert Upload.store(@upload_file) == {:error, "Errors"} + Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end)) + end + end + + describe "Tried storing a file when http callback doesn't response by timeout" do + defmodule(TestUploader, do: def(put_file(_upload), do: :wait_callback)) + setup do: [uploader: TestUploader] + setup [:ensure_local_uploader] + + test "it returns error" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"} + end + end + describe "Storing a file with the Local uploader" do setup [:ensure_local_uploader] diff --git a/test/web/uploader_controller_test.exs b/test/web/uploader_controller_test.exs new file mode 100644 index 000000000..70028df1c --- /dev/null +++ b/test/web/uploader_controller_test.exs @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.UploaderControllerTest do + use Pleroma.Web.ConnCase + alias Pleroma.Uploaders.Uploader + + describe "callback/2" do + test "it returns 400 response when process callback isn't alive", %{conn: conn} do + res = + conn + |> post(uploader_path(conn, :callback, "test-path")) + + assert res.status == 400 + assert res.resp_body == "{\"error\":\"bad request\"}" + end + + test "it returns success result", %{conn: conn} do + task = + Task.async(fn -> + receive do + {Uploader, pid, conn, _params} -> + conn = + conn + |> put_status(:ok) + |> Phoenix.Controller.json(%{upload_path: "test-path"}) + + send(pid, {Uploader, conn}) + end + end) + + :global.register_name({Uploader, "test-path"}, task.pid) + + res = + conn + |> post(uploader_path(conn, :callback, "test-path")) + |> json_response(200) + + assert res == %{"upload_path" => "test-path"} + end + end +end From 88d064d80e4a3272a2a7101089b5f924fd175866 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:06:58 +0000 Subject: [PATCH 32/48] http signature plug: remove redundant checks handled by HTTPSignatures library the redundant checks assumed a POST request, which will not work for signed GETs. this check was originally needed because the HTTPSignatures adapter assumed that the requests were also POST requests. but now, the adapter has been corrected. --- lib/pleroma/plugs/http_signature.ex | 51 ++++++++++--------------- test/plugs/http_signature_plug_test.exs | 18 --------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index e2874c469..d87fa52fa 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do - alias Pleroma.Web.ActivityPub.Utils import Plug.Conn require Logger @@ -16,38 +15,30 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do end def call(conn, _opts) do - user = Utils.get_ap_id(conn.params["actor"]) - Logger.debug("Checking sig for #{user}") [signature | _] = get_req_header(conn, "signature") - cond do - signature && String.contains?(signature, user) -> - # set (request-target) header to the appropriate value - # we also replace the digest header with the one we computed - conn = - conn - |> put_req_header( - "(request-target)", - String.downcase("#{conn.method}") <> " #{conn.request_path}" - ) - - conn = - if conn.assigns[:digest] do - conn - |> put_req_header("digest", conn.assigns[:digest]) - else - conn - end - - assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) - - signature -> - Logger.debug("Signature not from actor") - assign(conn, :valid_signature, false) - - true -> - Logger.debug("No signature header!") + if signature do + # set (request-target) header to the appropriate value + # we also replace the digest header with the one we computed + conn = conn + |> put_req_header( + "(request-target)", + String.downcase("#{conn.method}") <> " #{conn.request_path}" + ) + + conn = + if conn.assigns[:digest] do + conn + |> put_req_header("digest", conn.assigns[:digest]) + else + conn + end + + assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) + else + Logger.debug("No signature header!") + conn end end end diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index efd811df7..d6fd9ea81 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -26,22 +26,4 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do assert called(HTTPSignatures.validate_conn(:_)) end end - - test "bails out early if the signature isn't by the activity actor" do - params = %{"actor" => "https://mst3k.interlinked.me/users/luciferMysticus"} - conn = build_conn(:get, "/doesntmattter", params) - - with_mock HTTPSignatures, validate_conn: fn _ -> false end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == false - refute called(HTTPSignatures.validate_conn(:_)) - end - end end From 18d8d12d53567b7c0c246bb793ee724d0d2e4c77 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:35:42 +0000 Subject: [PATCH 33/48] signature: make key_id_to_actor_id() public --- lib/pleroma/signature.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index a45c70a9d..2a0823ecf 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Signature do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - defp key_id_to_actor_id(key_id) do + def key_id_to_actor_id(key_id) do URI.parse(key_id) |> Map.put(:fragment, nil) |> URI.to_string() From 184fa61fb3a1bc8c5d5515bb7748c12816b11ebf Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:38:45 +0000 Subject: [PATCH 34/48] plugs: add MappedSignatureToIdentityPlug --- .../mapped_signature_to_identity_plug.ex | 64 +++++++++++++++++++ lib/pleroma/web/router.ex | 1 + 2 files changed, 65 insertions(+) create mode 100644 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex new file mode 100644 index 000000000..ae9339595 --- /dev/null +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do + alias Pleroma.Signature + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Utils + + import Plug.Conn + require Logger + + def init(options), do: options + + defp key_id_from_conn(conn) do + with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do + Signature.key_id_to_actor_id(key_id) + else + _ -> + nil + end + end + + defp user_from_key_id(conn) do + with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn), + %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do + user + else + _ -> + nil + end + end + + def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn + + # if this has payload make sure it is signed by the same actor that made it + def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do + with actor_id <- Utils.get_ap_id(actor), + %User{} = user <- user_from_key_id(conn), + true <- user.ap_id == actor_id do + assign(conn, :mapped_identity, user) + else + _ -> + Logger.debug("Failed to map identity from signature (payload actor mismatch?)") + Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + conn + end + end + + # no payload, probably a signed fetch + def call(%{assigns: %{valid_signature: true}} = conn, _opts) do + with %User{} = user <- user_from_key_id(conn) do + assign(conn, :mapped_identity, user) + else + _ -> + Logger.debug("Failed to map identity from signature (no payload actor mismatch)") + Logger.debug("key_id=#{key_id_from_conn(conn)}") + conn + end + end + + # no signature at all + def call(conn, _opts), do: conn +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8095ac4b1..518720d38 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -617,6 +617,7 @@ defmodule Pleroma.Web.Router do pipeline :activitypub do plug(:accepts, ["activity+json", "json"]) plug(Pleroma.Web.Plugs.HTTPSignaturePlug) + plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) end scope "/", Pleroma.Web.ActivityPub do From cdf0038d0f5d56cf78f640fb149e9141e74800fc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 15:51:58 +0000 Subject: [PATCH 35/48] mix: update http signatures dependency --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index e4ea19bb6..c12b0a500 100644 --- a/mix.exs +++ b/mix.exs @@ -138,7 +138,7 @@ defp deps do ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", - ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"}, + ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:pleroma_job_queue, "~> 0.2.0"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 6477c1ed5..45142ba8f 100644 --- a/mix.lock +++ b/mix.lock @@ -38,7 +38,7 @@ "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, - "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "a2a5982fa167fb1352fbd518ce6b606ba233a989", [ref: "a2a5982fa167fb1352fbd518ce6b606ba233a989"]}, + "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, From 5ea0cd69f7457086fc486f13e072f13d2c1ef547 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:01:21 +0000 Subject: [PATCH 36/48] mapped signature plug: don't invalidate in cases where a signature is actually not present (testsuite) --- .../plugs/mapped_signature_to_identity_plug.ex | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index ae9339595..2a8ed4470 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -36,12 +36,18 @@ def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn # if this has payload make sure it is signed by the same actor that made it def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do with actor_id <- Utils.get_ap_id(actor), - %User{} = user <- user_from_key_id(conn), - true <- user.ap_id == actor_id do + {:user, %User{} = user} <- {:user, user_from_key_id(conn)}, + {:user_match, true} <- {:user_match, user.ap_id == actor_id} do assign(conn, :mapped_identity, user) else - _ -> - Logger.debug("Failed to map identity from signature (payload actor mismatch?)") + {:user_match, false} -> + Logger.debug("Failed to map identity from signature (payload actor mismatch)") + Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + assign(conn, :valid_signature, false) + + # remove me once testsuite uses mapped capabilities instead of what we do now + {:user, nil} -> + Logger.debug("Failed to map identity from signature (lookup failure)") Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") conn end @@ -55,7 +61,7 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") Logger.debug("key_id=#{key_id_from_conn(conn)}") - conn + assign(conn, :valid_signature, false) end end From a8af0ac053713102204418fe7a28d322f81eb3ea Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:27:50 +0000 Subject: [PATCH 37/48] mapped signature plug: fix user lookup --- lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index 2a8ed4470..1e7da4f50 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -23,7 +23,7 @@ defp key_id_from_conn(conn) do defp user_from_key_id(conn) do with key_actor_id when is_binary(key_actor_id) <- key_id_from_conn(conn), - %User{} = user <- User.get_or_fetch_by_ap_id(key_actor_id) do + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(key_actor_id) do user else _ -> From 621cacf667e7d5519a2731fadf86c429f70f4489 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 16:28:36 +0000 Subject: [PATCH 38/48] tests: add tests for mapped signature plug --- ...mapped_identity_to_signature_plug_test.exs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/plugs/mapped_identity_to_signature_plug_test.exs diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs new file mode 100644 index 000000000..9aca534e1 --- /dev/null +++ b/test/plugs/mapped_identity_to_signature_plug_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do + use Pleroma.Web.ConnCase + alias Pleroma.Web.Plugs.MappedSignatureToIdentityPlug + + import Tesla.Mock + import Plug.Conn + + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + defp set_signature(conn, key_id) do + conn + |> put_req_header("signature", "keyId=\"#{key_id}\"") + |> assign(:valid_signature, true) + end + + test "it successfully maps a valid identity with a valid signature" do + conn = + build_conn(:get, "/doesntmattter") + |> set_signature("http://mastodon.example.org/users/admin") + |> MappedSignatureToIdentityPlug.call(%{}) + + refute is_nil(conn.assigns.mapped_identity) + end + + test "it successfully maps a valid identity with a valid signature with payload" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("http://mastodon.example.org/users/admin") + |> MappedSignatureToIdentityPlug.call(%{}) + + refute is_nil(conn.assigns.mapped_identity) + end + + test "it considers a mapped identity to be invalid when it mismatches a payload" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("https://niu.moe/users/rye") + |> MappedSignatureToIdentityPlug.call(%{}) + + assert %{valid_signature: false} == conn.assigns + end + + @tag skip: "known breakage; the testsuite presently depends on it" + test "it considers a mapped identity to be invalid when the identity cannot be found" do + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("http://niu.moe/users/rye") + |> MappedSignatureToIdentityPlug.call(%{}) + + assert %{valid_signature: false} == conn.assigns + end +end From f435217e5003dac5d749e4bb905572d51c383b29 Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 18 Jul 2019 20:29:51 +0000 Subject: [PATCH 39/48] tests for Plugs.AuthenticationPlug --- lib/pleroma/plugs/authentication_plug.ex | 23 ++++++++++------------ test/plugs/authentication_plug_test.exs | 25 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex index eec514892..567674a0b 100644 --- a/lib/pleroma/plugs/authentication_plug.ex +++ b/lib/pleroma/plugs/authentication_plug.ex @@ -8,22 +8,19 @@ defmodule Pleroma.Plugs.AuthenticationPlug do alias Pleroma.User require Logger - def init(options) do - options + def init(options), do: options + + def checkpw(password, "$6" <> _ = password_hash) do + :crypt.crypt(password, password_hash) == password_hash end - def checkpw(password, password_hash) do - cond do - String.starts_with?(password_hash, "$pbkdf2") -> - Pbkdf2.checkpw(password, password_hash) + def checkpw(password, "$pbkdf2" <> _ = password_hash) do + Pbkdf2.checkpw(password, password_hash) + end - String.starts_with?(password_hash, "$6") -> - :crypt.crypt(password, password_hash) == password_hash - - true -> - Logger.error("Password hash not recognized") - false - end + def checkpw(_password, _password_hash) do + Logger.error("Password hash not recognized") + false end def call(%{assigns: %{user: %User{}}} = conn, _), do: conn diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 6158086ea..b55e746f8 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -54,4 +54,29 @@ test "with a wrong password in the credentials, it does nothing", %{conn: conn} assert conn == ret_conn end + + describe "checkpw/2" do + test "check pbkdf2 hash" do + hash = + "$pbkdf2-sha512$160000$loXqbp8GYls43F0i6lEfIw$AY.Ep.2pGe57j2hAPY635sI/6w7l9Q9u9Bp02PkPmF3OrClDtJAI8bCiivPr53OKMF7ph6iHhN68Rom5nEfC2A" + + assert AuthenticationPlug.checkpw("test-password", hash) + refute AuthenticationPlug.checkpw("test-password1", hash) + end + + test "check sha512-crypt hash" do + hash = + "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" + + assert AuthenticationPlug.checkpw("password", hash) + refute AuthenticationPlug.checkpw("password1", hash) + end + + test "it returns false when hash invalid" do + hash = + "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" + + refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash) + end + end end From c947cfec5ab49f90fb2de83f61bda77568298d6f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 18 Jul 2019 20:31:25 +0000 Subject: [PATCH 40/48] mapped signature plug: use `user` assign like authentication plug --- lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 6 +++--- test/plugs/mapped_identity_to_signature_plug_test.exs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index 1e7da4f50..ce8494b9d 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -31,14 +31,14 @@ defp user_from_key_id(conn) do end end - def call(%{assigns: %{mapped_identity: _}} = conn, _opts), do: conn + def call(%{assigns: %{user: _}} = conn, _opts), do: conn # if this has payload make sure it is signed by the same actor that made it def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do with actor_id <- Utils.get_ap_id(actor), {:user, %User{} = user} <- {:user, user_from_key_id(conn)}, {:user_match, true} <- {:user_match, user.ap_id == actor_id} do - assign(conn, :mapped_identity, user) + assign(conn, :user, user) else {:user_match, false} -> Logger.debug("Failed to map identity from signature (payload actor mismatch)") @@ -56,7 +56,7 @@ def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = con # no payload, probably a signed fetch def call(%{assigns: %{valid_signature: true}} = conn, _opts) do with %User{} = user <- user_from_key_id(conn) do - assign(conn, :mapped_identity, user) + assign(conn, :user, user) else _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs index 9aca534e1..bb45d9edf 100644 --- a/test/plugs/mapped_identity_to_signature_plug_test.exs +++ b/test/plugs/mapped_identity_to_signature_plug_test.exs @@ -26,7 +26,7 @@ test "it successfully maps a valid identity with a valid signature" do |> set_signature("http://mastodon.example.org/users/admin") |> MappedSignatureToIdentityPlug.call(%{}) - refute is_nil(conn.assigns.mapped_identity) + refute is_nil(conn.assigns.user) end test "it successfully maps a valid identity with a valid signature with payload" do @@ -35,7 +35,7 @@ test "it successfully maps a valid identity with a valid signature with payload" |> set_signature("http://mastodon.example.org/users/admin") |> MappedSignatureToIdentityPlug.call(%{}) - refute is_nil(conn.assigns.mapped_identity) + refute is_nil(conn.assigns.user) end test "it considers a mapped identity to be invalid when it mismatches a payload" do From de9906ad56bd25d6c8c38bef1307192df2e95445 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 19 Jul 2019 11:43:42 +0545 Subject: [PATCH 41/48] change the structure of image ttl parsar --- ..._set_richmedia_cache_ttl_based_on_image.md | 2 +- lib/pleroma/web/rich_media/parser.ex | 36 +++++----- .../rich_media/parsers/ttl/aws_signed_url.ex | 10 ++- lib/pleroma/web/rich_media/parsers/ttl/ttl.ex | 3 + test/web/rich_media/aws_signed_url_test.exs | 70 +++++++++++++++---- 5 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 lib/pleroma/web/rich_media/parsers/ttl/ttl.ex diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md index 489f9ece8..5846b6ab0 100644 --- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -4,7 +4,7 @@ Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. In such cases the old image url (expired) is returned from the media cache. -So to avoid such situation we can define a moddule that will set ttl based no image. +So to avoid such situation we can define a moddule that will set ttl based on image. The module must have a `run` function and it should be registered in the config. diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index ba8dc6f2a..b69b2be61 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -35,17 +35,17 @@ def parse(url) do @doc """ Set the rich media cache based on the expiration time of image. - Define a module that has `run` function + Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL` ## Example defmodule MyModule do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, url) do image_url = Map.get(data, :image) # do some parsing in the url and get the ttl of the image - # ttl is unix time - ttl = parse_ttl_from_url(image_url) - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + # and return ttl is unix time + parse_ttl_from_url(image_url) end end @@ -55,22 +55,26 @@ def run(data, url) do ttl_setters: [MyModule] """ def set_ttl_based_on_image({:ok, data}, url) do - case Cachex.ttl(:rich_media_cache, url) do - {:ok, nil} -> - modules = Pleroma.Config.get([:rich_media, :ttl_setters]) - - if Enum.count(modules) > 0 do - Enum.each(modules, & &1.run(data, url)) - end - - {:ok, data} - + with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url) do + ttl = get_ttl_from_image(data, url) + Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + {:ok, data} + else _ -> {:ok, data} end end - def set_ttl_based_on_image(data, _url), do: data + defp get_ttl_from_image(data, url) do + Pleroma.Config.get([:rich_media, :ttl_setters]) + |> Enum.reduce({:ok, nil}, fn + module, {:ok, _ttl} -> + module.ttl(data, url) + + _, error -> + error + end) + end defp parse_url(url) do try do diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex index d57107939..014c0935f 100644 --- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -1,5 +1,8 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + + @impl Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, _url) do image = Map.get(data, :image) if is_aws_signed_url(image) do @@ -7,7 +10,6 @@ def run(data, url) do |> parse_query_params() |> format_query_params() |> get_expiration_timestamp() - |> set_ttl(url) end end @@ -47,8 +49,4 @@ defp get_expiration_timestamp(params) when is_map(params) do Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) end - - defp set_ttl(ttl, url) do - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) - end end diff --git a/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex new file mode 100644 index 000000000..6b3ec6d30 --- /dev/null +++ b/lib/pleroma/web/rich_media/parsers/ttl/ttl.ex @@ -0,0 +1,3 @@ +defmodule Pleroma.Web.RichMedia.Parser.TTL do + @callback ttl(Map.t(), String.t()) :: {:ok, Integer.t()} | {:error, String.t()} +end diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs index 75bf6c6df..122787bc2 100644 --- a/test/web/rich_media/aws_signed_url_test.exs +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do use ExUnit.Case, async: true - test "amazon signed url is parsed and correct ttl is set for rich media" do + test "s3 signed url is parsed correct for expiration time" do url = "https://pleroma.social/amz" {:ok, timestamp} = @@ -16,22 +16,66 @@ test "amazon signed url is parsed and correct ttl is set for rich media" do # in seconds valid_till = 30 - data = %{ - image: - "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ - timestamp - }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host", - locale: "en_US", - site_name: "Pleroma", - title: "PLeroma", - url: url - } + metadata = construct_metadata(timestamp, valid_till, url) + + expire_time = + Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till) + + assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url) + end + + test "s3 signed url is parsed and correct ttl is set for rich media" do + url = "https://pleroma.social/amz" + + {:ok, timestamp} = + Timex.now() + |> DateTime.truncate(:second) + |> Timex.format("{ISO:Basic:Z}") + + # in seconds + valid_till = 30 + + metadata = construct_metadata(timestamp, valid_till, url) + + body = """ + + + + + + """ + + Tesla.Mock.mock(fn + %{ + method: :get, + url: "https://pleroma.social/amz" + } -> + %Tesla.Env{status: 200, body: body} + end) + + Cachex.put(:rich_media_cache, url, metadata) + + Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url) - Cachex.put(:rich_media_cache, url, data) - assert {:ok, _} = Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.run(data, url) {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) # as there is delay in setting and pulling the data from cache we ignore 1 second assert_in_delta(valid_till * 1000, cache_ttl, 1000) end + + defp construct_s3_url(timestamp, valid_till) do + "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{ + timestamp + }&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" + end + + defp construct_metadata(timestamp, valid_till, url) do + %{ + image: construct_s3_url(timestamp, valid_till), + site: "Pleroma", + title: "Pleroma", + description: "Pleroma", + url: url + } + end end From 581756ccc50cc08823957a2f24f506bf23c7cd22 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 19 Jul 2019 11:50:47 +0545 Subject: [PATCH 42/48] update the docs --- ...owto_set_richmedia_cache_ttl_based_on_image.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md index 5846b6ab0..bfee5a9e6 100644 --- a/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md +++ b/docs/config/howto_set_richmedia_cache_ttl_based_on_image.md @@ -4,20 +4,21 @@ Richmedia are cached without the ttl but the rich media may have image which can expire, like aws signed url. In such cases the old image url (expired) is returned from the media cache. -So to avoid such situation we can define a moddule that will set ttl based on image. - -The module must have a `run` function and it should be registered in the config. +So to avoid such situation we can define a module that will set ttl based on image. +The module must adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL` ### Example ```exs defmodule MyModule do - def run(data, url) do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + + @impl Pleroma.Web.RichMedia.Parser.TTL + def ttl(data, url) do image_url = Map.get(data, :image) # do some parsing in the url and get the ttl of the image - # ttl is unix time - ttl = parse_ttl_from_url(image_url) - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) + # return ttl is unix time + parse_ttl_from_url(image_url) end end ``` From b0b9eca37d43c8cc5e9e0e01fa4b1325a71ce4d2 Mon Sep 17 00:00:00 2001 From: tom79 Date: Fri, 19 Jul 2019 08:39:22 +0000 Subject: [PATCH 43/48] Update clients.md for Fedilab - Change owner (@fedilab@framapiaf.org), Source code: Framagit, Add some other supported features --- docs/clients.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/clients.md b/docs/clients.md index 91096970e..dc4ab37b1 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -32,10 +32,10 @@ Feel free to contact us to be added to this list! ### Fedilab - Homepage: -- Source Code: -- Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) +- Source Code: +- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab) - Platforms: Android -- Features: Streaming Ready +- Features: Streaming Ready, Moderation, Text Formatting ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) From 5c4a555b1da1d2cb6649a9384e386a48bbcf905f Mon Sep 17 00:00:00 2001 From: tom79 Date: Fri, 19 Jul 2019 08:40:47 +0000 Subject: [PATCH 44/48] Fix domain for the contact clients.md --- docs/clients.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clients.md b/docs/clients.md index dc4ab37b1..9029361f8 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -33,7 +33,7 @@ Feel free to contact us to be added to this list! ### Fedilab - Homepage: - Source Code: -- Contact: [@fedilab@mastodon.social](https://framapiaf.org/users/fedilab) +- Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab) - Platforms: Android - Features: Streaming Ready, Moderation, Text Formatting From c2e2aadc4254fe931ea519a9813854ccdac456b8 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 19 Jul 2019 16:20:23 +0000 Subject: [PATCH 45/48] #1110 fixed /api/pleroma/healthcheck --- .../controllers/util_controller.ex | 32 ++++++---- test/web/twitter_api/util_controller_test.exs | 64 ++++++++++++++++++- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index c10c66ff2..9e4da7dca 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -8,7 +8,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do require Logger alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Emoji + alias Pleroma.Healthcheck alias Pleroma.Notification alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.User @@ -23,7 +25,8 @@ def help_test(conn, _params) do end def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do - with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do + with %User{} = user <- User.get_cached_by_nickname(nick), + avatar = User.avatar_url(user) do conn |> render("subscribe.html", %{nickname: nick, avatar: avatar, error: false}) else @@ -338,20 +341,21 @@ def captcha(conn, _params) do end def healthcheck(conn, _params) do - info = - if Pleroma.Config.get([:instance, :healthcheck]) do - Pleroma.Healthcheck.system_info() - else - %{} - end + with true <- Config.get([:instance, :healthcheck]), + %{healthy: true} = info <- Healthcheck.system_info() do + json(conn, info) + else + %{healthy: false} = info -> + service_unavailable(conn, info) - conn = - if info[:healthy] do - conn - else - Plug.Conn.put_status(conn, :service_unavailable) - end + _ -> + service_unavailable(conn, %{}) + end + end - json(conn, info) + defp service_unavailable(conn, info) do + conn + |> put_status(:service_unavailable) + |> json(info) end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 21324399f..3d699e1df 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory + import Mock setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -231,10 +232,67 @@ test "show follow account page if the `acct` is a account link", %{conn: conn} d end end - test "GET /api/pleroma/healthcheck", %{conn: conn} do - conn = get(conn, "/api/pleroma/healthcheck") + describe "GET /api/pleroma/healthcheck" do + setup do + config_healthcheck = Pleroma.Config.get([:instance, :healthcheck]) - assert conn.status in [200, 503] + on_exit(fn -> + Pleroma.Config.put([:instance, :healthcheck], config_healthcheck) + end) + + :ok + end + + test "returns 503 when healthcheck disabled", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], false) + + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert response == %{} + end + + test "returns 200 when healthcheck enabled and all ok", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: true} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(200) + + assert %{ + "active" => _, + "healthy" => true, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end + + test "returns 503 when healthcheck enabled and health is false", %{conn: conn} do + Pleroma.Config.put([:instance, :healthcheck], true) + + with_mock Pleroma.Healthcheck, + system_info: fn -> %Pleroma.Healthcheck{healthy: false} end do + response = + conn + |> get("/api/pleroma/healthcheck") + |> json_response(503) + + assert %{ + "active" => _, + "healthy" => false, + "idle" => _, + "memory_used" => _, + "pool_size" => _ + } = response + end + end end describe "POST /api/pleroma/disable_account" do From 9a8eb2c94d2243fadd69786eb74d94cc6116468f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 19 Jul 2019 19:11:04 +0000 Subject: [PATCH 46/48] mix: add pleroma.user unsubscribe_all_from_instance --- lib/mix/tasks/pleroma/user.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 8a78b4fe6..c9b84b8f9 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -62,6 +62,10 @@ defmodule Mix.Tasks.Pleroma.User do mix pleroma.user unsubscribe NICKNAME + ## Unsubscribe local users from an entire instance and deactivate all accounts + + mix pleroma.user unsubscribe_all_from_instance INSTANCE + ## Create a password reset link. mix pleroma.user reset_password NICKNAME @@ -246,6 +250,20 @@ def run(["unsubscribe", nickname]) do end end + def run(["unsubscribe_all_from_instance", instance]) do + start_pleroma() + + Pleroma.User.Query.build(%{nickname: "@#{instance}"}) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> + run(["unsubscribe", user.nickname]) + end) + end) + |> Stream.run() + end + def run(["set", nickname | rest]) do start_pleroma() From d0198fe215a7542ce506e40e1e4860a27ee2d01e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 20 Jul 2019 13:03:34 +0300 Subject: [PATCH 47/48] [#1112] Preserving `id` on user insert conflict on order not to violate conversation_partipations_user_id_fkey constraint. --- CHANGELOG.md | 1 + lib/pleroma/user.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e79b5420..f60f3ed97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set +- Existing user id not being preserved on insert conflict ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c91fbb68a..5ea2b518b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1211,7 +1211,7 @@ def insert_or_update_user(data) do data |> Map.put(:name, blank?(data[:name]) || data[:nickname]) |> remote_user_creation() - |> Repo.insert(on_conflict: :replace_all, conflict_target: :nickname) + |> Repo.insert(on_conflict: :replace_all_except_primary_key, conflict_target: :nickname) |> set_cache() end From 43a7cd27feb6ef52e4b130a974af3f819a9b266d Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 20 Jul 2019 13:07:51 +0000 Subject: [PATCH 48/48] [tests] Mock :crypt.crypt/2 function in AuthenticationPlugTest --- test/plugs/authentication_plug_test.exs | 12 ++++-- test/signature_test.exs | 13 +++++-- test/upload_test.exs | 16 ++++++-- .../mastodon_api/search_controller_test.exs | 30 ++++++++------ test/web/ostatus/ostatus_controller_test.exs | 39 ++++++++++++------- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index b55e746f8..7ca045616 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -8,6 +8,9 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.User + import ExUnit.CaptureLog + import Mock + setup %{conn: conn} do user = %User{ id: 1, @@ -68,15 +71,18 @@ test "check sha512-crypt hash" do hash = "$6$9psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - assert AuthenticationPlug.checkpw("password", hash) - refute AuthenticationPlug.checkpw("password1", hash) + with_mock :crypt, crypt: fn _password, password_hash -> password_hash end do + assert AuthenticationPlug.checkpw("password", hash) + end end test "it returns false when hash invalid" do hash = "psBWV8gxkGOZWBz$PmfCycChoxeJ3GgGzwvhlgacb9mUoZ.KUXNCssekER4SJ7bOK53uXrHNb2e4i8yPFgSKyzaW9CcmrDXWIEMtD1" - refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash) + assert capture_log(fn -> + refute Pleroma.Plugs.AuthenticationPlug.checkpw("password", hash) + end) =~ "[error] Password hash not recognized" end end end diff --git a/test/signature_test.exs b/test/signature_test.exs index 840987cd6..7400cae9a 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.SignatureTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock @@ -46,8 +47,10 @@ test "it returns key" do end test "it returns error when not found user" do - assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == - {:error, :error} + assert capture_log(fn -> + assert Signature.fetch_public_key(make_fake_conn("test-ap_id")) == + {:error, :error} + end) =~ "[error] Could not decode user" end test "it returns error if public key is empty" do @@ -67,8 +70,10 @@ test "it returns key" do end test "it returns error when not found user" do - assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == - {:error, {:error, :ok}} + assert capture_log(fn -> + assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == + {:error, {:error, :ok}} + end) =~ "[error] Could not decode user" end end diff --git a/test/upload_test.exs b/test/upload_test.exs index f7b1893ad..32c6977d1 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.UploadTest do use Pleroma.DataCase + import ExUnit.CaptureLog + alias Pleroma.Upload alias Pleroma.Uploaders.Uploader @@ -77,8 +79,12 @@ def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) test "it returns error" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == {:error, "Errors"} - Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end)) + + assert capture_log(fn -> + assert Upload.store(@upload_file) == {:error, "Errors"} + Task.await(Agent.get(TestUploaderError, fn task_pid -> task_pid end)) + end) =~ + "[error] Elixir.Pleroma.Upload store (using Pleroma.UploadTest.TestUploaderError) failed: \"Errors\"" end end @@ -89,7 +95,11 @@ test "it returns error" do test "it returns error" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"} + + assert capture_log(fn -> + assert Upload.store(@upload_file) == {:error, "Uploader callback timeout"} + end) =~ + "[error] Elixir.Pleroma.Upload store (using Pleroma.UploadTest.TestUploader) failed: \"Uploader callback timeout\"" end end diff --git a/test/web/mastodon_api/search_controller_test.exs b/test/web/mastodon_api/search_controller_test.exs index 9f50c09f4..043b96c14 100644 --- a/test/web/mastodon_api/search_controller_test.exs +++ b/test/web/mastodon_api/search_controller_test.exs @@ -24,12 +24,16 @@ test "it returns empty result if user or status search return undefined error", {Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]}, {Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]} ] do - conn = get(conn, "/api/v2/search", %{"q" => "2hu"}) + capture_log(fn -> + results = + conn + |> get("/api/v2/search", %{"q" => "2hu"}) + |> json_response(200) - assert results = json_response(conn, 200) - - assert results["accounts"] == [] - assert results["statuses"] == [] + assert results["accounts"] == [] + assert results["statuses"] == [] + end) =~ + "[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}" end end @@ -99,14 +103,16 @@ test "it returns empty result if user or status search return undefined error", {Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]}, {Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]} ] do - conn = - conn - |> get("/api/v1/search", %{"q" => "2hu"}) + capture_log(fn -> + results = + conn + |> get("/api/v1/search", %{"q" => "2hu"}) + |> json_response(200) - assert results = json_response(conn, 200) - - assert results["accounts"] == [] - assert results["statuses"] == [] + assert results["accounts"] == [] + assert results["statuses"] == [] + end) =~ + "[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}" end end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 3dd8c6491..bb7648bdd 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -4,7 +4,10 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase + + import ExUnit.CaptureLog import Pleroma.Factory + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -27,24 +30,28 @@ test "decodes a salmon", %{conn: conn} do user = insert(:user) salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" end test "decodes a salmon with a changed magic key", %{conn: conn} do user = insert(:user) salmon = File.read!("test/fixtures/salmon.xml") - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + conn + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" # Set a wrong magic-key for a user so it has to refetch salmon_user = User.get_cached_by_ap_id("http://gs.example.org:4040/index.php/user/1") @@ -61,12 +68,14 @@ test "decodes a salmon with a changed magic key", %{conn: conn} do |> Ecto.Changeset.put_embed(:info, info_cng) |> User.update_and_set_cache() - conn = - build_conn() - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) + assert capture_log(fn -> + conn = + build_conn() + |> put_req_header("content-type", "application/atom+xml") + |> post("/users/#{user.nickname}/salmon", salmon) - assert response(conn, 200) + assert response(conn, 200) + end) =~ "[error]" end end