From 62e42b03abd2cede85e85f62c35f62a8c42e8ea1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 May 2019 20:10:16 +0300 Subject: [PATCH 01/62] Handle incoming Question objects --- .../web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/fixtures/httpoison_mock/rinpatch.json | 64 ++++++++++++ test/fixtures/mastodon-question-activity.json | 99 +++++++++++++++++++ test/support/http_request_mock.ex | 8 ++ test/web/activity_pub/transmogrifier_test.exs | 17 ++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/httpoison_mock/rinpatch.json create mode 100644 test/fixtures/mastodon-question-activity.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 508f3532f..c2596cfec 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -399,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page"] do + when objtype in ["Article", "Note", "Video", "Page", "Question"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 581b9d1ab..de91fa03f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question"] # Some implementations send the actor URI as the actor field, others send the entire actor object, # so figure out what the actor's URI is based on what we have. diff --git a/test/fixtures/httpoison_mock/rinpatch.json b/test/fixtures/httpoison_mock/rinpatch.json new file mode 100644 index 000000000..59311ecb6 --- /dev/null +++ b/test/fixtures/httpoison_mock/rinpatch.json @@ -0,0 +1,64 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "toot": "http://joinmastodon.org/ns#", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + }, + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "Hashtag": "as:Hashtag", + "Emoji": "toot:Emoji", + "IdentityProof": "toot:IdentityProof", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch", + "type": "Person", + "following": "https://mastodon.sdf.org/users/rinpatch/following", + "followers": "https://mastodon.sdf.org/users/rinpatch/followers", + "inbox": "https://mastodon.sdf.org/users/rinpatch/inbox", + "outbox": "https://mastodon.sdf.org/users/rinpatch/outbox", + "featured": "https://mastodon.sdf.org/users/rinpatch/collections/featured", + "preferredUsername": "rinpatch", + "name": "rinpatch", + "summary": "

umu

", + "url": "https://mastodon.sdf.org/@rinpatch", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "https://mastodon.sdf.org/users/rinpatch#main-key", + "owner": "https://mastodon.sdf.org/users/rinpatch", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1vbhYKDopb5xzfJB2TZY\n0ZvgxqdAhbSKKkQC5Q2b0ofhvueDy2AuZTnVk1/BbHNlqKlwhJUSpA6LiTZVvtcc\nMn6cmSaJJEg30gRF5GARP8FMcuq8e2jmceiW99NnUX17MQXsddSf2JFUwD0rUE8H\nBsgD7UzE9+zlA/PJOTBO7fvBEz9PTQ3r4sRMTJVFvKz2MU/U+aRNTuexRKMMPnUw\nfp6VWh1F44VWJEQOs4tOEjGiQiMQh5OfBk1w2haT3vrDbQvq23tNpUP1cRomLUtx\nEBcGKi5DMMBzE1RTVT1YUykR/zLWlA+JSmw7P6cWtsHYZovs8dgn8Po3X//6N+ng\nTQIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "tag": [], + "attachment": [], + "endpoints": { + "sharedInbox": "https://mastodon.sdf.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://mastodon.sdf.org/system/accounts/avatars/000/067/580/original/bf05521bf711b7a0.jpg?1533238802" + }, + "image": { + "type": "Image", + "mediaType": "image/gif", + "url": "https://mastodon.sdf.org/system/accounts/headers/000/067/580/original/a99b987e798f7063.gif?1533278217" + } +} diff --git a/test/fixtures/mastodon-question-activity.json b/test/fixtures/mastodon-question-activity.json new file mode 100644 index 000000000..ac329c7d5 --- /dev/null +++ b/test/fixtures/mastodon-question-activity.json @@ -0,0 +1,99 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/activity", + "type": "Create", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "published": "2019-05-10T09:03:36Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "object": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "type": "Question", + "summary": null, + "inReplyTo": null, + "published": "2019-05-10T09:03:36Z", + "url": "https://mastodon.sdf.org/@rinpatch/102070944809637304", + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "sensitive": false, + "atomUri": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "inReplyToAtomUri": null, + "conversation": "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation", + "content": "

Why is Tenshi eating a corndog so cute?

", + "contentMap": { + "en": "

Why is Tenshi eating a corndog so cute?

" + }, + "endTime": "2019-05-11T09:03:36Z", + "closed": "2019-05-11T09:03:36Z", + "attachment": [], + "tag": [], + "replies": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "partOf": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "items": [] + } + }, + "oneOf": [ + { + "type": "Note", + "name": "Dunno", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "Everyone knows that!", + "replies": { + "type": "Collection", + "totalItems": 1 + } + }, + { + "type": "Note", + "name": "25 char limit is dumb", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "I can't even fit a funny", + "replies": { + "type": "Collection", + "totalItems": 1 + } + } + ] + } +} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 5b355bfe6..3064c032b 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -52,6 +52,14 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/httpoison_mock/rinpatch.json") + }} + end + def get( "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie", _, diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index c24b50f8c..727abbd17 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -113,6 +113,23 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert Enum.at(object.data["tag"], 2) == "moo" end + test "it works for incoming questions" do + data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity) + + assert Enum.all?(object.data["oneOf"], fn choice -> + choice["name"] in [ + "Dunno", + "Everyone knows that!", + "25 char limit is dumb", + "I can't even fit a funny" + ] + end) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From 642a67dd4492f31b5b9fe457e34c1589c9d70c3f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 17 May 2019 11:44:47 +0300 Subject: [PATCH 02/62] Render polls in statuses --- .../web/mastodon_api/views/status_view.ex | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bd2372944..6337c50e8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -232,6 +232,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do spoiler_text: summary_html, visibility: get_visibility(object), media_attachments: attachments, + poll: render("poll.json", %{object: object, for: opts[:for]}), mentions: mentions, tags: build_tags(tags), application: %{ @@ -321,6 +322,57 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end + # TODO: Add tests for this view + def render("poll.json", %{object: object} = opts) do + {multiple, options} = + case object.data do + %{"anyOf" => options} when is_list(options) -> {true, options} + %{"oneOf" => options} when is_list(options) -> {false, options} + _ -> {nil, nil} + end + + if options do + end_time = + (object.data["closed"] || object.data["endTime"]) + |> NaiveDateTime.from_iso8601!() + + votes_count = object.data["votes_count"] || 0 + + expired = + end_time + |> NaiveDateTime.compare(NaiveDateTime.utc_now()) + |> case do + :lt -> true + _ -> false + end + + options = + Enum.map(options, fn %{"name" => name} = option -> + name = + HTML.filter_tags( + name, + User.html_filter_policy(opts[:for]) + ) + + %{title: name, votes_count: option["replies"]["votes_count"] || 0} + end) + + %{ + # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + id: object.id, + expires_at: Utils.to_masto_date(end_time), + expired: expired, + multiple: multiple, + votes_count: votes_count, + options: options, + voted: false, + emojis: build_emojis(object.data["emoji"]) + } + else + nil + end + end + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity) From fd920c897339b9cedea042dd6698d14380cedae7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 18 May 2019 13:29:28 +0300 Subject: [PATCH 03/62] Mastodon API: Add support for posting polls --- lib/pleroma/web/common_api/common_api.ex | 4 +- lib/pleroma/web/common_api/utils.ex | 64 ++++++++++++++++--- .../mastodon_api_controller_test.exs | 22 +++++++ test/web/mastodon_api/status_view_test.exs | 1 + 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index b53869c75..335ae70b0 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -149,6 +149,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), + {poll, mentions, tags} <- make_poll_data(data, mentions, tags), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -164,7 +165,8 @@ defmodule Pleroma.Web.CommonAPI do in_reply_to, tags, cw, - cc + cc, + poll ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1dfe50b40..13cdffbbd 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,6 +102,48 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end + def make_poll_data( + %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, + mentions, + tags + ) + when is_list(options) and is_integer(expires_in) do + content_type = get_content_type(data["content_type"]) + # XXX: There is probably a more performant/cleaner way to do this + {poll, {mentions, tags}} = + Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> + # TODO: Custom emoji + {option, mentions_merge, tags_merge} = format_input(option, content_type) + mentions = mentions ++ mentions_merge + tags = tags ++ tags_merge + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, {mentions, tags}} + end) + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, mentions, tags} + end + + def make_poll_data(data, mentions, tags) do + IO.inspect(data, label: "data") + {%{}, mentions, tags} + end + def make_content_html( status, attachments, @@ -223,8 +265,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + merge \\ %{} ) do + IO.inspect(merge, label: "merge") + object = %{ "type" => "Note", "to" => to, @@ -237,14 +282,17 @@ defmodule Pleroma.Web.CommonAPI.Utils do "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if in_reply_to do - in_reply_to_object = Object.normalize(in_reply_to) + object = + if in_reply_to do + in_reply_to_object = Object.normalize(in_reply_to) - object - |> Map.put("inReplyTo", in_reply_to_object.data["id"]) - else - object - end + object + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) + else + object + end + + Map.merge(object, merge) end def format_naive_asctime(date) do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 505e45010..ce581c092 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -132,6 +132,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the best girl?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) + + response = json_response(conn, 200) + + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) + + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + test "posting a sensitive status", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index d7c800e83..9f2ebda4e 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -103,6 +103,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do muted: false, pinned: false, sensitive: false, + poll: nil, spoiler_text: HtmlSanitizeEx.basic_html(note.data["object"]["summary"]), visibility: "public", media_attachments: [], From 1d90f9b96999f8bad3fa3e3ec58bf50c8666be4f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:06:44 +0300 Subject: [PATCH 04/62] Remove tags/mentions/rich text from poll options because Mastodon and add custom emoji --- lib/pleroma/web/common_api/common_api.ex | 4 +-- lib/pleroma/web/common_api/utils.ex | 25 ++++++------------- .../web/mastodon_api/views/status_view.ex | 14 +++++------ 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index bc8f80389..374967a1b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -154,7 +154,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), - {poll, mentions, tags} <- make_poll_data(data, mentions, tags), + {poll, poll_emoji} <- make_poll_data(data), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -179,7 +179,7 @@ defmodule Pleroma.Web.CommonAPI do Map.put( object, "emoji", - Formatter.get_emoji_map(full_payload) + Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) ) do res = ActivityPub.create( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 2ea997789..cd8483c11 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,26 +102,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data( - %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, - mentions, - tags - ) + def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - content_type = get_content_type(data["content_type"]) - # XXX: There is probably a more performant/cleaner way to do this - {poll, {mentions, tags}} = - Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> - # TODO: Custom emoji - {option, mentions_merge, tags_merge} = format_input(option, content_type) - mentions = mentions ++ mentions_merge - tags = tags ++ tags_merge - + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> {%{ "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, {mentions, tags}} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} end) end_time = @@ -136,11 +125,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do %{"type" => "Question", "oneOf" => poll, "closed" => end_time} end - {poll, mentions, tags} + {poll, emoji} end - def make_poll_data(_data, mentions, tags) do - {%{}, mentions, tags} + def make_poll_data(_data) do + {%{}, %{}} end def make_content_html( diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2a5691e1f..0df8bb5c2 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -325,7 +325,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end # TODO: Add tests for this view - def render("poll.json", %{object: object} = opts) do + def render("poll.json", %{object: object} = _opts) do {multiple, options} = case object.data do %{"anyOf" => options} when is_list(options) -> {true, options} @@ -350,13 +350,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do options = Enum.map(options, fn %{"name" => name} = option -> - name = - HTML.filter_tags( - name, - User.html_filter_policy(opts[:for]) - ) - - %{title: name, votes_count: option["replies"]["votes_count"] || 0} + %{ + title: HTML.strip_tags(name), + votes_count: option["replies"]["votes_count"] || 0 + } end) %{ @@ -367,6 +364,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do multiple: multiple, votes_count: votes_count, options: options, + # TODO: Actually check for a vote voted: false, emojis: build_emojis(object.data["emoji"]) } From 6430cb1bf78e7949cc023a30df7a8d1547c36524 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:44:18 +0300 Subject: [PATCH 05/62] Restrict poll replies from fetch queries by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..035fb75d5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -820,6 +820,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query + + defp restrict_poll_replies(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -873,6 +883,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() + |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do From 76a7429befb2e9a819b653ff8328cc42a565c29d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 09:13:10 +0300 Subject: [PATCH 06/62] Add poll limits to /api/v1/instance and initial state --- config/config.exs | 6 ++++++ docs/config.md | 5 +++++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 +++- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9a10b0ff7..47d8dfb42 100644 --- a/config/config.exs +++ b/config/config.exs @@ -211,6 +211,12 @@ config :pleroma, :instance, avatar_upload_limit: 2_000_000, background_upload_limit: 4_000_000, banner_upload_limit: 4_000_000, + poll_limits: %{ + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 + }, registrations_open: true, federating: true, federation_reachability_timeout_days: 7, diff --git a/docs/config.md b/docs/config.md index 450d73fda..f9903332c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -71,6 +71,11 @@ config :pleroma, Pleroma.Emails.Mailer, * `avatar_upload_limit`: File size limit of user’s profile avatars * `background_upload_limit`: File size limit of user’s profile backgrounds * `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) * `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1051861ff..81cc5a972 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -197,7 +197,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do languages: ["en"], registrations: Pleroma.Config.get([:instance, :registrations_open]), # Extra (not present in Mastodon): - max_toot_chars: Keyword.get(instance, :limit) + max_toot_chars: Keyword.get(instance, :limit), + poll_limits: Keyword.get(instance, :poll_limits) } json(conn, response) @@ -1331,6 +1332,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do max_toot_chars: limit, mascot: "/images/pleroma-fox-tan-smol.png" }, + poll_limits: Config.get([:instance, :poll_limits]), rights: %{ delete_others_notice: present?(user.info.is_moderator), admin: present?(user.info.is_admin) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 68fe9c1b4..48268d4f7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2494,7 +2494,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "stats" => _, "thumbnail" => _, "languages" => _, - "registrations" => _ + "registrations" => _, + "poll_limits" => _ } = result assert email == from_config_email From 3f96b3e4b8114ec1cf924d452907b17c2aea2003 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 10:54:20 +0300 Subject: [PATCH 07/62] Enforce poll limits and add error handling for MastodonAPI's post endpoint --- lib/pleroma/web/common_api/utils.ex | 71 ++++++++---- .../mastodon_api/mastodon_api_controller.ex | 43 ++++--- .../mastodon_api_controller_test.exs | 107 +++++++++++++++--- 3 files changed, 173 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index cd8483c11..97172fd94 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -104,28 +104,61 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - {poll, emoji} = - Enum.map_reduce(options, %{}, fn option, emoji -> - {%{ - "name" => option, - "type" => "Note", - "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} - end) + %{max_expiration: max_expiration, min_expiration: min_expiration} = + limits = Pleroma.Config.get([:instance, :poll_limits]) - end_time = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(expires_in) - |> NaiveDateTime.to_iso8601() - - poll = - if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do - %{"type" => "Question", "anyOf" => poll, "closed" => end_time} - else - %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + # XXX: There is probably a cleaner way of doing this + try do + if Enum.count(options) > limits.max_options do + raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end - {poll, emoji} + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> + if String.length(option) > limits.max_option_chars do + raise ArgumentError, + message: + "Poll options cannot be longer than #{limits.max_option_chars} characters each" + end + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} + end) + + case expires_in do + expires_in when expires_in > max_expiration -> + raise ArgumentError, message: "Expiration date is too far in the future" + + expires_in when expires_in < min_expiration -> + raise ArgumentError, message: "Expiration date is too soon" + + _ -> + :noop + end + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, emoji} + rescue + e in ArgumentError -> e.message + end + end + + def make_poll_data(%{"poll" => _}) do + "Invalid poll" end def make_poll_data(_data) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 81cc5a972..e2cab86f1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -473,12 +473,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) - idempotency_key = - case get_req_header(conn, "idempotency-key") do - [key] -> key - _ -> Ecto.UUID.generate() - end - scheduled_at = params["scheduled_at"] if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do @@ -491,17 +485,40 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do else params = Map.drop(params, ["scheduled_at"]) - {:ok, activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - CommonAPI.post(user, params) - end) + case get_cached_status_or_post(conn, params) do + {:ignore, message} -> + conn + |> put_status(401) + |> json(%{error: message}) - conn - |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + {:error, message} -> + conn + |> put_status(401) + |> json(%{error: message}) + + {_, activity} -> + conn + |> put_view(StatusView) + |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + end end end + defp get_cached_status_or_post(%{assigns: %{user: user}} = conn, params) do + idempotency_key = + case get_req_header(conn, "idempotency-key") do + [key] -> key + _ -> Ecto.UUID.generate() + end + + Cachex.fetch(:idempotency_cache, idempotency_key, fn _ -> + case CommonAPI.post(user, params) do + {:ok, activity} -> activity + {:error, message} -> {:ignore, message} + end + end) + end + def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do json(conn, %{}) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 48268d4f7..e1df79ffb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -146,26 +146,101 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end - test "posting a poll", %{conn: conn} do - user = insert(:user) - time = NaiveDateTime.utc_now() + describe "posting polls" do + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() - conn = - conn - |> assign(:user, user) - |> post("/api/v1/statuses", %{ - "status" => "Who is the best girl?", - "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} - }) + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the #bestgrill?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) - response = json_response(conn, 200) + response = json_response(conn, 200) - assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> - title in ["Rei", "Asuka", "Misato"] - end) + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) - assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 - refute response["poll"]["expred"] + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + + test "option limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_options]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "desu~", + "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll can't contain more than #{limit} options" + end + + test "option character limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_option_chars]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "...", + "poll" => %{ + "options" => [Enum.reduce(0..limit, "", fn _, acc -> acc <> "." end)], + "expires_in" => 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll options cannot be longer than #{limit} characters each" + end + + test "minimal date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :min_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit - 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too soon" + end + + test "maximum date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit + 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too far in the future" + end end test "posting a sensitive status", %{conn: conn} do From aafe30d94e68ccf251c56139d07bda154dde3af9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:12:10 +0300 Subject: [PATCH 08/62] Handle poll votes --- lib/pleroma/object.ex | 30 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++ test/fixtures/mastodon-vote.json | 16 ++++++++++ test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 test/fixtures/mastodon-vote.json diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..a0f7659eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -188,4 +188,34 @@ defmodule Pleroma.Object do _ -> {:error, "Not found"} end end + + def increase_vote_count(ap_id, name) do + with %Object{} = object <- Object.normalize(ap_id), + "Question" <- object.data["type"] do + multiple = Map.has_key?(object.data, "anyOf") + + options = + (object.data["anyOf"] || object.data["oneOf"] || []) + |> Enum.map(fn + %{"name" => ^name} = option -> + Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1)) + + option -> + option + end) + + data = + if multiple do + Map.put(object.data, "anyOf", options) + else + Map.put(object.data, "oneOf", options) + end + + object + |> Object.change(%{data: data}) + |> update_and_set_cache() + else + _ -> :noop + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 035fb75d5..ce78d895b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -108,6 +108,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def decrease_replies_count_if_reply(_object), do: :noop + def increase_poll_votes_if_vote(%{ + "object" => %{"inReplyTo" => reply_ap_id, "name" => name}, + "type" => "Create" + }) do + Object.increase_vote_count(reply_ap_id, name) + end + + def increase_poll_votes_if_vote(_create_data), do: :noop + def insert(map, local \\ true, fake \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), @@ -235,6 +244,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_poll_votes_if_vote(create_data), # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), diff --git a/test/fixtures/mastodon-vote.json b/test/fixtures/mastodon-vote.json new file mode 100644 index 000000000..c2c5f40c0 --- /dev/null +++ b/test/fixtures/mastodon-vote.json @@ -0,0 +1,16 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387/activity", + "nickname": "rin", + "object": { + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387", + "inReplyTo": "https://testing.uguu.ltd/objects/9d300947-2dcb-445d-8978-9a3b4b84fa14", + "name": "suya..", + "to": "https://testing.uguu.ltd/users/rin", + "type": "Note" + }, + "to": "https://testing.uguu.ltd/users/rin", + "type": "Create" +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 727abbd17..32d94e3e9 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,6 +130,37 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end + test "in increments vote counters on question activities" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + object = Object.normalize(activity) + + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + + object = Object.get_by_ap_id(object.data["id"]) + + assert Enum.any?( + object.data["oneOf"], + fn + %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true + _ -> false + end + ) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From a53d06273080525fdda332291838b0c95ed69690 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:19:03 +0300 Subject: [PATCH 09/62] Fix posting non-polls from mastofe --- lib/pleroma/web/common_api/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 97172fd94..66153a105 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data(%{"poll" => _}) do + def make_poll_data(%{"poll" => poll}) when is_map(poll) do "Invalid poll" end From f28747858bf3265e8b82eb587919f5a89386bed7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:27:09 +0300 Subject: [PATCH 10/62] Actual vote count in poll view --- .../web/mastodon_api/views/status_view.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0df8bb5c2..c501c213c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -338,8 +338,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do (object.data["closed"] || object.data["endTime"]) |> NaiveDateTime.from_iso8601!() - votes_count = object.data["votes_count"] || 0 - expired = end_time |> NaiveDateTime.compare(NaiveDateTime.utc_now()) @@ -348,12 +346,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> false end - options = - Enum.map(options, fn %{"name" => name} = option -> - %{ - title: HTML.strip_tags(name), - votes_count: option["replies"]["votes_count"] || 0 - } + {options, votes_count} = + Enum.map_reduce(options, 0, fn %{"name" => name} = option, count -> + current_count = option["replies"]["totalItems"] || 0 + + {%{ + title: HTML.strip_tags(name), + votes_count: current_count + }, current_count + count} end) %{ From d7c4d029c82c833278b3640d585235dc704f30d1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:35:20 +0300 Subject: [PATCH 11/62] Restrict poll replies when fetching activiites for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ce78d895b..c04e6c2b8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -491,6 +491,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) + |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], From ee682441415d7abe4acb2643e1d76fe8d78e80c1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 16:58:15 +0300 Subject: [PATCH 12/62] Do not stream out poll replies --- lib/pleroma/web/activity_pub/activity_pub.ex | 54 ++++++++++---------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c04e6c2b8..fdebf1f6b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -192,40 +192,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce", "Delete"] do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + object = Object.normalize(activity) + # Do not stream out poll replies + unless object.data["name"] do + Pleroma.Web.Streamer.stream("user", activity) + Pleroma.Web.Streamer.stream("list", activity) - if Enum.member?(activity.data["to"], public) do - Pleroma.Web.Streamer.stream("public", activity) + if Enum.member?(activity.data["to"], public) do + Pleroma.Web.Streamer.stream("public", activity) - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end - if activity.data["type"] in ["Create"] do - object = Object.normalize(activity) + if activity.data["type"] in ["Create"] do + object.data + |> Map.get("tag", []) + |> Enum.filter(fn tag -> is_bitstring(tag) end) + |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) + if object.data["attachment"] != [] do + Pleroma.Web.Streamer.stream("public:media", activity) - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) + if activity.local do + Pleroma.Web.Streamer.stream("public:local:media", activity) + end end end + else + # TODO: Write test, replace with visibility test + if !Enum.member?(activity.data["cc"] || [], public) && + !Enum.member?( + activity.data["to"], + User.get_cached_by_ap_id(activity.data["actor"]).follower_address + ), + do: Pleroma.Web.Streamer.stream("direct", activity) end - else - # TODO: Write test, replace with visibility test - if !Enum.member?(activity.data["cc"] || [], public) && - !Enum.member?( - activity.data["to"], - User.get_cached_by_ap_id(activity.data["actor"]).follower_address - ), - do: Pleroma.Web.Streamer.stream("direct", activity) end end end From 0407ffe75f7e91db240d491492eadf1385b1726b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:12:38 +0300 Subject: [PATCH 13/62] Change validation error status codes to be more appropriate --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e2cab86f1..aef2abf0b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -488,12 +488,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do case get_cached_status_or_post(conn, params) do {:ignore, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {:error, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {_, activity} -> diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e1df79ffb..4f332f83c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -181,7 +181,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll can't contain more than #{limit} options" end @@ -200,7 +200,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll options cannot be longer than #{limit} characters each" end @@ -219,7 +219,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too soon" end @@ -238,7 +238,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too far in the future" end end From 5f67c26baf2bdc98480fa4cda5895f33351b13ab Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:30:51 +0300 Subject: [PATCH 14/62] Accept strings in expires_in because sasuga javascript --- lib/pleroma/web/common_api/utils.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 66153a105..1a239de97 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -103,12 +103,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) - when is_list(options) and is_integer(expires_in) do + when is_list(options) do %{max_expiration: max_expiration, min_expiration: min_expiration} = limits = Pleroma.Config.get([:instance, :poll_limits]) # XXX: There is probably a cleaner way of doing this try do + # In some cases mastofe sends out strings instead of integers + expires_in = if is_binary(expires_in), do: String.to_integer(expires_in), else: expires_in + if Enum.count(options) > limits.max_options do raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end From ff61d345020b9ee79b243fb8d23a37a06cc41b8e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:33:54 +0300 Subject: [PATCH 15/62] Accept question objects for conversations --- lib/pleroma/conversation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 238c1acf2..bc97b39ca 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Conversation do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), "Create" <- activity.data["type"], object <- Pleroma.Object.normalize(activity), - "Note" <- object.data["type"], + true <- object.data["type"] in ["Note", "Question"], ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do {:ok, conversation} = create_for_ap_id(ap_id) From 63b0b7190cb652cd27d236e3f9daaaf5e50701a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 20:40:35 +0300 Subject: [PATCH 16/62] MastoAPI: Add GET /api/v1/polls/:id --- lib/pleroma/object.ex | 3 ++ .../mastodon_api/mastodon_api_controller.ex | 20 +++++++++ lib/pleroma/web/router.ex | 2 + .../mastodon_api_controller_test.exs | 44 +++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a0f7659eb..3ffa290eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -35,6 +35,9 @@ defmodule Pleroma.Object do |> unique_constraint(:ap_id, name: :objects_unique_apid_index) end + def get_by_id(nil), do: nil + def get_by_id(id), do: Repo.get(Object, id) + def get_by_ap_id(nil), do: nil def get_by_ap_id(ap_id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index aef2abf0b..ecb7df459 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -410,6 +410,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with %Object{} = object <- Object.get_by_id(id), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + conn + |> put_view(StatusView) + |> try_render("poll.json", %{object: object, for: user}) + else + nil -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + + false -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + end + end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do conn diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..e0611e3fc 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -423,6 +423,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) + get("/polls/:id", MastodonAPIController, :get_poll) + get("/accounts/:id/statuses", MastodonAPIController, :user_statuses) get("/accounts/:id/followers", MastodonAPIController, :followers) get("/accounts/:id/following", MastodonAPIController, :following) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 4f332f83c..0d56b6ff2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3453,4 +3453,48 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} end end + + describe "GET /api/v1/polls/:id" do + test "returns poll entity for object id", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20} + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/polls/#{object.id}") + + response = json_response(conn, 200) + id = object.id + assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + end + + test "does not expose polls for private statuses", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20}, + "visibility" => "private" + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, other_user) + |> get("/api/v1/polls/#{object.id}") + + assert json_response(conn, 404) + end + end end From 10ca1f91de12c3ff3a1adff2cf36e9ec47c659dd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 11:56:53 +0300 Subject: [PATCH 17/62] Add GIN index on object data->'name' --- .../migrations/20190522084924_add_object_index_on_name.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20190522084924_add_object_index_on_name.exs diff --git a/priv/repo/migrations/20190522084924_add_object_index_on_name.exs b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs new file mode 100644 index 000000000..886e8499e --- /dev/null +++ b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddObjectIndexOnName do + use Ecto.Migration + + def change do + create(index(:objects, ["(data->'name')"], name: :objects_name_index, using: :gin)) + end +end From 19c90d47c4f649d0962098050f6cbc65eeb889d0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:17:57 +0300 Subject: [PATCH 18/62] Normalize poll votes to Answer objects --- lib/pleroma/web/activity_pub/activity_pub.ex | 14 +-------- .../web/activity_pub/transmogrifier.ex | 22 ++++++++++++- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++-- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fdebf1f6b..5b3fb7e84 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -194,7 +194,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if activity.data["type"] in ["Create", "Announce", "Delete"] do object = Object.normalize(activity) # Do not stream out poll replies - unless object.data["name"] do + unless object.data["type"] == "Answer" do Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -493,7 +493,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) - |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], @@ -833,16 +832,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query - - defp restrict_poll_replies(query, _) do - if has_named_binding?(query, :object) do - from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) - else - query - end - end - defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -896,7 +885,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() - |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8b2427258..70b467b3f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_likes |> fix_addressing |> fix_summary + |> fix_type end def fix_summary(%{"summary" => nil} = object) do @@ -328,6 +329,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object + def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do + reply = Object.normalize(reply_id) + + if reply.data["type"] == "Question" and object["name"] do + Map.put(object, "type", "Answer") + else + object + end + end + + def fix_type(object), do: object + defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id), @@ -398,7 +411,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page", "Question"] do + when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) data = @@ -731,6 +744,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> set_reply_to_uri |> strip_internal_fields |> strip_internal_tags + |> set_type end # @doc @@ -895,6 +909,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "sensitive", "nsfw" in tags) end + def set_type(%{"type" => "Answer"} = object) do + Map.put(object, "type", "Note") + end + + def set_type(object), do: object + def add_attributed_to(object) do attributed_to = object["attributedTo"] || object["actor"] diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 63454e3f7..9646bbee9 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page", "Question"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer"] @supported_report_states ~w(open closed resolved) @valid_visibilities ~w(public unlisted private direct) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 32d94e3e9..8422fc3d5 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,7 +130,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end - test "in increments vote counters on question activities" do + test "it rewrites Note votes to Answers and increments vote counters on question activities" do user = insert(:user) {:ok, activity} = @@ -148,8 +148,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) |> Kernel.put_in(["object", "to"], user.ap_id) - {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) - + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + answer_object = Object.normalize(activity) + assert answer_object.data["type"] == "Answer" object = Object.get_by_ap_id(object.data["id"]) assert Enum.any?( @@ -1257,4 +1258,28 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, _} = Transmogrifier.prepare_outgoing(activity.data) end end + + test "Rewrites Answers to Notes" do + user = insert(:user) + + {:ok, poll_activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + poll_object = Object.normalize(poll_activity) + # TODO: Replace with CommonAPI vote creation when implemented + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) + + assert data["object"]["type"] == "Note" + end end From ac7702f8009bf244a9e77cba39114dbb6584739f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:49:19 +0300 Subject: [PATCH 19/62] Exclude Answers from fetching by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5b3fb7e84..db5a4a7ee 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,6 +492,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) + |> Activity.with_preloaded_object() + |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) |> where( @@ -832,6 +834,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => "true"}), do: query + + defp exclude_poll_votes(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, + where: fragment("not(?->>'type' = ?)", o.data, "Answer") + ) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -865,6 +879,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_order(opts) + |> exclude_poll_votes(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) From e6b175ed6cafcd6a05120e003cfe40a04b38849f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:57:46 +0300 Subject: [PATCH 20/62] Fix credo issues --- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c501c213c..7a393a817 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -357,7 +357,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) %{ - # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + # Mastodon uses separate ids for polls, but an object can't have + # more than one poll embedded so object id is fine id: object.id, expires_at: Utils.to_masto_date(end_time), expired: expired, From 8b2d39c1ec8f47df8a2159c23eabdc61b983ddf0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 23 May 2019 14:03:16 +0300 Subject: [PATCH 21/62] Change the order of preloading when fetching activities for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index db5a4a7ee..b06200cb5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,7 +492,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) - |> Activity.with_preloaded_object() + |> maybe_preload_objects(opts) |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) @@ -513,7 +513,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_activities_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(opts) - |> Activity.with_preloaded_object() |> Repo.all() end @@ -521,7 +520,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Pleroma.FlakeId.t() | nil def fetch_latest_activity_id_for_context(context, opts \\ %{}) do context - |> fetch_activities_for_context_query(opts) + |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts)) |> limit(1) |> select([a], a.id) |> Repo.one() From 4030837d91b7ff8525513589d5d810e9a1a6b959 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:19:47 +0000 Subject: [PATCH 22/62] notification: add non_follows/non_followers notification control settings --- lib/pleroma/notification.ex | 32 +++++++++++++++++++++++++++++++- lib/pleroma/user/info.ex | 11 +++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 844264307..4095e0474 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -166,7 +166,17 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: [] def skip?(activity, user) do - [:self, :blocked, :local, :muted, :followers, :follows, :recently_followed] + [ + :self, + :blocked, + :local, + :muted, + :followers, + :follows, + :non_followers, + :non_follows, + :recently_followed + ] |> Enum.any?(&skip?(&1, activity, user)) end @@ -201,12 +211,32 @@ defmodule Pleroma.Notification do User.following?(follower, user) end + def skip?( + :non_followers, + activity, + %{info: %{notification_settings: %{"non_followers" => false}}} = user + ) do + actor = activity.data["actor"] + follower = User.get_cached_by_ap_id(actor) + !User.following?(follower, user) + end + def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => false}}} = user) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) end + def skip?( + :non_follows, + activity, + %{info: %{notification_settings: %{"non_follows" => false}}} = user + ) do + actor = activity.data["actor"] + followed = User.get_cached_by_ap_id(actor) + !User.following?(user, followed) + end + def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 6397e2737..fb4cf3cc3 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -47,7 +47,14 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, - default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} + default: %{ + "remote" => true, + "local" => true, + "followers" => true, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } ) # Found in the wild @@ -71,7 +78,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows"]) + |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 1542cccbbcc2935add83d1428b4cd9e4b146f1ec Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:22:13 +0000 Subject: [PATCH 23/62] tests: chase notification setting changes --- test/web/mastodon_api/account_view_test.exs | 4 +++- test/web/twitter_api/util_controller_test.exs | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index aaf2261bb..6f8480ee2 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -81,7 +81,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do "remote" => true, "local" => true, "followers" => true, - "follows" => true + "follows" => true, + "non_follows" => true, + "non_followers" => true } privacy = user.info.default_scope diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 2cd82b3e7..ca0b8cc26 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -110,8 +110,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) - assert %{"remote" => false, "local" => true, "followers" => false, "follows" => true} == - user.info.notification_settings + assert %{ + "remote" => false, + "local" => true, + "followers" => false, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } == user.info.notification_settings end end From 0f7eeb0943472da80405639f17b9bc99e4fcd417 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:25:40 +0000 Subject: [PATCH 24/62] tests: add tests for non-follows/non-followers settings --- test/notification_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/notification_test.exs b/test/notification_test.exs index 581db58a8..b54414dcd 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -113,6 +113,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, followed) end + test "it disables notifications from non-followers" do + follower = insert(:user) + followed = insert(:user, info: %{notification_settings: %{"non_followers" => false}}) + {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"}) + assert nil == Notification.create_notification(activity, followed) + end + test "it disables notifications from people the user follows" do follower = insert(:user, info: %{notification_settings: %{"follows" => false}}) followed = insert(:user) @@ -122,6 +129,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, follower) end + test "it disables notifications from people the user does not follow" do + follower = insert(:user, info: %{notification_settings: %{"non_follows" => false}}) + followed = insert(:user) + {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"}) + assert nil == Notification.create_notification(activity, follower) + end + test "it doesn't create a notification for user if he is the activity author" do activity = insert(:note_activity) author = User.get_cached_by_ap_id(activity.data["actor"]) From 59a703fcbe6436c92d0e276caaf55f599c3165f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:31:13 +0000 Subject: [PATCH 25/62] twitter api: user view: expose user notification settings under pleroma object --- lib/pleroma/web/twitter_api/views/user_view.ex | 7 +++++++ test/web/twitter_api/views/user_view_test.exs | 2 ++ 2 files changed, 9 insertions(+) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index f0a4ddbd3..550f35f5f 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -121,6 +121,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "tags" => user.tags } |> maybe_with_activation_status(user, for_user) + |> with_notification_settings(user, for_user) } |> maybe_with_user_settings(user, for_user) |> maybe_with_role(user, for_user) @@ -132,6 +133,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do end end + defp with_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do + Map.put(data, "notification_settings", user.info.notification_settings) + end + + defp with_notification_settings(data, _, _), do: data + defp maybe_with_activation_status(data, user, %User{info: %{is_admin: true}}) do Map.put(data, "deactivated", user.info.deactivated) end diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs index 74526673c..a48fc9b78 100644 --- a/test/web/twitter_api/views/user_view_test.exs +++ b/test/web/twitter_api/views/user_view_test.exs @@ -112,9 +112,11 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do as_user = UserView.render("show.json", %{user: user, for: user}) assert as_user["default_scope"] == user.info.default_scope assert as_user["no_rich_text"] == user.info.no_rich_text + assert as_user["pleroma"]["notification_settings"] == user.info.notification_settings as_stranger = UserView.render("show.json", %{user: user}) refute as_stranger["default_scope"] refute as_stranger["no_rich_text"] + refute as_stranger["pleroma"]["notification_settings"] end test "A user for a given other follower", %{user: user} do From e7e2e7a1a633e2f7e493d040e290c931320d8cc8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:54:02 +0000 Subject: [PATCH 26/62] user info: allow formdata for notification settings like every other API --- lib/pleroma/user/info.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fb4cf3cc3..b0bfdf4f4 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -75,6 +75,11 @@ defmodule Pleroma.User.Info do end def update_notification_settings(info, settings) do + settings = + settings + |> Enum.map(fn {k, v} -> {k, v in [true, "true", "True", "1"]} end) + |> Map.new() + notification_settings = info.notification_settings |> Map.merge(settings) From 5fbbc57c1b497f8d194f3bbdefd889f0a943525a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 07:25:13 +0000 Subject: [PATCH 27/62] add migration to add notification settings to user accounts --- ...d_non_followers_fields_to_notification_settings.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs diff --git a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs new file mode 100644 index 000000000..a88b0ea61 --- /dev/null +++ b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddNonFollowsAndNonFollowersFieldsToNotificationSettings do + use Ecto.Migration + + def change do + execute(""" + update users set info = jsonb_set(info, '{notification_settings}', '{"local": true, "remote": true, "follows": true, "followers": true, "non_follows": true, "non_followers": true}') + where local=true + """) + end +end From 750ede5764d30063587182696f7ebc50c05f8278 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:05:47 +0000 Subject: [PATCH 28/62] notification: remove local/remote match rules (too complicated) --- lib/pleroma/notification.ex | 7 ------- lib/pleroma/user/info.ex | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 4095e0474..35beffb87 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -169,7 +169,6 @@ defmodule Pleroma.Notification do [ :self, :blocked, - :local, :muted, :followers, :follows, @@ -189,12 +188,6 @@ defmodule Pleroma.Notification do User.blocks?(user, %{ap_id: actor}) end - def skip?(:local, %{local: true}, %{info: %{notification_settings: %{"local" => false}}}), - do: true - - def skip?(:local, %{local: false}, %{info: %{notification_settings: %{"remote" => false}}}), - do: true - def skip?(:muted, activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index b0bfdf4f4..74573ba83 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -48,8 +48,6 @@ defmodule Pleroma.User.Info do field(:notification_settings, :map, default: %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, @@ -83,7 +81,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) + |> Map.take(["followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 45e4642a58f5299d2cd3f142aea110a474eb477f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:20:54 +0000 Subject: [PATCH 29/62] tests: chase remote/local removal --- test/notification_test.exs | 27 ------------------- test/web/mastodon_api/account_view_test.exs | 2 -- test/web/twitter_api/util_controller_test.exs | 3 --- 3 files changed, 32 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index b54414dcd..be292abd9 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -78,33 +78,6 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, muter) end - test "it disables notifications from people on remote instances" do - user = insert(:user, info: %{notification_settings: %{"remote" => false}}) - other_user = insert(:user) - - create_activity = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "type" => "Create", - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "actor" => other_user.ap_id, - "object" => %{ - "type" => "Note", - "content" => "Hi @#{user.nickname}", - "attributedTo" => other_user.ap_id - } - } - - {:ok, %{local: false} = activity} = Transmogrifier.handle_incoming(create_activity) - assert nil == Notification.create_notification(activity, user) - end - - test "it disables notifications from people on the local instance" do - user = insert(:user, info: %{notification_settings: %{"local" => false}}) - other_user = insert(:user) - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) - assert nil == Notification.create_notification(activity, user) - end - test "it disables notifications from followers" do follower = insert(:user) followed = insert(:user, info: %{notification_settings: %{"followers" => false}}) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index 6f8480ee2..23f250990 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -78,8 +78,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user) notification_settings = %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index ca0b8cc26..cab9e5d90 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -102,7 +102,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn |> assign(:user, user) |> put("/api/pleroma/notification_settings", %{ - "remote" => false, "followers" => false, "bar" => 1 }) @@ -111,8 +110,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) assert %{ - "remote" => false, - "local" => true, "followers" => false, "follows" => true, "non_follows" => true, From 672fddb7213b69f3ee8b9a7728426373090847ed Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 08:06:26 -0500 Subject: [PATCH 30/62] Default search limit should be 40 https://docs.joinmastodon.org/api/rest/search/ --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0fe09c285..2110027c3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1084,7 +1084,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - limit: 20 + limit: 40 ) q = From db94294dfb6f25ce0f08acfb78b3c4cc2241e9f5 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 29 May 2019 14:04:58 +0000 Subject: [PATCH 31/62] [#936] fix tests --- .../admin_api/admin_api_controller_test.exs | 176 ++++++++++-------- 1 file changed, 96 insertions(+), 80 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c15c67e31..43dcf945a 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -437,27 +437,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do user = insert(:user, local: false, tags: ["foo", "bar"]) conn = get(conn, "/api/pleroma/admin/users?page=1") + users = + [ + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => false, + "tags" => ["foo", "bar"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => false, - "tags" => ["foo", "bar"] - } - ] + "users" => users } end @@ -659,35 +663,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> assign(:user, admin) |> get("/api/pleroma/admin/users?filters=local") + users = + [ + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => old_admin.id, + "local" => true, + "nickname" => old_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 3, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => old_admin.id, - "local" => true, - "nickname" => old_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [] - } - ] + "users" => users } end @@ -698,27 +706,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do conn = get(conn, "/api/pleroma/admin/users?filters=is_admin") + users = + [ + %{ + "deactivated" => false, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => admin.local, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => second_admin.id, + "nickname" => second_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => second_admin.local, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => admin.local, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => second_admin.id, - "nickname" => second_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => second_admin.local, - "tags" => [] - } - ] + "users" => users } end @@ -753,27 +765,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second") + users = + [ + %{ + "deactivated" => false, + "id" => user1.id, + "nickname" => user1.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user1.local, + "tags" => ["first"] + }, + %{ + "deactivated" => false, + "id" => user2.id, + "nickname" => user2.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user2.local, + "tags" => ["second"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => user1.id, - "nickname" => user1.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"] - }, - %{ - "deactivated" => false, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user2.local, - "tags" => ["second"] - } - ] + "users" => users } end From 6aec0d1b58c12db2fa669fcff043338174290a35 Mon Sep 17 00:00:00 2001 From: kaniini Date: Wed, 29 May 2019 22:10:16 +0000 Subject: [PATCH 32/62] Revert "Merge branch 'feature/search-authenticated-only' into 'develop'" This reverts merge request !1209 --- lib/pleroma/web/router.ex | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 08c74a742..352268b96 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -414,12 +414,7 @@ defmodule Pleroma.Web.Router do get("/trends", MastodonAPIController, :empty_array) - scope [] do - pipe_through(:oauth_read) - - get("/search", MastodonAPIController, :search) - get("/accounts/search", MastodonAPIController, :account_search) - end + get("/accounts/search", MastodonAPIController, :account_search) scope [] do pipe_through(:oauth_read_or_public) @@ -436,12 +431,14 @@ defmodule Pleroma.Web.Router do get("/accounts/:id/following", MastodonAPIController, :following) get("/accounts/:id", MastodonAPIController, :user) + get("/search", MastodonAPIController, :search) + get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites) end end scope "/api/v2", Pleroma.Web.MastodonAPI do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/search", MastodonAPIController, :search2) end @@ -483,13 +480,8 @@ defmodule Pleroma.Web.Router do get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status) get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation) - get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) - end - - scope [] do - pipe_through(:oauth_read) - get("/search", TwitterAPI.Controller, :search) + get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) end end @@ -508,7 +500,7 @@ defmodule Pleroma.Web.Router do end scope "/api", Pleroma.Web, as: :twitter_api_search do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/pleroma/search_user", TwitterAPI.Controller, :search_user) end From 99f70c7e2011ab29746a8a61d3a4b354f4513045 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 30 May 2019 15:33:58 +0700 Subject: [PATCH 33/62] Use Pleroma.Config everywhere --- lib/healthcheck.ex | 4 +- lib/pleroma/emoji.ex | 4 +- lib/pleroma/html.ex | 20 ++---- lib/pleroma/http/http.ex | 7 +- lib/pleroma/plugs/federating_plug.ex | 2 +- lib/pleroma/reverse_proxy.ex | 2 +- lib/pleroma/user.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++-- .../activity_pub/activity_pub_controller.ex | 2 +- lib/pleroma/web/activity_pub/mrf.ex | 4 +- lib/pleroma/web/endpoint.ex | 2 +- lib/pleroma/web/media_proxy/media_proxy.ex | 34 +++++----- .../web/nodeinfo/nodeinfo_controller.ex | 66 ++++++++----------- lib/pleroma/web/templates/layout/app.html.eex | 4 +- .../mastodon_api/mastodon/index.html.eex | 2 +- .../web/twitter_api/twitter_api_controller.ex | 2 +- test/web/plugs/federating_plug_test.exs | 12 +--- .../twitter_api_controller_test.exs | 48 +++----------- 18 files changed, 80 insertions(+), 151 deletions(-) diff --git a/lib/healthcheck.ex b/lib/healthcheck.ex index 646fb3b9d..32aafc210 100644 --- a/lib/healthcheck.ex +++ b/lib/healthcheck.ex @@ -29,13 +29,13 @@ defmodule Pleroma.Healthcheck do end defp assign_db_info(healthcheck) do - database = Application.get_env(:pleroma, Repo)[:database] + database = Pleroma.Config.get([Repo, :database]) query = "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;" result = Repo.query!(query) - pool_size = Application.get_env(:pleroma, Repo)[:pool_size] + pool_size = Pleroma.Config.get([Repo, :pool_size]) db_info = Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states -> diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 6390cce4c..7d12eff7f 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Emoji do @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] - @groups Application.get_env(:pleroma, :emoji)[:groups] + @groups Pleroma.Config.get([:emoji, :groups]) @doc false def start_link do @@ -112,7 +112,7 @@ defmodule Pleroma.Emoji do # Compat thing for old custom emoji handling & default emoji, # it should run even if there are no emoji packs - shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) emojis = (load_from_file("config/emoji.txt") ++ diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index d1da746de..e5e78ee4f 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -104,7 +104,6 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do paragraphs, breaks and links are allowed through the filter. """ - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) require HtmlSanitizeEx.Scrubber.Meta @@ -142,9 +141,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do Meta.allow_tag_with_these_attributes("span", []) # allow inline images for custom emoji - @allow_inline_images Keyword.get(@markup, :allow_inline_images) - - if @allow_inline_images do + if Pleroma.Config.get([:markup, :allow_inline_images]) do # restrict img tags to http/https only, because of MediaProxy. Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"]) @@ -168,7 +165,6 @@ defmodule Pleroma.HTML.Scrubber.Default do # credo:disable-for-previous-line # No idea how to fix this one… - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) Meta.remove_cdata_sections_before_scrub() @@ -213,7 +209,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"]) Meta.allow_tag_with_these_attributes("span", []) - @allow_inline_images Keyword.get(@markup, :allow_inline_images) + @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do # restrict img tags to http/https only, because of MediaProxy. @@ -228,9 +224,7 @@ defmodule Pleroma.HTML.Scrubber.Default do ]) end - @allow_tables Keyword.get(@markup, :allow_tables) - - if @allow_tables do + if Pleroma.Config.get([:markup, :allow_tables]) do Meta.allow_tag_with_these_attributes("table", []) Meta.allow_tag_with_these_attributes("tbody", []) Meta.allow_tag_with_these_attributes("td", []) @@ -239,9 +233,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("tr", []) end - @allow_headings Keyword.get(@markup, :allow_headings) - - if @allow_headings do + if Pleroma.Config.get([:markup, :allow_headings]) do Meta.allow_tag_with_these_attributes("h1", []) Meta.allow_tag_with_these_attributes("h2", []) Meta.allow_tag_with_these_attributes("h3", []) @@ -249,9 +241,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("h5", []) end - @allow_fonts Keyword.get(@markup, :allow_fonts) - - if @allow_fonts do + if Pleroma.Config.get([:markup, :allow_fonts]) do Meta.allow_tag_with_these_attributes("font", ["face"]) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index c5f720bc9..c96ee7353 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,12 +65,9 @@ defmodule Pleroma.HTTP do end def process_request_options(options) do - config = Application.get_env(:pleroma, :http, []) - proxy = Keyword.get(config, :proxy_url, nil) - - case proxy do + case Pleroma.Config.get([:http, :proxy_url]) do nil -> options - _ -> options ++ [proxy: proxy] + proxy -> options ++ [proxy: proxy] end end diff --git a/lib/pleroma/plugs/federating_plug.ex b/lib/pleroma/plugs/federating_plug.ex index effc154bf..4dc4e9279 100644 --- a/lib/pleroma/plugs/federating_plug.ex +++ b/lib/pleroma/plugs/federating_plug.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.FederatingPlug do end def call(conn, _opts) do - if Keyword.get(Application.get_env(:pleroma, :instance), :federating) do + if Pleroma.Config.get([:instance, :federating]) do conn else conn diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 6e5feb4c3..983e156f5 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @hackney Application.get_env(:pleroma, :hackney, :hackney) + @hackney Pleroma.Config.get(:hackney, :hackney) @default_hackney_options [] diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 653dec95f..5c91dc7d4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -366,9 +366,7 @@ defmodule Pleroma.User do end def follow(%User{} = follower, %User{info: info} = followed) do - user_config = Application.get_env(:pleroma, :user) - deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked) - + deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) ap_followers = followed.follower_address cond do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index aa0229db7..bdd7e78d2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -399,16 +399,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def block(blocker, blocked, activity_id \\ nil, local \\ true) do - ap_config = Application.get_env(:pleroma, :activitypub) - unfollow_blocked = Keyword.get(ap_config, :unfollow_blocked) - outgoing_blocks = Keyword.get(ap_config, :outgoing_blocks) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + unfollow_blocked = Pleroma.Config.get([:activitypub, :unfollow_blocked]) - with true <- unfollow_blocked do + if unfollow_blocked do follow_activity = fetch_latest_follow(blocker, blocked) - - if follow_activity do - unfollow(blocker, blocked, nil, local) - end + if follow_activity, do: unfollow(blocker, blocked, nil, local) end with true <- outgoing_blocks, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index ad2ca1e54..0182bda46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug(:relay_active? when action in [:relay]) def relay_active?(conn, _) do - if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do + if Pleroma.Config.get([:instance, :allow_relay]) do conn else conn diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 1aaa20050..3bf7955f3 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -17,9 +17,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do end def get_policies do - Application.get_env(:pleroma, :instance, []) - |> Keyword.get(:rewrite_policy, []) - |> get_policies() + Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies() end defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 8cd7a2270..bd76e4295 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.Endpoint do parsers: [:urlencoded, :multipart, :json], pass: ["*/*"], json_decoder: Jason, - length: Application.get_env(:pleroma, :instance) |> Keyword.get(:upload_limit), + length: Pleroma.Config.get([:instance, :upload_limit]), body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} ) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 5762e767b..cee6d8481 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -12,25 +12,27 @@ defmodule Pleroma.Web.MediaProxy do def url("/" <> _ = url), do: url def url(url) do - config = Application.get_env(:pleroma, :media_proxy, []) - domain = URI.parse(url).host - - cond do - !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) -> - url - - Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> - String.equivalent?(domain, pattern) - end) -> - url - - true -> - encode_url(url) + if !enabled?() or local?(url) or whitelisted?(url) do + url + else + encode_url(url) end end + defp enabled?, do: Pleroma.Config.get([:media_proxy, :enabled], false) + + defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) + + defp whitelisted?(url) do + %{host: domain} = URI.parse(url) + + Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> + String.equivalent?(domain, pattern) + end) + end + def encode_url(url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) # Must preserve `%2F` for compatibility with S3 # https://git.pleroma.social/pleroma/pleroma/issues/580 @@ -52,7 +54,7 @@ defmodule Pleroma.Web.MediaProxy do end def decode_url(sig, url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) sig = Base.url_decode64!(sig, @base64_opts) local_sig = :crypto.hmac(:sha, secret, url) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 45f90c579..59f3d4e11 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -32,20 +32,15 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field # under software. def raw_nodeinfo do - instance = Application.get_env(:pleroma, :instance) - media_proxy = Application.get_env(:pleroma, :media_proxy) - suggestions = Application.get_env(:pleroma, :suggestions) - chat = Application.get_env(:pleroma, :chat) - gopher = Application.get_env(:pleroma, :gopher) stats = Stats.get_stats() mrf_simple = - Application.get_env(:pleroma, :mrf_simple) + Config.get(:mrf_simple) |> Enum.into(%{}) # This horror is needed to convert regex sigils to strings mrf_keyword = - Application.get_env(:pleroma, :mrf_keyword, []) + Config.get(:mrf_keyword, []) |> Enum.map(fn {key, value} -> {key, Enum.map(value, fn @@ -74,14 +69,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do MRF.get_policies() |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - quarantined = Keyword.get(instance, :quarantined_instances) - - quarantined = - if is_list(quarantined) do - quarantined - else - [] - end + quarantined = Config.get([:instance, :quarantined_instances], []) staff_accounts = User.all_superusers() @@ -92,7 +80,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) federation_response = - if Keyword.get(instance, :mrf_transparency) do + if Config.get([:instance, :mrf_transparency]) do %{ mrf_policies: mrf_policies, mrf_simple: mrf_simple, @@ -109,22 +97,22 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "pleroma_api", "mastodon_api", "mastodon_api_streaming", - if Keyword.get(media_proxy, :enabled) do + if Config.get([:media_proxy, :enabled]) do "media_proxy" end, - if Keyword.get(gopher, :enabled) do + if Config.get([:gopher, :enabled]) do "gopher" end, - if Keyword.get(chat, :enabled) do + if Config.get([:chat, :enabled]) do "chat" end, - if Keyword.get(suggestions, :enabled) do + if Config.get([:suggestions, :enabled]) do "suggestions" end, - if Keyword.get(instance, :allow_relay) do + if Config.get([:instance, :allow_relay]) do "relay" end, - if Keyword.get(instance, :safe_dm_mentions) do + if Config.get([:instance, :safe_dm_mentions]) do "safe_dm_mentions" end ] @@ -141,7 +129,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do inbound: [], outbound: [] }, - openRegistrations: Keyword.get(instance, :registrations_open), + openRegistrations: Config.get([:instance, :registrations_open]), usage: %{ users: %{ total: stats.user_count || 0 @@ -149,29 +137,29 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do localPosts: stats.status_count || 0 }, metadata: %{ - nodeName: Keyword.get(instance, :name), - nodeDescription: Keyword.get(instance, :description), - private: !Keyword.get(instance, :public, true), + nodeName: Config.get([:instance, :name]), + nodeDescription: Config.get([:instance, :description]), + private: !Config.get([:instance, :public], true), suggestions: %{ - enabled: Keyword.get(suggestions, :enabled, false), - thirdPartyEngine: Keyword.get(suggestions, :third_party_engine, ""), - timeout: Keyword.get(suggestions, :timeout, 5000), - limit: Keyword.get(suggestions, :limit, 23), - web: Keyword.get(suggestions, :web, "") + enabled: Config.get([:suggestions, :enabled], false), + thirdPartyEngine: Config.get([:suggestions, :third_party_engine], ""), + timeout: Config.get([:suggestions, :timeout], 5000), + limit: Config.get([:suggestions, :limit], 23), + web: Config.get([:suggestions, :web], "") }, staffAccounts: staff_accounts, federation: federation_response, - postFormats: Keyword.get(instance, :allowed_post_formats), + postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ - general: Keyword.get(instance, :upload_limit), - avatar: Keyword.get(instance, :avatar_upload_limit), - banner: Keyword.get(instance, :banner_upload_limit), - background: Keyword.get(instance, :background_upload_limit) + general: Config.get([:instance, :upload_limit]), + avatar: Config.get([:instance, :avatar_upload_limit]), + banner: Config.get([:instance, :banner_upload_limit]), + background: Config.get([:instance, :background_upload_limit]) }, - accountActivationRequired: Keyword.get(instance, :account_activation_required, false), - invitesEnabled: Keyword.get(instance, :invites_enabled, false), + accountActivationRequired: Config.get([:instance, :account_activation_required], false), + invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) } } end diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 3389c91cc..85ec4d76c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -4,7 +4,7 @@ - <%= Application.get_env(:pleroma, :instance)[:name] %> + <%= Pleroma.Config.get([:instance, :name]) %>