From 9854978b8bc3c3e4326b6dde9758dd276cd36e1c Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 9 Jan 2019 12:38:23 +0100 Subject: [PATCH 01/30] Remove recent activity restriction. This should be fine now, everything should be covered by indices. --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4685f6d95..ea65538b6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -517,15 +517,6 @@ defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or defp restrict_reblogs(query, _), do: query - # Only search through last 100_000 activities by default - defp restrict_recent(query, %{"whole_db" => true}), do: query - - defp restrict_recent(query, _) do - since = (Repo.aggregate(Activity, :max, :id) || 0) - 100_000 - - from(activity in query, where: activity.id > ^since) - end - defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do blocks = info.blocks || [] domain_blocks = info.domain_blocks || [] @@ -570,7 +561,6 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_actor(opts) |> restrict_type(opts) |> restrict_favorited_by(opts) - |> restrict_recent(opts) |> restrict_blocked(opts) |> restrict_media(opts) |> restrict_visibility(opts) From cc3a83a7306262d9334b8cfb30ac99c024523d54 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 10 Jan 2019 21:29:05 +0000 Subject: [PATCH 02/30] Fix nginx caching issues Nginx is currently not caching data because proxy_buffering needs to be enabled for caching to work at all, and we are receiving a Cache-Control header from Pleroma that states "max-age=0, private, must-revalidate" Even disregarding the Cache-Control header that should actually be set to "public, max-age=1209600" as defined in the reverse_proxy code, we don't want to obey this header at all as it overrides our Nginx caching rules. --- installation/pleroma.nginx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index 46b84fb50..a24bb0e61 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -79,8 +79,10 @@ server { proxy_cache_valid 200 206 301 304 1h; proxy_cache_lock on; proxy_ignore_client_abort on; - proxy_buffering off; + proxy_buffering on; chunked_transfer_encoding on; + proxy_ignore_headers Cache-Control; + proxy_hide_header Cache-Control; proxy_pass http://localhost:4000; } } From 144b48da95867b33315c6a0e6e865c457d89452b Mon Sep 17 00:00:00 2001 From: Sadposter Date: Sat, 12 Jan 2019 14:03:35 +0000 Subject: [PATCH 03/30] Add link headers to MastoAPI /favourites As documented at https://docs.joinmastodon.org/api/rest/favourites/ --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e00a3fb87..b83539bad 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -836,6 +836,7 @@ def favourites(%{assigns: %{user: user}} = conn, _) do |> Enum.reverse() conn + |> add_link_headers(:favourites, activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end From 9daf16246121f3d5eeb4d6caeffc237cd5526546 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Sat, 12 Jan 2019 14:42:52 +0000 Subject: [PATCH 04/30] Honour parameters on MastoAPI /favourites --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index b83539bad..a8fe9d708 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -824,9 +824,9 @@ def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) d json(conn, res) end - def favourites(%{assigns: %{user: user}} = conn, _) do + def favourites(%{assigns: %{user: user}} = conn, params) do params = - %{} + params |> Map.put("type", "Create") |> Map.put("favorited_by", user.ap_id) |> Map.put("blocking_user", user) From 1eb7318831e7239ec929457f6298fb05cb461b43 Mon Sep 17 00:00:00 2001 From: sxsdv1 Date: Sat, 12 Jan 2019 17:52:30 +0100 Subject: [PATCH 05/30] Prepare all types objects before serialising Activities returned from inbox can include other types of objects like Article --- .../web/activity_pub/transmogrifier.ex | 2 +- test/support/factory.ex | 25 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 15 +++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 87b7fc07f..b0f8c59cc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -641,7 +641,7 @@ def prepare_object(object) do # internal -> Mastodon # """ - def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do + def prepare_outgoing(%{"type" => "Create", "object" => object} = data) do object = object |> prepare_object diff --git a/test/support/factory.ex b/test/support/factory.ex index 57fa4a79d..4ac77981a 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -57,6 +57,11 @@ def direct_note_factory do %Pleroma.Object{data: Map.merge(data, %{"to" => [user2.ap_id]})} end + def article_factory do + note_factory() + |> Map.put("type", "Article") + end + def tombstone_factory do data = %{ "type" => "Tombstone", @@ -110,6 +115,26 @@ def note_activity_factory do } end + def article_activity_factory do + article = insert(:article) + + data = %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Create", + "actor" => article.data["actor"], + "to" => article.data["to"], + "object" => article.data, + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "context" => article.data["context"] + } + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + end + def announce_activity_factory do note_activity = insert(:note_activity) user = insert(:user) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a5fd87ed4..65c8ec36d 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -835,6 +835,21 @@ test "it strips internal fields" do assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["context_id"]) end + + test "it strips internal fields of article" do + activity = insert(:article_activity) + + {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) + + assert length(modified["object"]["tag"]) == 2 + + assert is_nil(modified["object"]["emoji"]) + assert is_nil(modified["object"]["likes"]) + assert is_nil(modified["object"]["like_count"]) + assert is_nil(modified["object"]["announcements"]) + assert is_nil(modified["object"]["announcement_count"]) + assert is_nil(modified["object"]["context_id"]) + end end describe "user upgrade" do From 36711e1c83bb24a2b104c4a8f384c475c1583638 Mon Sep 17 00:00:00 2001 From: sxsdv1 Date: Tue, 8 Jan 2019 19:22:26 +0100 Subject: [PATCH 06/30] Handle client submitted activitypub like activity --- .../activity_pub/activity_pub_controller.ex | 9 +++++++ .../activity_pub_controller_test.exs | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 73ca07e84..6bc8b7195 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -204,6 +204,15 @@ def handle_user_activity(user, %{"type" => "Delete"} = params) do end end + def handle_user_activity(user, %{"type" => "Like"} = params) do + with %Object{} = object <- Object.normalize(params["object"]), + {:ok, activity, _object} <- ActivityPub.like(user, object) do + {:ok, activity} + else + _ -> {:error, "Can't like object"} + end + end + def handle_user_activity(_, _) do {:error, "Unhandled activity type"} end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 7aed8c71d..82ad42144 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -292,6 +292,31 @@ test "it rejects delete activity of object from other actor", %{conn: conn} do assert json_response(conn, 400) end + + test "it increases like count when receiving a like action", %{conn: conn} do + note_activity = insert(:note_activity) + user = User.get_cached_by_ap_id(note_activity.data["actor"]) + + data = %{ + type: "Like", + object: %{ + id: note_activity.data["object"]["id"] + } + } + + conn = + conn + |> assign(:user, user) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/outbox", data) + + result = json_response(conn, 201) + assert Activity.get_by_ap_id(result["id"]) + + object = Object.get_by_ap_id(note_activity.data["object"]["id"]) + assert object + assert object.data["like_count"] == 1 + end end describe "/users/:nickname/followers" do From 581edd5a91189e6fb2a94a277b96f9c8197617b8 Mon Sep 17 00:00:00 2001 From: sxsdv1 Date: Fri, 11 Jan 2019 23:34:32 +0100 Subject: [PATCH 07/30] Add route to get object like activities --- .../activity_pub/activity_pub_controller.ex | 30 ++++++++++++++++ lib/pleroma/web/activity_pub/utils.ex | 21 ++++++++++++ .../web/activity_pub/views/object_view.ex | 34 +++++++++++++++++++ lib/pleroma/web/router.ex | 1 + .../activity_pub_controller_test.exs | 15 ++++++++ 5 files changed, 101 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 6bc8b7195..7eed0a600 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -54,6 +54,36 @@ def object(conn, %{"uuid" => uuid}) do end end + def object_likes(conn, %{"uuid" => uuid, "page" => page}) do + with ap_id <- o_status_url(conn, :object, uuid), + %Object{} = object <- Object.get_cached_by_ap_id(ap_id), + {_, true} <- {:public?, ActivityPub.is_public?(object)}, + likes <- Utils.get_object_likes(object) do + {page, _} = Integer.parse(page) + + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ObjectView.render("likes.json", ap_id, likes, page)) + else + {:public?, false} -> + {:error, :not_found} + end + end + + def object_likes(conn, %{"uuid" => uuid}) do + with ap_id <- o_status_url(conn, :object, uuid), + %Object{} = object <- Object.get_cached_by_ap_id(ap_id), + {_, true} <- {:public?, ActivityPub.is_public?(object)}, + likes <- Utils.get_object_likes(object) do + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(ObjectView.render("likes.json", ap_id, likes)) + else + {:public?, false} -> + {:error, :not_found} + end + end + def activity(conn, %{"uuid" => uuid}) do with ap_id <- o_status_url(conn, :activity, uuid), %Activity{} = activity <- Activity.normalize(ap_id), diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index b313996db..6ecab773c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -231,6 +231,27 @@ def get_existing_like(actor, %{data: %{"id" => id}}) do Repo.one(query) end + @doc """ + Returns like activities targeting an object + """ + def get_object_likes(%{data: %{"id" => id}}) do + query = + from( + activity in Activity, + # this is to use the index + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ?", + activity.data, + activity.data, + ^id + ), + where: fragment("(?)->>'type' = 'Like'", activity.data) + ) + + Repo.all(query) + end + def make_like_data(%User{ap_id: ap_id} = actor, %{data: %{"id" => id}} = object, activity_id) do data = %{ "type" => "Like", diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index b5c9bf8d0..193042056 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -35,4 +35,38 @@ def render("object.json", %{object: %Activity{} = activity}) do Map.merge(base, additional) end + + def render("likes.json", ap_id, likes, page) do + collection(likes, "#{ap_id}/likes", page) + |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) + end + + def render("likes.json", ap_id, likes) do + %{ + "id" => "#{ap_id}/likes", + "type" => "OrderedCollection", + "totalItems" => length(likes), + "first" => collection(likes, "#{ap_id}/followers", 1) + } + |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) + end + + def collection(collection, iri, page) do + offset = (page - 1) * 10 + items = Enum.slice(collection, offset, 10) + items = Enum.map(items, fn object -> Transmogrifier.prepare_object(object.data) end) + total = length(collection) + + map = %{ + "id" => "#{iri}?page=#{page}", + "type" => "OrderedCollectionPage", + "partOf" => iri, + "totalItems" => total, + "orderedItems" => items + } + + if offset < total do + Map.put(map, "next", "#{iri}?page=#{page + 1}") + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a5f4d8126..7a0c9fd25 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -421,6 +421,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/followers", ActivityPubController, :followers) get("/users/:nickname/following", ActivityPubController, :following) get("/users/:nickname/outbox", ActivityPubController, :outbox) + get("/objects/:uuid/likes", ActivityPubController, :object_likes) end pipeline :activitypub_client do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 82ad42144..52e67f046 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -89,6 +89,21 @@ test "it returns 404 for tombstone objects", %{conn: conn} do end end + describe "/object/:uuid/likes" do + test "it returns the like activities in a collection", %{conn: conn} do + like = insert(:like_activity) + uuid = String.split(like.data["object"], "/") |> List.last() + + result = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}/likes") + |> json_response(200) + + assert List.first(result["first"]["orderedItems"])["id"] == like.data["id"] + end + end + describe "/activities/:uuid" do test "it returns a json representation of the activity", %{conn: conn} do activity = insert(:note_activity) From 868034375c5122175f872967e49559dafed9403c Mon Sep 17 00:00:00 2001 From: sxsdv1 Date: Wed, 9 Jan 2019 09:22:00 +0100 Subject: [PATCH 08/30] Add likes to activitypub object representation Top level of the likes OrderedCollection is inlined to get immediate access to totalItems. Because the count can be returned without scanning the database for like activities the extra query is saved when the client only wants to display the total. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 18 +++++++++++++++++- test/web/activity_pub/transmogrifier_test.exs | 10 ++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b0f8c59cc..86d11c874 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -629,6 +629,7 @@ def prepare_object(object) do |> add_mention_tags |> add_emoji_tags |> add_attributed_to + |> add_likes |> prepare_attachments |> set_conversation |> set_reply_to_uri @@ -788,6 +789,22 @@ def add_attributed_to(object) do |> Map.put("attributedTo", attributedTo) end + def add_likes(%{"id" => id, "like_count" => likes} = object) do + likes = %{ + "id" => "#{id}/likes", + "first" => "#{id}/likes?page=1", + "type" => "OrderedCollection", + "totalItems" => likes + } + + object + |> Map.put("likes", likes) + end + + def add_likes(object) do + object + end + def prepare_attachments(object) do attachments = (object["attachment"] || []) @@ -803,7 +820,6 @@ def prepare_attachments(object) do defp strip_internal_fields(object) do object |> Map.drop([ - "likes", "like_count", "announcements", "announcement_count", diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 65c8ec36d..87d0ab559 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -829,7 +829,6 @@ test "it strips internal fields" do assert length(modified["object"]["tag"]) == 2 assert is_nil(modified["object"]["emoji"]) - assert is_nil(modified["object"]["likes"]) assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) @@ -844,12 +843,19 @@ test "it strips internal fields of article" do assert length(modified["object"]["tag"]) == 2 assert is_nil(modified["object"]["emoji"]) - assert is_nil(modified["object"]["likes"]) assert is_nil(modified["object"]["like_count"]) assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["context_id"]) end + + test "it adds like collection to object" do + activity = insert(:note_activity) + {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) + + assert modified["object"]["likes"]["type"] == "OrderedCollection" + assert modified["object"]["likes"]["totalItems"] == 0 + end end describe "user upgrade" do From b8a77c5d704a4a76f227854c1cab560eb98a7382 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 13 Jan 2019 02:06:50 +0200 Subject: [PATCH 09/30] Add OEmbed parser --- lib/pleroma/web/rich_media/parser.ex | 6 ++- .../web/rich_media/parsers/oembed_parser.ex | 27 ++++++++++++ test/fixtures/rich_media/oembed.html | 3 ++ test/fixtures/rich_media/oembed.json | 1 + test/web/rich_media/parser_test.exs | 41 +++++++++++++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/rich_media/parsers/oembed_parser.ex create mode 100644 test/fixtures/rich_media/oembed.html create mode 100644 test/fixtures/rich_media/oembed.json diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index fe092bf19..6da83c6e4 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -1,5 +1,9 @@ defmodule Pleroma.Web.RichMedia.Parser do - @parsers [Pleroma.Web.RichMedia.Parsers.OGP, Pleroma.Web.RichMedia.Parsers.TwitterCard] + @parsers [ + Pleroma.Web.RichMedia.Parsers.OGP, + Pleroma.Web.RichMedia.Parsers.TwitterCard, + Pleroma.Web.RichMedia.Parsers.OEmbed + ] if Mix.env() == :test do def parse(url), do: parse_url(url) diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex new file mode 100644 index 000000000..ca7226faf --- /dev/null +++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex @@ -0,0 +1,27 @@ +defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do + def parse(html, _data) do + with elements = [_ | _] <- get_discovery_data(html), + {:ok, oembed_url} <- get_oembed_url(elements), + {:ok, oembed_data} <- get_oembed_data(oembed_url) do + {:ok, oembed_data} + else + _e -> {:error, "No OEmbed data found"} + end + end + + defp get_discovery_data(html) do + html |> Floki.find("link[type='application/json+oembed']") + end + + defp get_oembed_url(nodes) do + {"link", attributes, _children} = nodes |> hd() + + {:ok, Enum.into(attributes, %{})["href"]} + end + + defp get_oembed_data(url) do + {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url) + + {:ok, Poison.decode!(json)} + end +end diff --git a/test/fixtures/rich_media/oembed.html b/test/fixtures/rich_media/oembed.html new file mode 100644 index 000000000..55f17004b --- /dev/null +++ b/test/fixtures/rich_media/oembed.html @@ -0,0 +1,3 @@ + diff --git a/test/fixtures/rich_media/oembed.json b/test/fixtures/rich_media/oembed.json new file mode 100644 index 000000000..2a5f7a771 --- /dev/null +++ b/test/fixtures/rich_media/oembed.json @@ -0,0 +1 @@ +{"type":"photo","flickr_type":"photo","title":"Bacon Lollys","author_name":"\u202e\u202d\u202cbees\u202c","author_url":"https:\/\/www.flickr.com\/photos\/bees\/","width":"1024","height":"768","url":"https:\/\/farm4.staticflickr.com\/3040\/2362225867_4a87ab8baf_b.jpg","web_page":"https:\/\/www.flickr.com\/photos\/bees\/2362225867\/","thumbnail_url":"https:\/\/farm4.staticflickr.com\/3040\/2362225867_4a87ab8baf_q.jpg","thumbnail_width":150,"thumbnail_height":150,"web_page_short_url":"https:\/\/flic.kr\/p\/4AK2sc","license":"All Rights Reserved","license_id":0,"html":"\"Bacon<\/a>", + "license" => "All Rights Reserved", + "license_id" => 0, + "provider_name" => "Flickr", + "provider_url" => "https://www.flickr.com/", + "thumbnail_height" => 150, + "thumbnail_url" => + "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_q.jpg", + "thumbnail_width" => 150, + "title" => "Bacon Lollys", + "type" => "photo", + "url" => "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_b.jpg", + "version" => "1.0", + "web_page" => "https://www.flickr.com/photos/bees/2362225867/", + "web_page_short_url" => "https://flic.kr/p/4AK2sc", + "width" => "1024" + }} + end end From 98d9ae071877cb3e6cd65d84bfc142e9cbb998b0 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Sun, 13 Jan 2019 15:17:47 +0000 Subject: [PATCH 10/30] Add test for mastodon API /favourites endpoint --- .../mastodon_api_controller_test.exs | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b448d13f5..fe8f845c7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1349,13 +1349,42 @@ test "returns the favorites of a user", %{conn: conn} do {:ok, _, _} = CommonAPI.favorite(activity.id, user) - conn = + first_conn = conn |> assign(:user, user) |> get("/api/v1/favourites") - assert [status] = json_response(conn, 200) + assert [status] = json_response(first_conn, 200) assert status["id"] == to_string(activity.id) + + assert [{"link", link_header}] = + Enum.filter(first_conn.resp_headers, fn element -> match?({"link", _}, element) end) + + # Honours query params + {:ok, second_activity} = + CommonAPI.post(other_user, %{ + "status" => + "Trees Are Never Sad Look At Them Every Once In Awhile They're Quite Beautiful." + }) + + {:ok, _, _} = CommonAPI.favorite(second_activity.id, user) + + last_like = status["id"] + + second_conn = + conn + |> assign(:user, user) + |> get("/api/v1/favourites?since_id=#{last_like}") + + assert [second_status] = json_response(second_conn, 200) + assert second_status["id"] == to_string(second_activity.id) + + third_conn = + conn + |> assign(:user, user) + |> get("/api/v1/favourites?limit=0") + + assert [] = json_response(third_conn, 200) end describe "updating credentials" do From 6e5b0406b9f2eda5c78148837927d89aa9b2d6f9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 14 Jan 2019 05:31:57 +0000 Subject: [PATCH 11/30] mrf: add no placeholder-text policy, strips pointless "." content from posts with images --- .../mrf/no_placeholder_text_policy.ex | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex new file mode 100644 index 000000000..081456046 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -0,0 +1,29 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF + + @impl true + def filter( + %{ + "type" => "Create", + "object" => %{"content" => content, "attachment" => _attachment} = child_object + } = object + ) + when content in [".", "

.

"] do + child_object = + child_object + |> Map.put("content", "") + + object = + object + |> Map.put("object", child_object) + + {:ok, object} + end + + @impl true + def filter(object), do: {:ok, object} +end From 42b7584068e51a58d2bfe76729fe039fe7f6a7cf Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Mon, 14 Jan 2019 11:31:44 -0500 Subject: [PATCH 12/30] URI escape file upload URLs --- lib/pleroma/upload.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 0b1bdeec4..185ba25fa 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -215,7 +215,7 @@ defp tempfile_for_image(data) do end defp url_from_spec(base_url, {:file, path}) do - [base_url, "media", path] + [base_url, "media", URI.encode(path)] |> Path.join() end From dcbe5bd58ccb1068d17ba15703169593d4bbb393 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Mon, 14 Jan 2019 13:29:38 -0500 Subject: [PATCH 13/30] Add attachment escaping test --- test/upload_test.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/upload_test.exs b/test/upload_test.exs index d4ea3a573..bda503361 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -137,5 +137,20 @@ test "copies the file to the configured folder with anonymizing filename" do refute data["name"] == "an [image.jpg" end + + test "escapes invalid characters in url" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + filename: "an… image.jpg" + } + + {:ok, data} = Upload.store(file) + [attachment_url | _] = data["url"] + + assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg" + end end end From e8eff9fe03faa1922357d438ed973f5b83605aab Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 15 Jan 2019 02:58:48 +0200 Subject: [PATCH 14/30] Fix Elixir 1.8 type annotation issue --- lib/pleroma/upload.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 185ba25fa..2a48331d7 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -34,8 +34,9 @@ defmodule Pleroma.Upload do require Logger @type source :: - Plug.Upload.t() | data_uri_string :: - String.t() | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()} + Plug.Upload.t() + | (data_uri_string :: String.t()) + | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()} @type option :: {:type, :avatar | :banner | :background} From e3eb75bd234c8e21ff937d4f9b2a4a1328007e32 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Jan 2019 07:40:00 +0100 Subject: [PATCH 15/30] Upload: Fix uploading with a ? in the filename --- lib/pleroma/upload.ex | 7 ++++++- test/upload_test.exs | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 185ba25fa..1d8b073af 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -215,7 +215,12 @@ defp tempfile_for_image(data) do end defp url_from_spec(base_url, {:file, path}) do - [base_url, "media", URI.encode(path)] + path = + path + |> URI.encode() + |> String.replace("?", "%3F") + + [base_url, "media", path] |> Path.join() end diff --git a/test/upload_test.exs b/test/upload_test.exs index bda503361..ffef74270 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -152,5 +152,20 @@ test "escapes invalid characters in url" do assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg" end + + test "replaces ? (question-mark) to %3f" do + File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg"), + filename: "an?image.jpg" + } + + {:ok, data} = Upload.store(file) + [attachment_url | _] = data["url"] + + assert Path.basename(attachment_url["href"]) == "an%3Fimage.jpg" + end end end From 9fcdca1bdca04bdb52b7ac9a0d69e0886b12cb87 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Jan 2019 07:57:48 +0100 Subject: [PATCH 16/30] Upload: Fix uploading with a : in the filename --- lib/pleroma/upload.ex | 1 + test/upload_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 1d8b073af..b19920dff 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -219,6 +219,7 @@ defp url_from_spec(base_url, {:file, path}) do path |> URI.encode() |> String.replace("?", "%3F") + |> String.replace(":", "%3A") [base_url, "media", path] |> Path.join() diff --git a/test/upload_test.exs b/test/upload_test.exs index ffef74270..b2d9eca38 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -153,19 +153,19 @@ test "escapes invalid characters in url" do assert Path.basename(attachment_url["href"]) == "an%E2%80%A6%20image.jpg" end - test "replaces ? (question-mark) to %3f" do + test "replaces : (colon) and ? (question-mark) to %3A and %3F (respectively)" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ content_type: "image/jpg", path: Path.absname("test/fixtures/image_tmp.jpg"), - filename: "an?image.jpg" + filename: "is:an?image.jpg" } {:ok, data} = Upload.store(file) [attachment_url | _] = data["url"] - assert Path.basename(attachment_url["href"]) == "an%3Fimage.jpg" + assert Path.basename(attachment_url["href"]) == "is%3Aan%3Fimage.jpg" end end end From 17da432dbbf5f5f9afc09e6b92ec51a368c30abb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 15 Jan 2019 18:00:20 +0000 Subject: [PATCH 17/30] websub: improve error handling --- lib/pleroma/web/websub/websub.ex | 6 ++++++ lib/pleroma/web/websub/websub_controller.ex | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 3a287edd9..7ca62c83b 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -121,6 +121,12 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d end end + def incoming_subscription_request(user, params) do + Logger.info("Unhandled WebSub request for #{user.nickname}: #{inspect(params)}") + + {:error, "Invalid WebSub request"} + end + defp get_subscription(topic, callback) do Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || %WebsubServerSubscription{} diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index 27304d988..e58f144e5 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -67,6 +67,13 @@ def websub_subscription_confirmation( end end + def websub_subscription_confirmation(conn, params) do + Logger.info("Invalid WebSub confirmation request: #{inspect(params)}") + + conn + |> send_resp(500, "Invalid parameters") + end + def websub_incoming(conn, %{"id" => id}) do with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), signature <- String.downcase(signature), From 2d3241753f57ef1364370128dbf0c6489e978b41 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 15 Jan 2019 19:31:13 +0000 Subject: [PATCH 18/30] http: add support for query parameters, use Jason for JSON encoding instead of Poison like everywhere else --- lib/pleroma/http/http.ex | 3 +++ lib/pleroma/http/request_builder.ex | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b8103cef6..75c58e6c9 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -31,12 +31,15 @@ def request(method, url, body \\ "", headers \\ [], options \\ []) do process_request_options(options) |> process_sni_options(url) + params = Keyword.get(options, :params, []) + %{} |> Builder.method(method) |> Builder.headers(headers) |> Builder.opts(options) |> Builder.url(url) |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) |> Enum.into([]) |> (&Tesla.request(Connection.new(), &1)).() end diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index bffc7c6fe..5f2cff2c0 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -100,6 +100,8 @@ def add_optional_params(request, definitions, [{key, value} | tail]) do Map """ @spec add_param(map(), atom, atom, any()) :: map() + def add_param(request, :query, :query, values), do: Map.put(request, :query, values) + def add_param(request, :body, :body, value), do: Map.put(request, :body, value) def add_param(request, :body, key, value) do @@ -107,7 +109,10 @@ def add_param(request, :body, key, value) do |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) |> Map.update!( :body, - &Tesla.Multipart.add_field(&1, key, Poison.encode!(value), + &Tesla.Multipart.add_field( + &1, + key, + Jason.encode!(value), headers: [{:"Content-Type", "application/json"}] ) ) From 1ddab78247c541eb3895042a40a2ffcbcc2d751f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 16 Jan 2019 03:48:43 +0000 Subject: [PATCH 19/30] html: allow microformats-related markup through the html filter --- lib/pleroma/html.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 0c5b0f03f..f5c6e5033 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -78,14 +78,14 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do # links Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) - Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"]) # paragraphs and linebreaks Meta.allow_tag_with_these_attributes("br", []) Meta.allow_tag_with_these_attributes("p", []) # microformats - Meta.allow_tag_with_these_attributes("span", []) + Meta.allow_tag_with_these_attributes("span", ["class"]) # allow inline images for custom emoji @allow_inline_images Keyword.get(@markup, :allow_inline_images) @@ -119,7 +119,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.strip_comments() Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) - Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"]) Meta.allow_tag_with_these_attributes("abbr", ["title"]) @@ -134,7 +134,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("ol", []) Meta.allow_tag_with_these_attributes("p", []) Meta.allow_tag_with_these_attributes("pre", []) - Meta.allow_tag_with_these_attributes("span", []) + Meta.allow_tag_with_these_attributes("span", ["class"]) Meta.allow_tag_with_these_attributes("strong", []) Meta.allow_tag_with_these_attributes("u", []) Meta.allow_tag_with_these_attributes("ul", []) From 461ab9489db63d375f0d604ad9288e8e916d0beb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 16 Jan 2019 03:53:36 +0000 Subject: [PATCH 20/30] formatter: improve microformats markup --- lib/pleroma/formatter.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index d80ae6576..4149265a2 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -145,7 +145,9 @@ def add_user_links({subs, text}, mentions) do short_match = String.split(match, "@") |> tl() |> hd() {uuid, - "@#{short_match}"} + "@#{ + short_match + }"} end) {subs, uuid_text} @@ -168,7 +170,7 @@ def add_hashtag_links({subs, text}, tags) do subs ++ Enum.map(tags, fn {tag_text, tag, uuid} -> url = - "" From 85a5be6220dd87e2884b5921fc1a6c92ee7cc745 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 16 Jan 2019 04:09:01 +0000 Subject: [PATCH 21/30] tests: fixup --- test/formatter_test.exs | 16 +++++++++------- .../mastodon_api_controller_test.exs | 18 +++++++++++------- .../twitter_api_controller_test.exs | 6 +++--- test/web/twitter_api/twitter_api_test.exs | 4 ++-- .../twitter_api/views/activity_view_test.exs | 6 ++++-- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index c76149e38..bd8844458 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -19,7 +19,7 @@ test "turns hashtags into links" do text = "I love #cofe and #2hu" expected_text = - "I love and " + "I love and " tags = Formatter.parse_tags(text) @@ -31,7 +31,7 @@ test "does not turn html characters to tags" do text = "Fact #3: pleroma does what mastodon't" expected_text = - "Fact : pleroma does what mastodon't" + "Fact : pleroma does what mastodon't" tags = Formatter.parse_tags(text) @@ -144,11 +144,13 @@ test "gives a replacement for user links" do Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) expected_text = - "@gsimg According to @gsimg According to @archa_eme_, that is @daggsy. Also hello @archa_eme_, that is @daggsy. Also hello @archaeme" + }' class='u-url mention' href='#{archaeme_remote.ap_id}'>@archaeme" assert expected_text == Formatter.finalize({subs, text}) end @@ -166,7 +168,7 @@ test "gives a replacement for user links when the user is using Osada" do Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) expected_text = - "@mike test" + "@mike test" assert expected_text == Formatter.finalize({subs, text}) end @@ -183,7 +185,7 @@ test "gives a replacement for single-character local nicknames" do Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) expected_text = - "@o hi" + "@o hi" assert expected_text == Formatter.finalize({subs, text}) end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index fe8f845c7..9e4ecedc5 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -386,7 +386,7 @@ test "get a filter", %{conn: conn} do |> assign(:user, user) |> get("/api/v1/filters/#{filter.filter_id}") - assert response = json_response(conn, 200) + assert _response = json_response(conn, 200) end test "update a filter", %{conn: conn} do @@ -600,7 +600,9 @@ test "list of notifications", %{conn: conn} do |> get("/api/v1/notifications") expected_response = - "hi @#{user.nickname}" + "hi @#{user.nickname}" assert [%{"status" => %{"content" => response}} | _rest] = json_response(conn, 200) assert response == expected_response @@ -621,7 +623,9 @@ test "getting a single notification", %{conn: conn} do |> get("/api/v1/notifications/#{notification.id}") expected_response = - "hi @#{user.nickname}" + "hi @#{user.nickname}" assert %{"status" => %{"content" => response}} = json_response(conn, 200) assert response == expected_response @@ -1357,7 +1361,7 @@ test "returns the favorites of a user", %{conn: conn} do assert [status] = json_response(first_conn, 200) assert status["id"] == to_string(activity.id) - assert [{"link", link_header}] = + assert [{"link", _link_header}] = Enum.filter(first_conn.resp_headers, fn element -> match?({"link", _}, element) end) # Honours query params @@ -1402,9 +1406,9 @@ test "updates the user's bio", %{conn: conn} do assert user = json_response(conn, 200) assert user["note"] == - "I drink #cofe with #cofe with @#{user2.nickname}" + }\" class=\"u-url mention\" href=\"#{user2.ap_id}\">@#{user2.nickname}" end test "updates the user's locking status", %{conn: conn} do @@ -1495,7 +1499,7 @@ test "put settings", %{conn: conn} do |> assign(:user, user) |> put("/api/web/settings", %{"data" => %{"programming" => "socks"}}) - assert result = json_response(conn, 200) + assert _result = json_response(conn, 200) user = User.get_cached_by_ap_id(user.ap_id) assert user.info.settings == %{"programming" => "socks"} diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 5f13e7959..e08edc525 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -1357,9 +1357,9 @@ test "it updates a user's profile", %{conn: conn} do assert user.name == "new name" assert user.bio == - "hi @#{ - user2.nickname - }" + "hi @#{user2.nickname}" assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user}) end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index b9feb23d4..547592ff2 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -38,7 +38,7 @@ test "create a status" do {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) expected_text = - "Hello again, @shp.<script></script>
This is on another :moominmamma: line.
image.jpg" + "Hello again, @shp.<script></script>
This is on another :moominmamma: line.
image.jpg" assert get_in(activity.data, ["object", "content"]) == expected_text assert get_in(activity.data, ["object", "type"]) == "Note" @@ -328,7 +328,7 @@ test "it registers a new user and parses mentions in the bio" do {:ok, user2} = TwitterAPI.register_user(data2) expected_text = - "@john test" + "@john test" assert user2.bio == expected_text end diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 8b5a16add..3d6b264b1 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -66,7 +66,7 @@ test "a create activity with a html status" do result = ActivityView.render("activity.json", activity: activity) assert result["statusnet_html"] == - "#Bike log - Commute Tuesday
https://pla.bike/posts/20181211/
#cycling #CHScycling #commute
MVIMG_20181211_054020.jpg" + "#Bike log - Commute Tuesday
https://pla.bike/posts/20181211/
#cycling #CHScycling #commute
MVIMG_20181211_054020.jpg" assert result["text"] == "#Bike log - Commute Tuesday\nhttps://pla.bike/posts/20181211/\n#cycling #CHScycling #commute\nMVIMG_20181211_054020.jpg" @@ -141,7 +141,9 @@ test "a create activity with a note" do "summary" => "", "summary_html" => "", "statusnet_html" => - "Hey @shp!", + "Hey @shp!", "tags" => [], "text" => "Hey @shp!", "uri" => activity.data["object"]["id"], From 90433b988e4d9519e0d2368452aefdb0dc565b52 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 16 Jan 2019 11:07:46 +0300 Subject: [PATCH 22/30] [#518] Fixed /api/v1/instance ("domain_count" value) and /api/v1/instance/peers responses. --- lib/pleroma/stats.ex | 3 +- .../mastodon_api_controller_test.exs | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8a030ecd0..65a6d58b5 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -34,10 +34,11 @@ def update_stats do peers = from( u in Pleroma.User, - select: fragment("distinct ?->'host'", u.info), + select: fragment("distinct split_part(?, '@', 2)", u.nickname), where: u.local != ^true ) |> Repo.all() + |> Enum.filter(& &1) domain_count = Enum.count(peers) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index fe8f845c7..62677638d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1471,20 +1471,36 @@ test "updates the user's banner", %{conn: conn} do test "get instance information", %{conn: conn} do insert(:user, %{local: true}) user = insert(:user, %{local: true}) - insert(:user, %{local: false}) + insert(:user, %{local: false, nickname: "u@peer1.com"}) + insert(:user, %{local: false, nickname: "u@peer2.com"}) {:ok, _} = TwitterAPI.create_status(user, %{"status" => "cofe"}) Pleroma.Stats.update_stats() - conn = - conn - |> get("/api/v1/instance") + conn = get(conn, "/api/v1/instance") assert result = json_response(conn, 200) - assert result["stats"]["user_count"] == 2 - assert result["stats"]["status_count"] == 1 + stats = result["stats"] + + assert stats + assert stats["user_count"] == 2 + assert stats["status_count"] == 1 + assert stats["domain_count"] == 2 + end + + test "get peers", %{conn: conn} do + insert(:user, %{local: false, nickname: "u@peer1.com"}) + insert(:user, %{local: false, nickname: "u@peer2.com"}) + + Pleroma.Stats.update_stats() + + conn = get(conn, "/api/v1/instance/peers") + + assert result = json_response(conn, 200) + + assert ["peer1.com", "peer2.com"] == Enum.sort(result) end test "put settings", %{conn: conn} do From 943324b66158d1bd2449894ec41bc544c56058a7 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 16 Jan 2019 15:13:09 +0100 Subject: [PATCH 23/30] MastoAPI: Don't break on missing users. --- lib/pleroma/user.ex | 10 +++++++ .../web/mastodon_api/views/status_view.ex | 19 ++++++++++-- .../web/twitter_api/views/activity_view.ex | 12 +------- test/web/mastodon_api/status_view_test.exs | 30 +++++++++++++++++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 681280539..a52e536d3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1040,4 +1040,14 @@ defp local_nickname_regex() do @strict_local_nickname_regex end end + + def error_user(ap_id) do + %User{ + name: ap_id, + ap_id: ap_id, + info: %User.Info{}, + nickname: "erroruser@example.com", + inserted_at: NaiveDateTime.utc_now() + } + end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index db543ffe5..7f5a52ea3 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -32,6 +32,19 @@ defp get_replied_to_activities(activities) do end) end + defp get_user(ap_id) do + cond do + user = User.get_cached_by_ap_id(ap_id) -> + user + + user = User.get_by_guessed_nickname(ap_id) -> + user + + true -> + User.error_user(ap_id) + end + end + def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) @@ -48,7 +61,7 @@ def render( "status.json", %{activity: %{data: %{"type" => "Announce", "object" => object}} = activity} = opts ) do - user = User.get_cached_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) reblogged = Activity.get_create_activity_by_object_ap_id(object) @@ -93,7 +106,7 @@ def render( end def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do - user = User.get_cached_by_ap_id(activity.data["actor"]) + user = get_user(activity.data["actor"]) like_count = object["like_count"] || 0 announcement_count = object["announcement_count"] || 0 @@ -116,7 +129,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} created_at = Utils.to_masto_date(object["published"]) reply_to = get_reply_to(activity, opts) - reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"]) + reply_to_user = reply_to && get_user(reply_to.data["actor"]) content = object diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 108e7bfc5..03708d84c 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -101,20 +101,10 @@ defp get_user(ap_id, opts) do user true -> - error_user(ap_id) + User.error_user(ap_id) end end - defp error_user(ap_id) do - %User{ - name: ap_id, - ap_id: ap_id, - info: %User.Info{}, - nickname: "erroruser@example.com", - inserted_at: NaiveDateTime.utc_now() - } - end - def render("index.json", opts) do context_ids = collect_context_ids(opts.activities) users = collect_users(opts.activities) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 1076b5002..d30ae6149 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -19,6 +19,36 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do :ok end + test "returns a temporary ap_id based user for activities missing db users" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + + Repo.delete(user) + Cachex.clear(:user_cache) + + %{account: ms_user} = StatusView.render("status.json", activity: activity) + + assert ms_user.acct == "erroruser@example.com" + end + + test "tries to get a user by nickname if fetching by ap_id doesn't work" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + + {:ok, user} = + user + |> Ecto.Changeset.change(%{ap_id: "#{user.ap_id}/extension/#{user.nickname}"}) + |> Repo.update() + + Cachex.clear(:user_cache) + + result = StatusView.render("status.json", activity: activity) + + assert result[:account][:id] == to_string(user.id) + end + test "a note with null content" do note = insert(:note_activity) From 8c368d42a20ea21d5d382838843ca1c57a86e882 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 17 Jan 2019 15:48:14 +0000 Subject: [PATCH 24/30] Make attachment links configurable Thanks @href! --- docs/config.md | 1 + lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index e3738271b..6bf7b9ea7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -95,6 +95,7 @@ config :pleroma, Pleroma.Mailer, older software for theses nicknames. * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. +* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2902905fd..7d2ac3b0f 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -103,7 +103,7 @@ def post(user, %{"status" => status} = data) do attachments, tags, get_content_type(data["content_type"]), - Enum.member?([true, "true"], data["no_attachment_links"]) + Enum.member?([true, "true"], Map.get(data, "no_attachment_links", Pleroma.Config.get([:instance, :no_attachment_links], true))) ), context <- make_context(inReplyTo), cw <- data["spoiler_text"], diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index a8fe9d708..daad89185 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -341,7 +341,6 @@ def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do params = params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) - |> Map.put("no_attachment_links", true) idempotency_key = case get_req_header(conn, "idempotency-key") do From 207489aa25c1d222b0ee2aaec976210ecfc24481 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 17 Jan 2019 15:50:34 +0000 Subject: [PATCH 25/30] Also add to default config --- config/config.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 1c55807b7..d30b0aad0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -139,7 +139,8 @@ finmoji_enabled: true, mrf_transparency: true, autofollowed_nicknames: [], - max_pinned_statuses: 1 + max_pinned_statuses: 1, + no_attachment_links: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because From 6bc9a641ba77146815f3bf6dddab751c1cce1637 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 17 Jan 2019 16:01:25 +0000 Subject: [PATCH 26/30] Default to disabled in the code in case the setting is absent from config.exs --- lib/pleroma/web/common_api/common_api.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 7d2ac3b0f..782e7da8f 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -103,7 +103,7 @@ def post(user, %{"status" => status} = data) do attachments, tags, get_content_type(data["content_type"]), - Enum.member?([true, "true"], Map.get(data, "no_attachment_links", Pleroma.Config.get([:instance, :no_attachment_links], true))) + Enum.member?([true, "true"], Map.get(data, "no_attachment_links", Pleroma.Config.get([:instance, :no_attachment_links], false))) ), context <- make_context(inReplyTo), cw <- data["spoiler_text"], From 849c83ed464e8cbf57c727da8e7b4f8e7daf8fef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 17 Jan 2019 16:10:26 +0000 Subject: [PATCH 27/30] formatting --- lib/pleroma/web/common_api/common_api.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 782e7da8f..504670439 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -103,7 +103,14 @@ def post(user, %{"status" => status} = data) do attachments, tags, get_content_type(data["content_type"]), - Enum.member?([true, "true"], Map.get(data, "no_attachment_links", Pleroma.Config.get([:instance, :no_attachment_links], false))) + Enum.member?( + [true, "true"], + Map.get( + data, + "no_attachment_links", + Pleroma.Config.get([:instance, :no_attachment_links], false) + ) + ) ), context <- make_context(inReplyTo), cw <- data["spoiler_text"], From 954dc4a4ad8a387ca7b18bb7d0ed32456491daec Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 17 Jan 2019 19:16:02 +0300 Subject: [PATCH 28/30] [#502] Fixed `user_count` in `/api/v1/instance` to include only active local users. --- lib/pleroma/stats.ex | 2 +- lib/pleroma/user.ex | 11 +++++++++-- .../web/mastodon_api/mastodon_api_controller_test.exs | 7 +++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 65a6d58b5..b3566ceb6 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -46,7 +46,7 @@ def update_stats do from(u in User.local_user_query(), select: fragment("sum((?->>'note_count')::int)", u.info)) status_count = Repo.one(status_query) - user_count = Repo.aggregate(User.local_user_query(), :count, :id) + user_count = Repo.aggregate(User.active_local_user_query(), :count, :id) Agent.update(__MODULE__, fn _ -> {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}} diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a52e536d3..c91f2d31a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -796,7 +796,7 @@ def unblock_domain(user, domain) do update_and_set_cache(cng) end - def local_user_query() do + def local_user_query do from( u in User, where: u.local == true, @@ -804,7 +804,14 @@ def local_user_query() do ) end - def moderator_user_query() do + def active_local_user_query do + from( + u in local_user_query(), + where: fragment("?->'deactivated' @> 'false'", u.info) + ) + end + + def moderator_user_query do from( u in User, where: u.local == true, diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index c83bb5bc8..dd84052a3 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1473,8 +1473,11 @@ test "updates the user's banner", %{conn: conn} do end test "get instance information", %{conn: conn} do - insert(:user, %{local: true}) user = insert(:user, %{local: true}) + + user2 = insert(:user, %{local: true}) + {:ok, _user2} = User.deactivate(user2, !user2.info.deactivated) + insert(:user, %{local: false, nickname: "u@peer1.com"}) insert(:user, %{local: false, nickname: "u@peer2.com"}) @@ -1489,7 +1492,7 @@ test "get instance information", %{conn: conn} do stats = result["stats"] assert stats - assert stats["user_count"] == 2 + assert stats["user_count"] == 1 assert stats["status_count"] == 1 assert stats["domain_count"] == 2 end From 948fba6f7663119ac4e0458aa803612ee1a2c1ca Mon Sep 17 00:00:00 2001 From: sxsdv1 Date: Thu, 17 Jan 2019 18:21:43 +0100 Subject: [PATCH 29/30] Fix bad link in likes collection Don't copy and paste, mkay --- lib/pleroma/web/activity_pub/views/object_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 193042056..394d82fbc 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -46,7 +46,7 @@ def render("likes.json", ap_id, likes) do "id" => "#{ap_id}/likes", "type" => "OrderedCollection", "totalItems" => length(likes), - "first" => collection(likes, "#{ap_id}/followers", 1) + "first" => collection(likes, "#{ap_id}/likes", 1) } |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) end From 2a818a3e77ebf276eb9123cc176a5986cc9e47ed Mon Sep 17 00:00:00 2001 From: shibayashi Date: Thu, 17 Jan 2019 19:49:54 +0100 Subject: [PATCH 30/30] Add comments and change default path of the Mix binary. --- installation/pleroma.service | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/installation/pleroma.service b/installation/pleroma.service index f1ed56cb3..72090bbc7 100644 --- a/installation/pleroma.service +++ b/installation/pleroma.service @@ -3,15 +3,23 @@ Description=Pleroma social network After=network.target postgresql.service [Service] -User=pleroma -WorkingDirectory=/home/pleroma/pleroma -Environment="HOME=/home/pleroma" -Environment="MIX_ENV=prod" -ExecStart=/usr/local/bin/mix phx.server ExecReload=/bin/kill $MAINPID KillMode=process Restart=on-failure +; Name of the user that runs the Pleroma service. +User=pleroma +; Declares that Pleroma runs in production mode. +Environment="MIX_ENV=prod" + +; Make sure that all paths fit your installation. +; Path to the home directory of the user running the Pleroma service. +Environment="HOME=/home/pleroma" +; Path to the folder containing the Pleroma installation. +WorkingDirectory=/home/pleroma/pleroma +; Path to the Mix binary. +ExecStart=/usr/bin/mix phx.server + ; Some security directives. ; Use private /tmp and /var/tmp folders inside a new file system namespace, which are discarded after the process stops. PrivateTmp=true