From 663007b42c185efce41db73854ff1376a5dae0e5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:44:04 +0000 Subject: [PATCH 01/63] activity: clean up direct use of object data --- lib/pleroma/activity.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c065f3b6c..3c7e150ee 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Activity do use Ecto.Schema - alias Pleroma.{Repo, Activity, Notification} + alias Pleroma.{Repo, Activity, Notification, Object} import Ecto.Query schema "activities" do @@ -83,9 +83,13 @@ def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) def normalize(_), do: nil - def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_activity_by_object_ap_id(ap_id) end - def get_in_reply_to_activity(_), do: nil + defp get_in_reply_to_activity_from_object(_), do: nil + + def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do + get_in_reply_to_activity_from_object(Object.normalize(object)) + end end From 0522b26883534fb93210ece32aed60fcfa4b7a5b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:48:37 +0000 Subject: [PATCH 02/63] gopher: use Object.normalize() --- lib/pleroma/gopher/server.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 3b0569a99..1ab15611c 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -37,6 +37,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.HTML + alias Pleroma.Object def start_link(ref, socket, transport, opts) do pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts]) @@ -70,14 +71,14 @@ def render_activities(activities) do |> Enum.map(fn activity -> user = User.get_cached_by_ap_id(activity.data["actor"]) - object = activity.data["object"] + object = Object.normalize(activity.data["object"]) like_count = object["like_count"] || 0 announcement_count = object["announcement_count"] || 0 link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <> info("#{like_count} likes, #{announcement_count} repeats") <> "i\tfake\t(NULL)\t0\r\n" <> - info(HTML.strip_tags(String.replace(activity.data["object"]["content"], "
", "\r"))) + info(HTML.strip_tags(String.replace(object["content"], "
", "\r"))) end) |> Enum.join("i\tfake\t(NULL)\t0\r\n") end From d6b266163b9161bf28df7919ffd1391c81b142e3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 18:57:38 +0000 Subject: [PATCH 03/63] common api: fetch visibility from normalized object --- lib/pleroma/web/common_api/common_api.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e3385310f..e0c9dedd5 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -64,7 +64,10 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat "public" inReplyTo -> - Pleroma.Web.MastodonAPI.StatusView.get_visibility(inReplyTo.data["object"]) + # XXX: these heuristics should be moved out of MastodonAPI. + with %Object{} = object <- Object.normalize(inReplyTo.data["object"]) do + Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data) + end end end From 67038ae15e1f4a0b136388416af23547d82cbd0d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 19:00:05 +0000 Subject: [PATCH 04/63] common api: utils: access inReplyTo object ID correctly --- lib/pleroma/web/common_api/utils.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 728f24c7e..ec66452c2 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -177,8 +177,10 @@ def make_note_data( } if inReplyTo do + inReplyToObject = Object.normalize(inReplyTo.data["object"]) + object - |> Map.put("inReplyTo", inReplyTo.data["object"]["id"]) + |> Map.put("inReplyTo", inReplyToObject.data["id"]) |> Map.put("inReplyToStatusId", inReplyTo.id) else object From d3fde9b5f28b802b9150e1f8a763f4a67b42aadb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 19:05:17 +0000 Subject: [PATCH 05/63] ostatus: note handler: appropriately use Object.normalize() --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 0d4080291..39004367a 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,8 @@ def handle_note(entry, doc \\ nil) do cw <- OStatus.get_cw(entry), inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo), - inReplyTo <- (inReplyToActivity && inReplyToActivity.data["object"]["id"]) || inReplyTo, + inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, + inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo, attachments <- OStatus.get_attachments(entry), context <- get_context(entry, inReplyTo), tags <- OStatus.get_tags(entry), From 4482ce7e2dbd23e3b7948ee8d084c29d84ae5a1e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:09:28 +0000 Subject: [PATCH 06/63] activitypub: normalize objects when streaming them out --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7e207c620..b11d3221f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -81,6 +81,8 @@ def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce"] do + object = Object.normalize(activity.data["object"]) + Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -91,12 +93,12 @@ def stream_out(activity) do Pleroma.Web.Streamer.stream("public:local", activity) end - activity.data["object"] + object.data |> Map.get("tag", []) |> Enum.filter(fn tag -> is_bitstring(tag) end) |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - if activity.data["object"]["attachment"] != [] do + if object.data["attachment"] != [] do Pleroma.Web.Streamer.stream("public:media", activity) if activity.local do From 95a458f3925fec4ae31a59face1bc130abdcde28 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:14:00 +0000 Subject: [PATCH 07/63] twitterapi: more object normalization work --- .../representers/activity_representer.ex | 34 ++++++++++--------- .../web/twitter_api/views/activity_view.ex | 6 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index fbd33f07e..4b4e202b3 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter - alias Pleroma.{Activity, User} + alias Pleroma.{Activity, User, Object} alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView} alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Formatter @@ -144,11 +144,13 @@ def to_map( %Activity{data: %{"object" => %{"content" => content} = object}} = activity, %{user: user} = opts ) do - created_at = object["published"] |> Utils.date_to_asctime() - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) + object = Object.normalize(object) + + created_at = object.data["published"] |> Utils.date_to_asctime() + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) mentions = opts[:mentioned] || [] @@ -160,8 +162,8 @@ def to_map( conversation_id = conversation_id(activity) - tags = activity.data["object"]["tag"] || [] - possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags @@ -169,16 +171,16 @@ def to_map( html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) - |> Formatter.emojify(object["emoji"]) + |> Formatter.emojify(object.data["emoji"]) video = if object["type"] == "Video" do - vid = [object] + vid = [object.data] else [] end - attachments = (object["attachment"] || []) ++ video + attachments = (object.data["attachment"] || []) ++ video reply_parent = Activity.get_in_reply_to_activity(activity) @@ -186,14 +188,14 @@ def to_map( %{ "id" => activity.id, - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, "text" => HTML.strip_tags(content), "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, - "in_reply_to_status_id" => object["inReplyToStatusId"], + "in_reply_to_status_id" => object.data["inReplyToStatusId"], "in_reply_to_screen_name" => reply_user && reply_user.nickname, "in_reply_to_profileurl" => User.profile_url(reply_user), "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id, @@ -205,12 +207,12 @@ def to_map( "repeat_num" => announcement_count, "favorited" => to_boolean(favorited), "repeated" => to_boolean(repeated), - "external_url" => object["external_url"] || object["id"], + "external_url" => object.data["external_url"] || object.data["id"], "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, - "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object), - "summary" => object["summary"] + "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data), + "summary" => object.data["summary"] } end diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 83e8fb765..7839fe878 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -225,8 +225,8 @@ def render( conversation_id = get_context_id(activity, opts) - tags = activity.data["object"]["tag"] || [] - possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags @@ -242,7 +242,7 @@ def render( %{ "id" => activity.id, - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, "text" => HTML.strip_tags(content), From e9b718cea2ee21a38f6c2137f32e3da6f5893cc2 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:21:42 +0000 Subject: [PATCH 08/63] mastodon api: status view: use Object.normalize() --- .../web/mastodon_api/views/status_view.ex | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2d9a915f0..3b4911d53 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Web.MastodonAPI.{AccountView, StatusView} - alias Pleroma.{User, Activity} + alias Pleroma.{User, Activity, Object} alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MediaProxy alias Pleroma.Repo @@ -11,8 +11,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_replied_to_activities(activities) do activities |> Enum.map(fn - %{data: %{"type" => "Create", "object" => %{"inReplyTo" => inReplyTo}}} -> - inReplyTo != "" && inReplyTo + %{data: %{"type" => "Create", "object" => object}} -> + object = Object.normalize(object) + object.data["inReplyTo"] != "" && object.data["inReplyTo"] _ -> nil @@ -21,7 +22,8 @@ defp get_replied_to_activities(activities) do |> Activity.create_activity_by_object_id_query() |> Repo.all() |> Enum.reduce(%{}, fn activity, acc -> - Map.put(acc, activity.data["object"]["id"], activity) + object = Object.normalize(activity.data["object"]) + Map.put(acc, object.data["id"], activity) end) end @@ -85,13 +87,15 @@ def render( end def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do + object = Object.normalize(object) + user = User.get_cached_by_ap_id(activity.data["actor"]) - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 - tags = object["tag"] || [] - sensitive = object["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") mentions = activity.recipients @@ -99,20 +103,20 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - attachment_data = object["attachment"] || [] - attachment_data = attachment_data ++ if object["type"] == "Video", do: [object], else: [] + attachment_data = object.data["attachment"] || [] + attachment_data = attachment_data ++ if object.data["type"] == "Video", do: [object], else: [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) - created_at = Utils.to_masto_date(object["published"]) + created_at = Utils.to_masto_date(object.data["published"]) reply_to = get_reply_to(activity, opts) reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"]) emojis = - (activity.data["object"]["emoji"] || []) + (object.data["emoji"] || []) |> Enum.map(fn {name, url} -> name = HTML.strip_tags(name) @@ -124,13 +128,13 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} end) content = - render_content(object) + render_content(object.data) |> HTML.filter_tags(User.html_filter_policy(opts[:for])) %{ id: to_string(activity.id), - uri: object["id"], - url: object["external_url"] || object["id"], + uri: object.data["id"], + url: object.data["external_url"] || object["id"], account: AccountView.render("account.json", %{user: user}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), @@ -144,7 +148,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} favourited: !!favorited, muted: false, sensitive: sensitive, - spoiler_text: object["summary"] || "", + spoiler_text: object.data["summary"] || "", visibility: get_visibility(object), media_attachments: attachments |> Enum.take(4), mentions: mentions, @@ -190,13 +194,15 @@ def render("attachment.json", %{attachment: attachment}) do end def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do - _id = activity.data["object"]["inReplyTo"] - replied_to_activities[activity.data["object"]["inReplyTo"]] + object = Object.normalize(activity.data["object"]) + replied_to_activities[object.data["inReplyTo"]] end def get_reply_to(%{data: %{"object" => object}}, _) do - if object["inReplyTo"] && object["inReplyTo"] != "" do - Activity.get_create_activity_by_object_ap_id(object["inReplyTo"]) + object = Object.normalize(object) + + if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do + Activity.get_create_activity_by_object_ap_id(object.data["inReplyTo"]) else nil end From 5d4a71906a15d4d6bff714ef0bb63fba1c5f4fb3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:27:00 +0000 Subject: [PATCH 09/63] mastodon api: use Object.normalize() in a few missing spots --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 543fdf416..90225460b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -464,7 +464,8 @@ def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do end def favourited_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"likes" => likes}}} <- Repo.get(Activity, id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) users = Repo.all(q) render(conn, AccountView, "accounts.json", %{users: users, as: :user}) @@ -474,7 +475,8 @@ def favourited_by(conn, %{"id" => id}) do end def reblogged_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Repo.get(Activity, id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) users = Repo.all(q) render(conn, AccountView, "accounts.json", %{users: users, as: :user}) From dd66cc2ca6c99d5b1586133dd85d8d36d5cbaaa7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:40:16 +0000 Subject: [PATCH 10/63] ostatus: use Object.normalize() where appropriate when representing activities --- .../web/ostatus/activity_representer.ex | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 537bd9f77..ceb4d79db 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -84,11 +84,13 @@ def to_simple_form(activity, user, with_author \\ false) def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do h = fn str -> [to_charlist(str)] end - updated_at = activity.data["object"]["published"] - inserted_at = activity.data["object"]["published"] + object = Object.normalize(activity.data["object"]) + + updated_at = object.data["published"] + inserted_at = object.data["published"] attachments = - Enum.map(activity.data["object"]["attachment"] || [], fn attachment -> + Enum.map(object.data["attachment"] || [], fn attachment -> url = hd(attachment["url"]) {:link, @@ -101,7 +103,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, mentions = activity.recipients |> get_mentions categories = - (activity.data["object"]["tag"] || []) + (object.data["tag"] || []) |> Enum.map(fn tag -> if is_binary(tag) do {:category, [term: to_charlist(tag)], []} @@ -111,11 +113,11 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, end) |> Enum.filter(& &1) - emoji_links = get_emoji_links(activity.data["object"]["emoji"] || %{}) + emoji_links = get_emoji_links(object.data["emoji"] || %{}) summary = - if activity.data["object"]["summary"] do - [{:summary, [], h.(activity.data["object"]["summary"])}] + if object.data["summary"] do + [{:summary, [], h.(object.data["summary"])}] else [] end @@ -124,10 +126,9 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, # For notes, federate the object id. - {:id, h.(activity.data["object"]["id"])}, + {:id, h.(object.data["id"])}, {:title, ['New note by #{user.nickname}']}, - {:content, [type: 'html'], - h.(activity.data["object"]["content"] |> String.replace(~r/[\n\r]/, ""))}, + {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], From 3b8e5bcbeb03f3cfa5de1c4e7a4f3a04871094d1 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 20:56:12 +0000 Subject: [PATCH 11/63] fix most tests --- lib/pleroma/web/mastodon_api/views/status_view.ex | 4 ++-- .../web/twitter_api/representers/activity_representer.ex | 4 ++-- lib/pleroma/web/twitter_api/views/activity_view.ex | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3b4911d53..31f4675c3 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -134,7 +134,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} %{ id: to_string(activity.id), uri: object.data["id"], - url: object.data["external_url"] || object["id"], + url: object.data["external_url"] || object.data["id"], account: AccountView.render("account.json", %{user: user}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), @@ -149,7 +149,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} muted: false, sensitive: sensitive, spoiler_text: object.data["summary"] || "", - visibility: get_visibility(object), + visibility: get_visibility(object.data), media_attachments: attachments |> Enum.take(4), mentions: mentions, # fix, diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index 4b4e202b3..436f9bf92 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -167,14 +167,14 @@ def to_map( tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags - {summary, content} = ActivityView.render_content(object) + {summary, content} = ActivityView.render_content(object.data) html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) |> Formatter.emojify(object.data["emoji"]) video = - if object["type"] == "Video" do + if object.data["type"] == "Video" do vid = [object.data] else [] diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 7839fe878..f202b6e97 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -265,7 +265,7 @@ def render( "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, - "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object), + "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data), "summary" => summary } end From 6f5f589f73da05ee09eef61b6a76a2ef611efa7c Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:08:55 +0000 Subject: [PATCH 12/63] test: fix mastodon api test failure --- test/web/mastodon_api/status_view_test.exs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 31554a07d..4f58ce8af 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -2,21 +2,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase alias Pleroma.Web.MastodonAPI.{StatusView, AccountView} - alias Pleroma.User + alias Pleroma.{Repo, User, Object} alias Pleroma.Web.OStatus alias Pleroma.Web.CommonAPI import Pleroma.Factory test "a note with null content" do note = insert(:note_activity) + note_object = Object.normalize(note.data["object"]) data = - note.data - |> put_in(["object", "content"], nil) + note_object.data + |> Map.put("content", nil) - note = - note - |> Map.put(:data, data) + Object.change(note_object, %{data: data}) + |> Repo.update() user = User.get_cached_by_ap_id(note.data["actor"]) From e8570758f90f0ef040eab011d0584c59379ba743 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:16:44 +0000 Subject: [PATCH 13/63] twitterapi: fix remaining test failures --- .../representers/activity_representer.ex | 2 +- .../activity_representer_test.exs | 41 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index 436f9bf92..8f91aeaf0 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -141,7 +141,7 @@ def to_map( end def to_map( - %Activity{data: %{"object" => %{"content" => content} = object}} = activity, + %Activity{data: %{"object" => object}} = activity, %{user: user} = opts ) do object = Object.normalize(object) diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index 7cae4e4a1..314f2b51f 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -87,6 +87,26 @@ test "an activity" do {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert() + note_object = %{ + "id" => "https://example.com/id/1", + "published" => date, + "type" => "Note", + "content" => content_html, + "summary" => "2hu", + "inReplyToStatusId" => 213_123, + "attachment" => [object.data], + "external_url" => "some url", + "like_count" => 5, + "announcement_count" => 3, + "context" => "2hu", + "tag" => ["content", "mentioning", "nsfw"], + "emoji" => %{ + "2hu" => "corndog.png" + } + } + + Object.create(note_object) + to = [ User.ap_followers(user), "https://www.w3.org/ns/activitystreams#Public", @@ -100,24 +120,7 @@ test "an activity" do "id" => "id", "to" => to, "actor" => User.ap_id(user), - "object" => %{ - "published" => date, - "type" => "Note", - "content" => content_html, - "summary" => "2hu", - "inReplyToStatusId" => 213_123, - "attachment" => [ - object - ], - "external_url" => "some url", - "like_count" => 5, - "announcement_count" => 3, - "context" => "2hu", - "tag" => ["content", "mentioning", "nsfw"], - "emoji" => %{ - "2hu" => "corndog.png" - } - }, + "object" => note_object["id"], "published" => date, "context" => "2hu" }, @@ -158,7 +161,7 @@ test "an activity" do "tags" => ["nsfw", "content", "mentioning"], "activity_type" => "post", "possibly_sensitive" => true, - "uri" => activity.data["object"]["id"], + "uri" => note_object["id"], "visibility" => "direct", "summary" => "2hu" } From 5ba5df1321b8c177419ae5342d37bc524f8f6656 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:20:29 +0000 Subject: [PATCH 14/63] object: normalize(): use object cache --- lib/pleroma/object.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 03a75dfbd..57a8b1d6b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -27,8 +27,8 @@ def get_by_ap_id(ap_id) do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: normalize(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) def normalize(_), do: nil if Mix.env() == :test do From 5ea64f4bf958d9801d89d1165fa20d0e17bf0570 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:34:36 +0000 Subject: [PATCH 15/63] activity: minor cleanups to normalization functions to align them with the object normalizers --- lib/pleroma/activity.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3c7e150ee..34d0a34b8 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -79,8 +79,8 @@ def get_create_activity_by_object_ap_id(ap_id) when is_binary(ap_id) do def get_create_activity_by_object_ap_id(_), do: nil - def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: normalize(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id(ap_id) def normalize(_), do: nil defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do From 57d90e7afe46a54cc7e1ef14a9b76b0650ed3db6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:43:26 +0000 Subject: [PATCH 16/63] activitypub: relay: fix improper use of Object.normalize() --- lib/pleroma/web/activity_pub/relay.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index fcdc6b1c0..a48a91ef7 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -35,7 +35,7 @@ def unfollow(target_instance) do def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), - %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do + %Object{} = object <- Object.normalize(activity.data["object"]) do ActivityPub.announce(user, object) else e -> Logger.error("error: #{inspect(e)}") From d13d953385b7659d7a9eb381ac9303a41e6fc294 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:44:03 +0000 Subject: [PATCH 17/63] activitypub: implement activity flattening --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b11d3221f..dcf670afb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -57,7 +57,7 @@ def insert(map, local \\ true) when is_map(map) do map <- lazy_put_activity_defaults(map), :ok <- check_actor_is_active(map["actor"]), {:ok, map} <- MRF.filter(map), - :ok <- insert_full_object(map) do + {:ok, map} <- insert_full_object(map) do {recipients, _, _} = get_recipients(map) {:ok, activity} = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 549148989..bc5b98f1a 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -180,14 +180,18 @@ def lazy_put_object_defaults(map, activity \\ %{}) do @doc """ Inserts a full object if it is contained in an activity. """ - def insert_full_object(%{"object" => %{"type" => type} = object_data}) + def insert_full_object(%{"object" => %{"type" => type} = object_data} = map) when is_map(object_data) and type in @supported_object_types do - with {:ok, _} <- Object.create(object_data) do - :ok + with {:ok, object} <- Object.create(object_data) do + map = + map + |> Map.put("object", object.data["id"]) + + {:ok, map} end end - def insert_full_object(_), do: :ok + def insert_full_object(map), do: {:ok, map} def update_object_in_activities(%{data: %{"id" => id}} = object) do # TODO From d6e65f9304c1598087a6dacc640bcd0bb4057009 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 21:50:46 +0000 Subject: [PATCH 18/63] common api: fix up improper Object.normalize() calls --- lib/pleroma/web/common_api/common_api.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e0c9dedd5..c83f8a6a9 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do import Pleroma.Web.CommonAPI.Utils def delete(activity_id, user) do - with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id), + with %Activity{data: %{"object" => object_id}} <- Repo.get(Activity, activity_id), %Object{} = object <- Object.normalize(object_id), true <- user.info.is_moderator || user.ap_id == object.data["actor"], {:ok, delete} <- ActivityPub.delete(object) do @@ -16,7 +16,7 @@ def delete(activity_id, user) do def repeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.announce(user, object) else _ -> @@ -26,7 +26,7 @@ def repeat(id_or_ap_id, user) do def unrepeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.unannounce(user, object) else _ -> @@ -36,7 +36,7 @@ def unrepeat(id_or_ap_id, user) do def favorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.like(user, object) else _ -> @@ -46,7 +46,7 @@ def favorite(id_or_ap_id, user) do def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity.data["object"]) do ActivityPub.unlike(user, object) else _ -> From cf139e3eec9bf978cfff24044dce5b5c388fa303 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:11:10 +0000 Subject: [PATCH 19/63] activitypub: transmogrifier: ensure we send nested object in Create --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 17b063609..5e3d40d9f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -616,9 +616,9 @@ def prepare_object(object) do # internal -> Mastodon # """ - def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do + def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do object = - object + Object.normalize(object_id).data |> prepare_object data = From 4ca4c83871a6026ba0e4eb148e99b352edbadb5b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:31:07 +0000 Subject: [PATCH 20/63] tests: fix most remaining failures --- test/web/activity_pub/activity_pub_test.exs | 6 +- test/web/activity_pub/transmogrifier_test.exs | 49 ++++++----- test/web/common_api/common_api_test.exs | 18 ++-- test/web/ostatus/ostatus_test.exs | 88 +++++++++++-------- 4 files changed, 95 insertions(+), 66 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 1d561d38d..afdf5c06a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -77,8 +77,10 @@ test "adds an id to a given object if it lacks one and is a note and inserts it } {:ok, %Activity{} = activity} = ActivityPub.insert(data) - assert is_binary(activity.data["object"]["id"]) - assert %Object{} = Object.get_by_ap_id(activity.data["object"]["id"]) + object = Object.normalize(activity.data["object"]) + + assert is_binary(object.data["id"]) + assert %Object{} = Object.get_by_ap_id(activity.data["object"]) end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e74b8f9a1..ef89752f5 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.OStatus - alias Pleroma.Activity + alias Pleroma.{Activity, Object} alias Pleroma.User alias Pleroma.Repo alias Pleroma.Web.Websub.WebsubClientSubscription @@ -40,16 +40,16 @@ test "it fetches replied-to activities if we don't have them" do |> Map.put("object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + returned_object = Object.normalize(returned_activity.data["object"]) assert activity = Activity.get_create_activity_by_object_ap_id( "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" ) - assert returned_activity.data["object"]["inReplyToAtomUri"] == - "https://shitposter.club/notice/2827873" + assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" - assert returned_activity.data["object"]["inReplyToStatusId"] == activity.id + assert returned_object.data["inReplyToStatusId"] == activity.id end test "it works for incoming notices" do @@ -72,7 +72,7 @@ test "it works for incoming notices" do assert data["actor"] == "http://mastodon.example.org/users/admin" - object = data["object"] + object = Object.normalize(data["object"]).data assert object["id"] == "http://mastodon.example.org/users/admin/statuses/99512778738411822" assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"] @@ -99,7 +99,9 @@ test "it works for incoming notices with hashtags" do data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - assert Enum.at(data["object"]["tag"], 2) == "moo" + object = Object.normalize(data["object"]) + + assert Enum.at(object.data["tag"], 2) == "moo" end test "it works for incoming notices with contentMap" do @@ -107,8 +109,9 @@ test "it works for incoming notices with contentMap" do File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "

@lain

" end @@ -116,8 +119,9 @@ test "it works for incoming notices with to/cc not being an array (kroeg)" do data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "

henlo from my Psion netBook

message sent from my Psion netBook

" end @@ -133,24 +137,27 @@ test "it works for incoming notices with tag not being an array (kroeg)" do data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["emoji"] == %{ + assert object.data["emoji"] == %{ "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png" } data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert "test" in data["object"]["tag"] + assert "test" in object.data["tag"] end test "it works for incoming notices with url not being a string (prismo)" do data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["url"] == "https://prismo.news/posts/83" + assert object.data["url"] == "https://prismo.news/posts/83" end test "it works for incoming follow requests" do @@ -193,14 +200,14 @@ test "it works for incoming likes" do data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) assert data["actor"] == "http://mastodon.example.org/users/admin" assert data["type"] == "Like" assert data["id"] == "http://mastodon.example.org/users/admin#likes/2" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] end test "it returns an error for incoming unlikes wihout a like activity" do @@ -210,7 +217,7 @@ test "it returns an error for incoming unlikes wihout a like activity" do data = File.read!("test/fixtures/mastodon-undo-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) assert Transmogrifier.handle_incoming(data) == :error end @@ -222,7 +229,7 @@ test "it works for incoming unlikes with an existing like activity" do like_data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: like_data, local: false}} = Transmogrifier.handle_incoming(like_data) @@ -264,7 +271,7 @@ test "it works for incoming announces with an existing activity" do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) @@ -274,7 +281,7 @@ test "it works for incoming announces with an existing activity" do assert data["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] assert Activity.get_create_activity_by_object_ap_id(data["object"]).id == activity.id end @@ -349,7 +356,7 @@ test "it works for incoming deletes" do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -370,7 +377,7 @@ test "it fails for incoming deletes with spoofed origin" do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -388,7 +395,7 @@ test "it works for incoming unannounces with an existing notice" do announce_data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: announce_data, local: false}} = Transmogrifier.handle_incoming(announce_data) @@ -403,7 +410,7 @@ test "it works for incoming unannounces with an existing notice" do assert data["type"] == "Undo" assert data["object"]["type"] == "Announce" - assert data["object"]["object"] == activity.data["object"]["id"] + assert data["object"]["object"] == activity.data["object"] assert data["object"]["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8fc65f4c0..3f81d952c 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -1,7 +1,7 @@ defmodule Pleroma.Web.CommonAPI.Test do use Pleroma.DataCase alias Pleroma.Web.CommonAPI - alias Pleroma.User + alias Pleroma.{User, Object} import Pleroma.Factory @@ -9,7 +9,9 @@ test "it de-duplicates tags" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"}) - assert activity.data["object"]["tag"] == ["2hu"] + object = Object.normalize(activity.data["object"]) + + assert object.data["tag"] == ["2hu"] end test "it adds emoji when updating profiles" do @@ -34,8 +36,10 @@ test "it filters out obviously bad tags when accepting a post as HTML" do "content_type" => "text/html" }) - content = activity.data["object"]["content"] - assert content == "

2hu

alert('xss')" + object = + Object.normalize(activity.data["object"]) + + assert object.data["content"] == "

2hu

alert('xss')" end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -49,8 +53,10 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do "content_type" => "text/markdown" }) - content = activity.data["object"]["content"] - assert content == "

2hu

alert('xss')" + object = + Object.normalize(activity.data["object"]) + + assert object.data["content"] == "

2hu

alert('xss')" end end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 32bf6691b..73e2b5482 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -15,34 +15,36 @@ test "don't insert create notes twice" do test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) user = User.get_by_ap_id(activity.data["actor"]) assert user.info.note_count == 1 assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["id"] == + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" - assert activity.data["object"]["published"] == "2017-04-23T14:51:03+00:00" + assert object.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] - assert activity.data["object"]["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} + assert object.data["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} assert activity.local == false end test "handle incoming notes - GS, subscription" do incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == "Will it blend?" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["content"] == "Will it blend?" user = User.get_cached_by_ap_id(activity.data["actor"]) assert User.ap_followers(user) in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -51,20 +53,22 @@ test "handle incoming notes - GS, subscription" do test "handle incoming notes with attachments - GS, subscription" do incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["attachment"] |> length == 2 - assert activity.data["object"]["external_url"] == "https://social.heldscal.la/notice/2020923" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["attachment"] |> length == 2 + assert object.data["external_url"] == "https://social.heldscal.la/notice/2020923" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming notes with tags" do incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) - assert activity.data["object"]["tag"] == ["nsfw"] + assert object.data["tag"] == ["nsfw"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -79,10 +83,11 @@ test "handle incoming notes - Mastodon, salmon, reply" do incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" assert activity.data["context"] == "2hu" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -90,42 +95,47 @@ test "handle incoming notes - Mastodon, salmon, reply" do test "handle incoming notes - Mastodon, with CW" do incoming = File.read!("test/fixtures/mastodon-note-cw.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"]["summary"] == "technologic" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["summary"] == "technologic" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming unlisted messages, put public into cc" do incoming = File.read!("test/fixtures/mastodon-note-unlisted.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) + refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["cc"] - refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in object.data["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in object.data["cc"] end test "handle incoming retweets - Mastodon, with CW" do incoming = File.read!("test/fixtures/cw_retweet.xml") {:ok, [[_activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) - assert retweeted_activity.data["object"]["summary"] == "Hey." + assert retweeted_object.data["summary"] == "Hey." end test "handle incoming notes - GS, subscription, reply" do incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == + assert object.data["content"] == "@shpbot why not indeed." - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -137,17 +147,19 @@ test "handle incoming retweets - GS, subscription" do assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"] refute activity.local retweeted_activity = Repo.get(Activity, retweeted_activity.id) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) + assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 - assert String.contains?(retweeted_activity.data["object"]["content"], "mastodon") - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + assert retweeted_object.data["announcement_count"] == 1 + assert String.contains?(retweeted_object.data["content"], "mastodon") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming retweets - GS, subscription - local message" do @@ -179,10 +191,11 @@ test "handle incoming retweets - GS, subscription - local message" do test "handle incoming retweets - Mastodon, salmon" do incoming = File.read!("test/fixtures/share.xml") {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert activity.data["id"] == "tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status" @@ -191,7 +204,7 @@ test "handle incoming retweets - Mastodon, salmon" do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming favorites - GS, websub" do @@ -201,7 +214,7 @@ test "handle incoming favorites - GS, websub" do assert activity.data["type"] == "Like" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == favorited_activity.data["object"]["id"] + assert activity.data["object"] == favorited_activity.data["object"] assert activity.data["id"] == "tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00" @@ -210,7 +223,7 @@ test "handle incoming favorites - GS, websub" do assert favorited_activity.data["type"] == "Create" assert favorited_activity.data["actor"] == "https://shitposter.club/user/1" - assert favorited_activity.data["object"]["id"] == + assert favorited_activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" refute favorited_activity.local @@ -245,16 +258,17 @@ test "handle incoming favorites with locally available object - GS, websub" do test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - assert activity.data["object"]["id"] == + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -466,7 +480,7 @@ test "it builds a missing status from an html url" do assert activity.data["actor"] == "https://shitposter.club/user/1" - assert activity.data["object"]["id"] == + assert activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" end) end @@ -475,7 +489,7 @@ test "it works for atom notes, too" do url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" {:ok, [activity]} = OStatus.fetch_activity_from_url(url) assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal" - assert activity.data["object"]["id"] == url + assert activity.data["object"] == url end end From f7e15d3257c0428ba9d37bdbbd45aa5fe6829a1b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 25 Nov 2018 22:49:39 +0000 Subject: [PATCH 21/63] tests: fix a lot of the remaining test failures --- test/web/common_api/common_api_test.exs | 6 ++--- .../mastodon_api_controller_test.exs | 5 ++-- test/web/ostatus/ostatus_test.exs | 6 ++--- test/web/twitter_api/twitter_api_test.exs | 26 ++++++++++--------- .../twitter_api/views/activity_view_test.exs | 18 +++++++------ 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 3f81d952c..3dc5f6f84 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -36,8 +36,7 @@ test "it filters out obviously bad tags when accepting a post as HTML" do "content_type" => "text/html" }) - object = - Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"]) assert object.data["content"] == "

2hu

alert('xss')" end @@ -53,8 +52,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do "content_type" => "text/markdown" }) - object = - Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"]) assert object.data["content"] == "

2hu

alert('xss')" end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 098acb59f..c30f253d9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Web.TwitterAPI.TwitterAPI - alias Pleroma.{Repo, User, Activity, Notification} + alias Pleroma.{Repo, User, Activity, Notification, Object} alias Pleroma.Web.{OStatus, CommonAPI} alias Pleroma.Web.ActivityPub.ActivityPub @@ -219,9 +219,10 @@ test "replying to a status", %{conn: conn} do assert %{"content" => "xD", "id" => id} = json_response(conn, 200) activity = Repo.get(Activity, id) + object = Object.normalize(activity.data["object"]) assert activity.data["context"] == replied_to.data["context"] - assert activity.data["object"]["inReplyToStatusId"] == replied_to.id + assert object.data["inReplyToStatusId"] == replied_to.id end test "posting a status with an invalid in_reply_to_id", %{conn: conn} do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 73e2b5482..b5805c668 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -22,8 +22,7 @@ test "handle incoming note - GS, Salmon" do assert activity.data["type"] == "Create" assert object.data["type"] == "Note" - assert object.data["id"] == - "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert object.data["published"] == "2017-04-23T14:51:03+00:00" @@ -268,8 +267,7 @@ test "handle incoming replies" do assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - assert object.data["id"] == - "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 28230699f..bc53fe68a 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -33,13 +33,14 @@ test "create a status" do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) expected_text = "Hello again, @shp.<script></script>
This is on another :moominmamma: line.
image.jpg" - assert get_in(activity.data, ["object", "content"]) == expected_text - assert get_in(activity.data, ["object", "type"]) == "Note" - assert get_in(activity.data, ["object", "actor"]) == user.ap_id + assert get_in(object.data, ["content"]) == expected_text + assert get_in(object.data, ["type"]) == "Note" + assert get_in(object.data, ["actor"]) == user.ap_id assert get_in(activity.data, ["actor"]) == user.ap_id assert Enum.member?(get_in(activity.data, ["cc"]), User.ap_followers(user)) @@ -52,18 +53,18 @@ test "create a status" do assert activity.local == true assert %{"moominmamma" => "http://localhost:4001/finmoji/128px/moominmamma-128.png"} = - activity.data["object"]["emoji"] + object.data["emoji"] # hashtags - assert activity.data["object"]["tag"] == ["2hu", "epic", "phantasmagoric"] + assert object.data["tag"] == ["2hu", "epic", "phantasmagoric"] # Add a context assert is_binary(get_in(activity.data, ["context"])) - assert is_binary(get_in(activity.data, ["object", "context"])) + assert is_binary(get_in(object.data, ["context"])) - assert is_list(activity.data["object"]["attachment"]) + assert is_list(object.data["attachment"]) - assert activity.data["object"] == Object.get_by_ap_id(activity.data["object"]["id"]).data + assert activity.data["object"] == object.data["id"] user = User.get_by_ap_id(user.ap_id) @@ -78,6 +79,7 @@ test "create a status that is a reply" do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) input = %{ "status" => "Here's your (you).", @@ -85,14 +87,14 @@ test "create a status that is a reply" do } {:ok, reply = %Activity{}} = TwitterAPI.create_status(user, input) + reply_object = Object.normalize(reply.data["object"]) assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"]) - assert get_in(reply.data, ["object", "context"]) == - get_in(activity.data, ["object", "context"]) + assert get_in(reply_object.data, ["context"]) == get_in(object.data, ["context"]) - assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"]) - assert get_in(reply.data, ["object", "inReplyToStatusId"]) == activity.id + assert get_in(reply_object.data, ["inReplyTo"]) == get_in(activity.data, ["object"]) + assert get_in(reply_object.data, ["inReplyToStatusId"]) == activity.id end test "Follow another user using user_id" do diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 5cef06f88..f4741cf24 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do alias Pleroma.Web.TwitterAPI.UserView alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Repo - alias Pleroma.Activity + alias Pleroma.{Activity, Object} alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -19,10 +19,11 @@ test "a create activity with a note" do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + object = Object.normalize(activity.data["object"]) result = ActivityView.render("activity.json", activity: activity) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) expected = %{ "activity_type" => "post", @@ -30,8 +31,8 @@ test "a create activity with a note" do "attentions" => [ UserView.render("show.json", %{user: other_user}) ], - "created_at" => activity.data["object"]["published"] |> Utils.date_to_asctime(), - "external_url" => activity.data["object"]["id"], + "created_at" => object.data["published"] |> Utils.date_to_asctime(), + "external_url" => object.data["id"], "fave_num" => 0, "favorited" => false, "id" => activity.id, @@ -50,7 +51,7 @@ test "a create activity with a note" do "Hey @shp!", "tags" => [], "text" => "Hey @shp!", - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user}), "visibility" => "direct", "summary" => nil @@ -63,8 +64,9 @@ test "a list of activities" do user = insert(:user) other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) + object = Object.normalize(activity.data["object"]) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) mocks = [ { @@ -162,9 +164,9 @@ test "an announce activity" do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) - {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user) + {:ok, announce, object} = CommonAPI.repeat(activity.id, other_user) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = TwitterAPI.context_to_conversation_id(object.data["context"]) activity = Repo.get(Activity, activity.id) From f168a2add66a312aa9911c880806534899f3fe08 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:06:20 +0000 Subject: [PATCH 22/63] ostatus: fix representing external objects --- .../web/ostatus/activity_representer.ex | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index ceb4d79db..fefd9459a 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -47,23 +47,16 @@ defp get_mentions(to) do end) end - defp get_links(%{local: true, data: data}) do + defp get_links(%{local: true}, %{"id" => object_id}) do h = fn str -> [to_charlist(str)] end [ - {:link, [type: ['application/atom+xml'], href: h.(data["object"]["id"]), rel: 'self'], []}, - {:link, [type: ['text/html'], href: h.(data["object"]["id"]), rel: 'alternate'], []} + {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []}, + {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []} ] end - defp get_links(%{ - local: false, - data: %{ - "object" => %{ - "external_url" => external_url - } - } - }) do + defp get_links(%{local: false}, %{"external_url" => external_url}) do h = fn str -> [to_charlist(str)] end [ @@ -71,7 +64,7 @@ defp get_links(%{ ] end - defp get_links(_activity), do: [] + defp get_links(_activity, _object_data), do: [] defp get_emoji_links(emojis) do Enum.map(emojis, fn {emoji, file} -> @@ -81,7 +74,7 @@ defp get_emoji_links(emojis) do def to_simple_form(activity, user, with_author \\ false) - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do + def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do h = fn str -> [to_charlist(str)] end object = Object.normalize(activity.data["object"]) @@ -136,7 +129,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} ] ++ summary ++ - get_links(activity) ++ + get_links(activity, object.data) ++ categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links end From fe000f82f8e1a37a063c7260033f6c3fd275c8cc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:27:59 +0000 Subject: [PATCH 23/63] tests: activitypub: fix broken test due to invalid child object --- test/web/activity_pub/activity_pub_test.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index afdf5c06a..231a334f9 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -93,7 +93,11 @@ test "removes doubled 'to' recipients" do to: ["user1", "user1", "user2"], actor: user, context: "", - object: %{} + object: %{ + "to" => ["user1", "user1", "user2"], + "type" => "Note", + "content" => "testing" + } }) assert activity.data["to"] == ["user1", "user2"] From 6f90f2c3ac70c74b9d06debb09530d5f479b5a8c Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 03:34:57 +0000 Subject: [PATCH 24/63] activitypub: rework thread filtering for split object view --- lib/pleroma/web/activity_pub/activity_pub.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index dcf670afb..34a84b045 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -799,18 +799,21 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, user), do: false - # child + # child / root def entire_thread_visible_for_user?( - %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, + %Activity{data: %{"object" => object_id}} = tail, user - ) - when is_binary(parent_id) do + ) do parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - end - # root - def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user) + cond do + !is_nil(parent) -> + visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) + + true -> + visible_for_user?(tail, user) + end + end # filter out broken threads def contain_broken_threads(%Activity{} = activity, %User{} = user) do From 5d753e1c7cb1b26ad224255c31b0b64ad917ebaa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 06:52:54 +0000 Subject: [PATCH 25/63] activity: add helpers for updating activities in the database --- lib/pleroma/activity.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 34d0a34b8..e3aa4eb97 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.{Repo, Activity, Notification, Object} - import Ecto.Query + import Ecto.{Query, Changeset} schema "activities" do field(:data, :map) @@ -22,6 +22,13 @@ def get_by_ap_id(ap_id) do ) end + def change(struct, params \\ %{}) do + struct + |> cast(params, [:data]) + |> validate_required([:data]) + |> unique_constraint(:ap_id, name: :activities_unique_apid_index) + end + # TODO: # Go through these and fix them everywhere. # Wrong name, only returns create activities From ef56488349a257def67d6c906a1f71e9bbed397e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 29 Nov 2018 06:53:17 +0000 Subject: [PATCH 26/63] mix: add task to compact the database --- lib/mix/tasks/compact_database.ex | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/mix/tasks/compact_database.ex diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex new file mode 100644 index 000000000..b84b340ac --- /dev/null +++ b/lib/mix/tasks/compact_database.ex @@ -0,0 +1,57 @@ +defmodule Mix.Tasks.CompactDatabase do + @moduledoc """ + Compact the database by flattening the object graph. + """ + + require Logger + + use Mix.Task + import Mix.Ecto + import Ecto.Query + alias Pleroma.{Repo, Object, Activity} + + defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do + data = + activity.data + |> Map.put("object", object_id) + + {:ok, activity} = + Activity.change(activity, %{data: data}) + |> Repo.update() + + {:ok, activity} + end + + defp maybe_compact(%Activity{} = activity), do: {:ok, activity} + + defp activity_query(min_id, max_id) do + from( + a in Activity, + where: fragment("?->>'type' = 'Create'", a.data), + where: a.id >= ^min_id, + where: a.id < ^max_id + ) + end + + def run(args) do + Application.ensure_all_started(:pleroma) + + max = Repo.aggregate(Activity, :max, :id) + Logger.info("Considering #{max} activities") + + chunks = 0..(round(max / 100)) + + Enum.each(chunks, fn (i) -> + min = i * 100 + max = min + 100 + + activity_query(min, max) + |> Repo.all() + |> Enum.each(&maybe_compact/1) + + IO.write(".") + end) + + Logger.info("Finished.") + end +end From 1a360a4eaa57d57ccc4f03dfa25e82a240d0175a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:17:28 +0000 Subject: [PATCH 27/63] compact database task: fix formatting --- lib/mix/tasks/compact_database.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex index b84b340ac..7de50812a 100644 --- a/lib/mix/tasks/compact_database.ex +++ b/lib/mix/tasks/compact_database.ex @@ -39,9 +39,9 @@ def run(args) do max = Repo.aggregate(Activity, :max, :id) Logger.info("Considering #{max} activities") - chunks = 0..(round(max / 100)) + chunks = 0..round(max / 100) - Enum.each(chunks, fn (i) -> + Enum.each(chunks, fn i -> min = i * 100 max = min + 100 From e8caecb5c7f49a829b857131ff98a46c705e3a80 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:29:41 +0000 Subject: [PATCH 28/63] object: move object containment out of transmogrifier into it's own module --- lib/pleroma/object/containment.ex | 64 +++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 5 +- .../web/activity_pub/transmogrifier.ex | 71 +++---------------- test/object/containment_test.exs | 66 +++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 57 --------------- 5 files changed, 144 insertions(+), 119 deletions(-) create mode 100644 lib/pleroma/object/containment.ex create mode 100644 test/object/containment_test.exs diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex new file mode 100644 index 000000000..010b768bd --- /dev/null +++ b/lib/pleroma/object/containment.ex @@ -0,0 +1,64 @@ +defmodule Pleroma.Object.Containment do + @moduledoc """ + # Object Containment + + This module contains some useful functions for containing objects to specific + origins and determining those origins. They previously lived in the + ActivityPub `Transmogrifier` module. + + Object containment is an important step in validating remote objects to prevent + spoofing, therefore removal of object containment functions is NOT recommended. + """ + + require Logger + + def get_actor(%{"actor" => actor}) when is_binary(actor) do + actor + end + + def get_actor(%{"actor" => actor}) when is_list(actor) do + if is_binary(Enum.at(actor, 0)) do + Enum.at(actor, 0) + else + Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) + |> Map.get("id") + end + end + + def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do + id + end + + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + @doc """ + Checks that an imported AP object's actor matches the domain it came from. + """ + def contain_origin(id, %{"actor" => nil}), do: :error + + def contain_origin(id, %{"actor" => actor} = params) do + id_uri = URI.parse(id) + actor_uri = URI.parse(get_actor(params)) + + if id_uri.host == actor_uri.host do + :ok + else + :error + end + end + + def contain_origin_from_id(id, %{"id" => nil}), do: :error + + def contain_origin_from_id(id, %{"id" => other_id} = params) do + id_uri = URI.parse(id) + other_uri = URI.parse(other_id) + + if id_uri.host == other_uri.host do + :ok + else + :error + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 34a84b045..517cf4b46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} + alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -739,7 +740,7 @@ def fetch_object_from_id(id) do "actor" => data["actor"] || data["attributedTo"], "object" => data }, - :ok <- Transmogrifier.contain_origin(id, params), + :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, Object.normalize(activity.data["object"])} else @@ -773,7 +774,7 @@ def fetch_and_contain_remote_object_from_id(id) do recv_timeout: 20000 ), {:ok, data} <- Jason.decode(body), - :ok <- Transmogrifier.contain_origin_from_id(id, data) do + :ok <- Containment.contain_origin_from_id(id, data) do {:ok, data} else e -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5e3d40d9f..1b5e57294 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.User alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub @@ -13,56 +14,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Logger - def get_actor(%{"actor" => actor}) when is_binary(actor) do - actor - end - - def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") - end - end - - def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do - id - end - - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - - @doc """ - Checks that an imported AP object's actor matches the domain it came from. - """ - def contain_origin(id, %{"actor" => nil}), do: :error - - def contain_origin(id, %{"actor" => actor} = params) do - id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - - if id_uri.host == actor_uri.host do - :ok - else - :error - end - end - - def contain_origin_from_id(id, %{"id" => nil}), do: :error - - def contain_origin_from_id(id, %{"id" => other_id} = params) do - id_uri = URI.parse(id) - other_uri = URI.parse(other_id) - - if id_uri.host == other_uri.host do - :ok - else - :error - end - end - @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -99,7 +50,7 @@ def fix_addressing(map) do def fix_actor(%{"attributedTo" => actor} = object) do object - |> Map.put("actor", get_actor(%{"actor" => actor})) + |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end def fix_likes(%{"likes" => likes} = object) @@ -277,7 +228,7 @@ def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) when objtype in ["Article", "Note", "Video", "Page"] do - actor = get_actor(data) + actor = Containment.get_actor(data) data = Map.put(data, "actor", actor) @@ -360,7 +311,7 @@ defp get_follow_activity(follow_object, followed) do def handle_incoming( %{"type" => "Accept", "object" => follow_object, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"), @@ -386,7 +337,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Reject", "object" => follow_object, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), @@ -410,7 +361,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Like", "object" => object_id, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do @@ -423,7 +374,7 @@ def handle_incoming( def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do @@ -477,10 +428,10 @@ def handle_incoming( ) do object_id = Utils.get_ap_id(object_id) - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), - :ok <- contain_origin(actor.ap_id, object.data), + :ok <- Containment.contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else @@ -496,7 +447,7 @@ def handle_incoming( "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do @@ -566,7 +517,7 @@ def handle_incoming( "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs new file mode 100644 index 000000000..fcedb2283 --- /dev/null +++ b/test/object/containment_test.exs @@ -0,0 +1,66 @@ +defmodule Pleroma.Object.ContainmentTest do + use Pleroma.DataCase + + alias Pleroma.User + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.ActivityPub + + import Pleroma.Factory + + describe "general origin containment" do + test "contain_origin_from_id() catches obvious spoofing attempts" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :error = + Containment.contain_origin_from_id( + "http://example.org/~alyssa/activities/1234.json", + data + ) + end + + test "contain_origin_from_id() allows alternate IDs within the same origin domain" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234", + data + ) + end + + test "contain_origin_from_id() allows matching IDs" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234.json", + data + ) + end + + test "users cannot be collided through fake direction spoofing attempts" do + user = + insert(:user, %{ + nickname: "rye@niu.moe", + local: false, + ap_id: "https://niu.moe/users/rye", + follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) + }) + + {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") + end + + test "all objects with fake directions are rejected by the object fetcher" do + {:error, _} = + ActivityPub.fetch_and_contain_remote_object_from_id( + "https://info.pleroma.site/activity4.json" + ) + end + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index ef89752f5..a71a2c5b1 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -945,61 +945,4 @@ test "it rejects activities which reference objects that have an incorrect attri :error = Transmogrifier.handle_incoming(data) end end - - describe "general origin containment" do - test "contain_origin_from_id() catches obvious spoofing attempts" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :error = - Transmogrifier.contain_origin_from_id( - "http://example.org/~alyssa/activities/1234.json", - data - ) - end - - test "contain_origin_from_id() allows alternate IDs within the same origin domain" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234", - data - ) - end - - test "contain_origin_from_id() allows matching IDs" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234.json", - data - ) - end - - test "users cannot be collided through fake direction spoofing attempts" do - user = - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") - end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end - end end From f85949cc696685496a8e4e17aebeb81168ede41a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:53:10 +0000 Subject: [PATCH 29/63] object: factor out fetching functions into Pleroma.Object.Fetcher module --- lib/pleroma/object/fetcher.ex | 69 +++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 63 +---------------- .../activity_pub/activity_pub_controller.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 6 +- .../mastodon_api/mastodon_api_controller.ex | 7 +- test/web/activity_pub/activity_pub_test.exs | 55 --------------- test/web/activity_pub/transmogrifier_test.exs | 12 ---- 7 files changed, 81 insertions(+), 134 deletions(-) create mode 100644 lib/pleroma/object/fetcher.ex diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex new file mode 100644 index 000000000..c27cb1577 --- /dev/null +++ b/lib/pleroma/object/fetcher.ex @@ -0,0 +1,69 @@ +defmodule Pleroma.Object.Fetcher do + alias Pleroma.{Object, Repo} + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.OStatus + + require Logger + + @httpoison Application.get_env(:pleroma, :httpoison) + + # TODO: + # This will create a Create activity, which we need internally at the moment. + def fetch_object_from_id(id) do + if object = Object.get_cached_by_ap_id(id) do + {:ok, object} + else + Logger.info("Fetching #{id} via AP") + + with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + nil <- Object.normalize(data), + params <- %{ + "type" => "Create", + "to" => data["to"], + "cc" => data["cc"], + "actor" => data["actor"] || data["attributedTo"], + "object" => data + }, + :ok <- Containment.contain_origin(id, params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, Object.normalize(activity.data["object"])} + else + {:error, {:reject, nil}} -> + {:reject, nil} + + object = %Object{} -> + {:ok, object} + + _e -> + Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + + case OStatus.fetch_activity_from_url(id) do + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + e -> e + end + end + end + end + + def fetch_and_contain_remote_object_from_id(id) do + Logger.info("Fetching #{id} via AP") + + with true <- String.starts_with?(id, "http"), + {:ok, %{body: body, status_code: code}} when code in 200..299 <- + @httpoison.get( + id, + [Accept: "application/activity+json"], + follow_redirect: true, + timeout: 10000, + recv_timeout: 20000 + ), + {:ok, data} <- Jason.decode(body), + :ok <- Containment.contain_origin_from_id(id, data) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 517cf4b46..fefefc320 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} - alias Pleroma.Object.Containment + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -629,7 +629,7 @@ def user_data_from_user_object(data) do end def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do user_data_from_user_object(data) else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") @@ -723,65 +723,6 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do ) end - # TODO: - # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do - if object = Object.get_cached_by_ap_id(id) do - {:ok, object} - else - Logger.info("Fetching #{id} via AP") - - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), - params <- %{ - "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], - "actor" => data["actor"] || data["attributedTo"], - "object" => data - }, - :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} - else - {:error, {:reject, nil}} -> - {:reject, nil} - - object = %Object{} -> - {:ok, object} - - _e -> - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") - - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} - e -> e - end - end - end - end - - def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") - - with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status_code: code}} when code in 200..299 <- - @httpoison.get( - id, - [Accept: "application/activity+json"], - follow_redirect: true, - timeout: 10000, - recv_timeout: 20000 - ), - {:ok, data} <- Jason.decode(body), - :ok <- Containment.contain_origin_from_id(id, data) do - {:ok, data} - else - e -> - {:error, e} - end - end - def is_public?(activity) do "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++ (activity.data["cc"] || [])) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3570a75cb..7b7c0e090 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller alias Pleroma.{User, Object} + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -122,7 +123,7 @@ def inbox(conn, %{"type" => "Create"} = params) do "Signature missing or not from author, relayed Create message, fetching object from source" ) - ActivityPub.fetch_object_from_id(params["object"]["id"]) + Fetcher.fetch_object_from_id(params["object"]["id"]) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1b5e57294..e76e29b95 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.User alias Pleroma.Object - alias Pleroma.Object.Containment + alias Pleroma.Object.{Containment, Fetcher} alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub @@ -529,8 +529,8 @@ def handle_incoming( def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"]) + def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id) + def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"]) def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 90225460b..71390be0d 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller alias Pleroma.{Repo, Object, Activity, User, Notification, Stats} + alias Pleroma.Object.Fetcher alias Pleroma.Web alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -658,7 +659,7 @@ def unblock_domain(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) d def status_search(query) do fetched = if Regex.match?(~r/https?:/, query) do - with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do + with {:ok, object} <- Fetcher.fetch_object_from_id(query) do [Activity.get_create_activity_by_object_ap_id(object.data["id"])] else _e -> [] @@ -986,7 +987,9 @@ def login(conn, %{"code" => code}) do def login(conn, _) do with {:ok, app} <- get_or_make_app() do path = - o_auth_path(conn, :authorize, + o_auth_path( + conn, + :authorize, response_type: "code", client_id: app.client_id, redirect_uri: ".", diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 231a334f9..bc9fcc75d 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -372,43 +372,6 @@ test "fetches the latest Follow activity" do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end - end - describe "following / unfollowing" do test "creates a follow activity" do follower = insert(:user) @@ -530,15 +493,6 @@ test "it filters broken threads" do end end - test "it can fetch plume articles" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" - ) - - assert object - end - describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) @@ -560,15 +514,6 @@ test "it creates an update activity with the new user data" do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - def data_uri do File.read!("test/fixtures/avatar_data_uri") end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a71a2c5b1..ea9d9fe58 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -894,10 +894,6 @@ test "it fixes the actor URL property to be a proper URI" do end describe "actor origin containment" do - test "it rejects objects with a bogus origin" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json") - end - test "it rejects activities which reference objects with bogus origins" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -911,10 +907,6 @@ test "it rejects activities which reference objects with bogus origins" do :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 1)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -928,10 +920,6 @@ test "it rejects activities which reference objects that have an incorrect attri :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 2)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", From 7a57db0d3a18dbeb9fb3682b98f741ef6ba5f518 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:02:32 +0000 Subject: [PATCH 30/63] federator: fix up contain_origin_from_id() call --- lib/pleroma/web/federator/federator.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index ac3d7c132..0644f8d0a 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -2,6 +2,7 @@ defmodule Pleroma.Web.Federator do use GenServer alias Pleroma.User alias Pleroma.Activity + alias Pleroma.Object.Containment alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.ActivityPub.ActivityPub @@ -106,7 +107,7 @@ def handle(:incoming_ap_doc, params) do # actor shouldn't be acting on objects outside their own AP server. with {:ok, _user} <- ap_enabled_actor(params["actor"]), nil <- Activity.normalize(params["id"]), - :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params), + :ok <- Containment.contain_origin_from_id(params["actor"], params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, activity} else From f0439617ef233c150bd3e9dff620b47cfb1f8895 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:03:03 +0000 Subject: [PATCH 31/63] tests: some minor cleanups --- test/object/containment_test.exs | 7 --- test/object/fetcher_test.exs | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 test/object/fetcher_test.exs diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index fcedb2283..268675c86 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -55,12 +55,5 @@ test "users cannot be collided through fake direction spoofing attempts" do {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end end end diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs new file mode 100644 index 000000000..3bbade9d1 --- /dev/null +++ b/test/object/fetcher_test.exs @@ -0,0 +1,85 @@ +defmodule Pleroma.Object.FetcherTest do + use Pleroma.DataCase + + alias Pleroma.{Activity, Object} + alias Pleroma.Object.Fetcher + + import Pleroma.Factory + + describe "actor origin containment" do + test "it rejects objects with a bogus origin" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") + end + + test "it rejects objects when attributedTo is wrong (variant 1)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json") + end + + test "it rejects objects when attributedTo is wrong (variant 2)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json") + end + end + + describe "fetching an object" do + test "it fetches an object" do + {:ok, object} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert [attachment] = object.data["attachment"] + assert is_list(attachment["url"]) + + assert object == object_again + end + + test "it works with objects only available via Ostatus" do + {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + + assert object == object_again + end + + test "it correctly stitches up conversations between ostatus and ap" do + last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + {:ok, object} = Fetcher.fetch_object_from_id(last) + + object = Object.get_by_ap_id(object.data["inReplyTo"]) + assert object + end + end + + describe "implementation quirks" do + test "it can fetch plume articles" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" + ) + + assert object + end + + test "it can fetch peertube videos" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" + ) + + assert object + end + + test "all objects with fake directions are rejected by the object fetcher" do + {:error, _} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://info.pleroma.site/activity4.json" + ) + end + end +end From 02288b5f1cd428c12f92b9ea4fb0b0a935bbdaaf Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 23:08:23 +0000 Subject: [PATCH 32/63] twitterapi: fix bad rebase --- .../web/twitter_api/views/activity_view.ex | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index f202b6e97..18b2ebb0b 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -207,15 +207,17 @@ def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity} def render( "activity.json", - %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts + %{activity: %{data: %{"type" => "Create", "object" => object_id}} = activity} = opts ) do user = get_user(activity.data["actor"], opts) - created_at = object["published"] |> Utils.date_to_asctime() - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) + object = Object.normalize(object_id) + + created_at = object.data["published"] |> Utils.date_to_asctime() + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) attentions = activity.recipients @@ -230,11 +232,11 @@ def render( tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags - {summary, content} = render_content(object) + {summary, content} = render_content(object.data) html = HTML.filter_tags(content, User.html_filter_policy(opts[:for])) - |> Formatter.emojify(object["emoji"]) + |> Formatter.emojify(object.data["emoji"]) reply_parent = Activity.get_in_reply_to_activity(activity) @@ -249,19 +251,19 @@ def render( "is_local" => activity.local, "is_post_verb" => true, "created_at" => created_at, - "in_reply_to_status_id" => object["inReplyToStatusId"], + "in_reply_to_status_id" => object.data["inReplyToStatusId"], "in_reply_to_screen_name" => reply_user && reply_user.nickname, "in_reply_to_profileurl" => User.profile_url(reply_user), "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id, "in_reply_to_user_id" => reply_user && reply_user.id, "statusnet_conversation_id" => conversation_id, - "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), + "attachments" => (object.data["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, "repeat_num" => announcement_count, "favorited" => !!favorited, "repeated" => !!repeated, - "external_url" => object["external_url"] || object["id"], + "external_url" => object.data["external_url"] || object.data["id"], "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, From fed9b5404c1f30a695b60b67348604fc262c14c5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 03:17:25 +0000 Subject: [PATCH 33/63] object: rework Object.normalize() a bit to support transparent fetching --- lib/pleroma/object.ex | 10 +++++++--- lib/pleroma/object/fetcher.ex | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 57a8b1d6b..0e9aefb63 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Object do use Ecto.Schema alias Pleroma.{Repo, Object, Activity} + alias Pleroma.Object.Fetcher import Ecto.{Query, Changeset} schema "objects" do @@ -27,9 +28,12 @@ def get_by_ap_id(ap_id) do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(obj) when is_map(obj), do: normalize(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(_), do: nil + def normalize(_, fetch_remote \\ true) + + def normalize(obj, fetch_remote) when is_map(obj), do: normalize(obj["id"], fetch_remote) + def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) + def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + def normalize(obj, _), do: nil if Mix.env() == :test do def get_cached_by_ap_id(ap_id) do diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c27cb1577..c98722f39 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -17,7 +17,7 @@ def fetch_object_from_id(id) do Logger.info("Fetching #{id} via AP") with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), + nil <- Object.normalize(data, false), params <- %{ "type" => "Create", "to" => data["to"], @@ -27,7 +27,7 @@ def fetch_object_from_id(id) do }, :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} + {:ok, Object.normalize(activity.data["object"], false)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -39,13 +39,22 @@ def fetch_object_from_id(id) do Logger.info("Couldn't get object via AP, trying out OStatus fetching...") case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)} e -> e end end end end + def fetch_object_from_id!(id) do + with {:ok, object} <- fetch_object_from_id(id) do + object + else + _e -> + nil + end + end + def fetch_and_contain_remote_object_from_id(id) do Logger.info("Fetching #{id} via AP") From b3b52b58c3b31b22d3b3227d06d9a5336fa8edd0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 03:18:10 +0000 Subject: [PATCH 34/63] activitypub: transmogrifier: remove obsolete fetch_obj_helper() --- lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index e76e29b95..c4567193f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -85,7 +85,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) "" end - case fetch_obj_helper(in_reply_to_id) do + case get_obj_helper(in_reply_to_id) do {:ok, replied_object} -> with %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do @@ -363,7 +363,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do {:ok, activity} else @@ -376,7 +376,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do {:ok, activity} else @@ -430,7 +430,7 @@ def handle_incoming( with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} @@ -449,7 +449,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do {:ok, activity} else @@ -519,7 +519,7 @@ def handle_incoming( ) do with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do {:ok, activity} else @@ -529,9 +529,6 @@ def handle_incoming( def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"]) - def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil end @@ -629,7 +626,7 @@ def prepare_outgoing(%{"type" => _type} = data) do def maybe_fix_object_url(data) do if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do - case fetch_obj_helper(data["object"]) do + case get_obj_helper(data["object"]) do {:ok, relative_object} -> if relative_object.data["external_url"] do _data = From 419d4bd5e43248957ea66039231e3389c608368d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 05:00:11 +0000 Subject: [PATCH 35/63] tests: add tests for Object.normalize() --- test/object_test.exs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/object_test.exs b/test/object_test.exs index 909605560..3907748f7 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -49,4 +49,22 @@ test "ensures cache is cleared for the object" do refute object == cached_object end end + + describe "normalizer" do + test "fetches unknown objects by default" do + %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "fetches unknown objects when fetch_remote is explicitly true" do + %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "does not fetch unknown objects when fetch_remote is false" do + assert is_nil(Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)) + end + end end From d6ab701a14f7c9fb4d59953648c425e04725fc62 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 4 Dec 2018 05:01:21 +0000 Subject: [PATCH 36/63] formatting --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++- test/object_test.exs | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 39004367a..ba232b0ec 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,8 @@ def handle_note(entry, doc \\ nil) do cw <- OStatus.get_cw(entry), inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo), - inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, + inReplyToObject <- + (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil, inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo, attachments <- OStatus.get_attachments(entry), context <- get_context(entry, inReplyTo), diff --git a/test/object_test.exs b/test/object_test.exs index 3907748f7..dac6c3be7 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -52,19 +52,23 @@ test "ensures cache is cleared for the object" do describe "normalizer" do test "fetches unknown objects by default" do - %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" end test "fetches unknown objects when fetch_remote is explicitly true" do - %Object{} = object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" end test "does not fetch unknown objects when fetch_remote is false" do - assert is_nil(Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)) + assert is_nil( + Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false) + ) end end end From dda4e0e2a8826e400cab0d9f34a699e4248e5e18 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 12:27:29 +0300 Subject: [PATCH 37/63] Fix warnings in object tests --- test/object/containment_test.exs | 3 +-- test/object/fetcher_test.exs | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index 268675c86..cb74c9f4f 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -3,7 +3,6 @@ defmodule Pleroma.Object.ContainmentTest do alias Pleroma.User alias Pleroma.Object.Containment - alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -45,7 +44,7 @@ test "contain_origin_from_id() allows matching IDs" do end test "users cannot be collided through fake direction spoofing attempts" do - user = + _user = insert(:user, %{ nickname: "rye@niu.moe", local: false, diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 3bbade9d1..568c5cc4c 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -1,11 +1,10 @@ defmodule Pleroma.Object.FetcherTest do use Pleroma.DataCase - alias Pleroma.{Activity, Object} + alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Object.Fetcher - import Pleroma.Factory - describe "actor origin containment" do test "it rejects objects with a bogus origin" do {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") From 462028688b7050bb2335914b0987632082fdf3c8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 12:34:19 +0300 Subject: [PATCH 38/63] Fix pinned posts relying on embded objects --- lib/pleroma/web/common_api/common_api.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 9c3daac2c..d6eb843f7 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -217,8 +217,10 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do with %Activity{ actor: ^user_ap_id, data: %{ - "type" => "Create", - "object" => %{ + "type" => "Create" + }, + object: %Object{ + data: %{ "to" => object_to, "type" => "Note" } From b09ae02c04d66e58c2bcc6ce10277c88d5fed576 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 13:04:58 +0300 Subject: [PATCH 39/63] Added some more normalization calls all in mastodon api controller --- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++-- lib/pleroma/web/mastodon_api/views/status_view.ex | 8 ++++---- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index d6eb843f7..6458a3449 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -127,7 +127,7 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat in_reply_to -> # XXX: these heuristics should be moved out of MastodonAPI. with %Object{} = object <- Object.normalize(in_reply_to) do - Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data) + Pleroma.Web.MastodonAPI.StatusView.get_visibility(object) end end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 24a2d4cb9..d2e3da449 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -543,10 +543,11 @@ def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do end def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.bookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e4de5ecfb..03dc587d9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -125,8 +125,8 @@ def render( } end - def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do - object = Object.normalize(object) + def render("status.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity) user = get_user(activity.data["actor"]) @@ -320,8 +320,8 @@ def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do end end - def get_reply_to(%{data: %{"object" => object}}, _) do - object = Object.normalize(object) + def get_reply_to(%{data: %{"object" => _object}} = activity, _) do + object = Object.normalize(activity) if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do Activity.get_create_by_object_ap_id(object.data["inReplyTo"]) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index f21cf677d..70ab92386 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1879,7 +1879,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do capture_log(fn -> conn = conn - |> get("/api/v1/search", %{"q" => activity.data["object"]["id"]}) + |> get("/api/v1/search", %{"q" => Object.normalize(activity).id}) assert results = json_response(conn, 200) From e7c3c367667e96ef7fe31ef9dd8337b563a3ccaa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:21:39 +0300 Subject: [PATCH 40/63] Update functions in object fetcher for tesla and set up a proper mock for tests --- lib/pleroma/object/fetcher.ex | 11 ++++------- test/object/fetcher_test.exs | 10 ++++++++-- test/web/mastodon_api/status_view_test.exs | 3 +-- test/web/twitter_api/views/activity_view_test.exs | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 19d9c51af..138e7866f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -27,7 +27,7 @@ def fetch_object_from_id(id) do }, :ok <- Containment.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"], false)} + {:ok, Object.normalize(activity, false)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -56,16 +56,13 @@ def fetch_object_from_id!(id) do end def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") + Logger.info("Fetching object #{id} via AP") with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status_code: code}} when code in 200..299 <- + {:ok, %{body: body, status: code}} when code in 200..299 <- @httpoison.get( id, - [Accept: "application/activity+json"], - follow_redirect: true, - timeout: 10000, - recv_timeout: 20000 + [{:Accept, "application/activity+json"}] ), {:ok, data} <- Jason.decode(body), :ok <- Containment.contain_origin_from_id(id, data) do diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 568c5cc4c..72f616782 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -4,6 +4,12 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Object.Fetcher + import Tesla.Mock + + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end describe "actor origin containment" do test "it rejects objects with a bogus origin" do @@ -24,7 +30,7 @@ test "it fetches an object" do {:ok, object} = Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) assert activity.data["id"] {:ok, object_again} = @@ -38,7 +44,7 @@ test "it fetches an object" do test "it works with objects only available via Ostatus" do {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) assert activity.data["id"] {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 4ea50c7c6..cc5a84b5d 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.User alias Pleroma.Repo alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -232,7 +231,7 @@ test "a peertube video" do user = insert(:user) {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index 7ef0270cc..b5440c612 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -360,7 +360,7 @@ test "a delete activity" do test "a peertube video" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) From e641651e2b0ba9e6c4d0a99ebf4f7c3b25f7ce67 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:27:02 +0300 Subject: [PATCH 41/63] Fix unbookmarking --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index d2e3da449..1e82b2f68 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -555,10 +555,11 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do end def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.unbookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) From ad681877df46c151ee20b58401bda9f84d884109 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:52:01 +0300 Subject: [PATCH 42/63] Make credo happy --- lib/pleroma/web/activity_pub/transmogrifier.ex | 5 ++--- lib/pleroma/web/common_api/utils.ex | 8 ++++---- lib/pleroma/web/federator/federator.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- test/object/containment_test.exs | 2 +- test/object_test.exs | 6 ++++++ test/web/common_api/common_api_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 4 ++-- 9 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0637b18dc..a80aa52c6 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -6,13 +6,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @moduledoc """ A module to handle coding from internal to wire ActivityPub and back. """ - alias Pleroma.User - alias Pleroma.Object - alias Pleroma.Object.Containment alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.Repo alias Pleroma.User + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 7781f1635..50a72aee5 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -208,7 +208,7 @@ def make_note_data( context, content_html, attachments, - inReplyTo, + in_reply_to, tags, cw \\ nil, cc \\ [] @@ -225,11 +225,11 @@ def make_note_data( "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if inReplyTo do - inReplyToObject = Object.normalize(inReplyTo.data["object"]) + if in_reply_to do + in_reply_to_object = Object.normalize(inReplyTo.data["object"]) object - |> Map.put("inReplyTo", inReplyToObject.data["id"]) + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) else object end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index a1f6373a4..1b4deb6dc 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity + alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -12,7 +13,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.OStatus - alias Pleroma.Object.Containment alias Pleroma.Web.Salmon alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1e82b2f68..4cec26c9b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -4,13 +4,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - alias Pleroma.Object.Fetcher alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Filter alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.ScheduledActivity diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 03dc587d9..f8961eb6c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.HTML - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index cb74c9f4f..452064093 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -1,8 +1,8 @@ defmodule Pleroma.Object.ContainmentTest do use Pleroma.DataCase - alias Pleroma.User alias Pleroma.Object.Containment + alias Pleroma.User import Pleroma.Factory diff --git a/test/object_test.exs b/test/object_test.exs index a30efd48c..d138ee091 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -5,9 +5,15 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase import Pleroma.Factory + import Tesla.Mock alias Pleroma.Object alias Pleroma.Repo + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + test "returns an object by it's AP id" do object = insert(:note) found_object = Object.get_by_ap_id(object.data["id"]) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index b9ed088e4..e12cc04c8 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -5,8 +5,8 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Activity - alias Pleroma.User alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index cc5a84b5d..ad6344006 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -6,9 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase alias Pleroma.Activity - alias Pleroma.User - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView From a53b917e7ff2df74f53f1f4578c212fe539acb00 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 14:55:26 +0300 Subject: [PATCH 43/63] oof --- lib/pleroma/web/common_api/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 50a72aee5..25f498fcb 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -226,7 +226,7 @@ def make_note_data( } if in_reply_to do - in_reply_to_object = Object.normalize(inReplyTo.data["object"]) + in_reply_to_object = Object.normalize(in_reply_to.data["object"]) object |> Map.put("inReplyTo", in_reply_to_object.data["id"]) From 54b82f236bac154f520442c4c5cd70323cb5cee6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:03:17 +0300 Subject: [PATCH 44/63] Fix note count test --- test/web/activity_pub/activity_pub_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 68bfb3858..0ab29742a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -775,10 +775,10 @@ test "decrements user note count only for public activities" do {:ok, a4} = CommonAPI.post(User.get_by_id(user.id), %{"status" => "yeah", "visibility" => "direct"}) - {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() + {:ok, _} = Object.normalize(a1) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a2) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a3) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a4) |> ActivityPub.delete() user = User.get_by_id(user.id) assert user.info.note_count == 10 From c3a20528060e4fb95292dd93768d9afc8926e66e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:11:22 +0300 Subject: [PATCH 45/63] If it's an object struct it is already normalized --- lib/pleroma/object.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 3f1d0fea1..740d687a3 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,6 +44,7 @@ def get_by_ap_id(ap_id) do def normalize(_, fetch_remote \\ true) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! + def normalize(%Object{} = object, _), do: object def normalize(%Activity{object: %Object{} = object}, _), do: object # A hack for fake activities From ff8d76c670eb25213948974d03bbc389421558a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:46:59 +0300 Subject: [PATCH 46/63] Refactor all tests that acessed the embeded object --- test/scheduled_activity_worker_test.exs | 2 +- test/user_test.exs | 7 ++++--- test/web/activity_pub/activity_pub_test.exs | 8 -------- test/web/activity_pub/transmogrifier_test.exs | 10 +++++----- test/web/common_api/common_api_test.exs | 2 +- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 ++-- test/web/mastodon_api/status_view_test.exs | 2 +- 7 files changed, 14 insertions(+), 21 deletions(-) diff --git a/test/scheduled_activity_worker_test.exs b/test/scheduled_activity_worker_test.exs index b9c91dda6..e3ad1244e 100644 --- a/test/scheduled_activity_worker_test.exs +++ b/test/scheduled_activity_worker_test.exs @@ -14,6 +14,6 @@ test "creates a status from the scheduled activity" do refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert activity.data["object"]["content"] == "hi" + assert Pleroma.Object.normalize(activity).data["content"] == "hi" end end diff --git a/test/user_test.exs b/test/user_test.exs index d2167a970..6ce5b9cf5 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Builders.UserBuilder alias Pleroma.Repo alias Pleroma.User @@ -256,7 +257,7 @@ test "it sends a welcome message if it is set" do activity = Repo.one(Pleroma.Activity) assert registered_user.ap_id in activity.recipients - assert activity.data["object"]["content"] =~ "cool site" + assert Object.normalize(activity).data["content"] =~ "cool site" assert activity.actor == welcome_user.ap_id Pleroma.Config.put([:instance, :welcome_user_nickname], nil) @@ -1132,14 +1133,14 @@ test "bookmarks" do "status" => "heweoo!" }) - id1 = activity1.data["object"]["id"] + id1 = Object.normalize(activity1).data["id"] {:ok, activity2} = CommonAPI.post(user, %{ "status" => "heweoo!" }) - id2 = activity2.data["object"]["id"] + id2 = Object.normalize(activity2).data["id"] assert {:ok, user_state1} = User.bookmark(user, id1) assert user_state1.bookmarks == [id1] diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0ab29742a..122690184 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -250,25 +250,21 @@ test "increases replies count" do # public {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 # unlisted {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # private {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # direct {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 end end @@ -820,22 +816,18 @@ test "decreases reply count" do _ = CommonAPI.delete(direct_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(private_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(public_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 _ = CommonAPI.delete(unlisted_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 0 assert object.data["repliesCount"] == 0 end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5559cdf87..b062b273f 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -187,15 +187,15 @@ test "it cleans up incoming notices which are not really DMs" do data = Map.put(data, "object", object) - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) assert data["to"] == [] assert data["cc"] == to - object = data["object"] + object_data = Object.normalize(activity).data - assert object["to"] == [] - assert object["cc"] == to + assert object_data["to"] == [] + assert object_data["cc"] == to end test "it works for incoming follow requests" do @@ -331,7 +331,7 @@ test "it does not clobber the addressing on announce activities" do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", Object.normalize(activity).data["id"]) |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"]) |> Map.put("cc", []) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e12cc04c8..3d2bb8929 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -42,7 +42,7 @@ test "it adds emoji in the object" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"}) - assert activity.data["object"]["emoji"]["moominmamma"] + assert Object.normalize(activity).data["emoji"]["moominmamma"] end test "it adds emoji when updating profiles" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 70ab92386..245887ff8 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2791,9 +2791,9 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c assert %{"content" => "xD", "id" => id} = json_response(conn1, 200) - activity = Activity.get_by_id(id) + activity = Activity.get_by_id_with_object(id) - assert activity.data["object"]["inReplyTo"] == replied_to.data["object"]["id"] + assert Object.normalize(activity).data["inReplyTo"] == Object.normalize(replied_to).data["id"] assert Activity.get_in_reply_to_activity(activity).id == replied_to.id # Reblog from the third user diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index ad6344006..a02c7c210 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -61,7 +61,7 @@ test "a note with null content" do |> Map.put("content", nil) Object.change(note_object, %{data: data}) - |> Repo.update() + |> Object.update_and_set_cache() User.get_cached_by_ap_id(note.data["actor"]) From 8e4d950f31ec3ea956f6892f9f36b419344bf930 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 15:54:09 +0300 Subject: [PATCH 47/63] Remove updating reply count for embeded objects --- lib/pleroma/activity.ex | 50 -------------------- lib/pleroma/web/activity_pub/activity_pub.ex | 2 - test/activity_test.exs | 14 ------ 3 files changed, 66 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 99cc9c077..478d16356 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -257,54 +257,4 @@ def all_by_actor_and_id(actor, status_ids) do |> where([s], s.actor == ^actor) |> Repo.all() end - - def increase_replies_count(nil), do: nil - - def increase_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end - - def decrease_replies_count(nil), do: nil - - def decrease_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a3b47cb3..28fca6116 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,6 @@ def increase_replies_count_if_reply(%{ "type" => "Create" }) do if is_public?(object) do - Activity.increase_replies_count(reply_ap_id) Object.increase_replies_count(reply_ap_id) end end @@ -106,7 +105,6 @@ def decrease_replies_count_if_reply(%Object{ data: %{"inReplyTo" => reply_ap_id} = object }) do if is_public?(object) do - Activity.decrease_replies_count(reply_ap_id) Object.decrease_replies_count(reply_ap_id) end end diff --git a/test/activity_test.exs b/test/activity_test.exs index dc9c56a21..ad889f544 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -28,18 +28,4 @@ test "returns the activity that created an object" do assert activity == found_activity end - - test "reply count" do - %{id: id, data: %{"object" => %{"id" => object_ap_id}}} = activity = insert(:note_activity) - - replies_count = activity.data["object"]["repliesCount"] || 0 - expected_increase = replies_count + 1 - Activity.increase_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_increase}}} = Activity.get_by_id(id) - assert expected_increase == actual_increase - expected_decrease = expected_increase - 1 - Activity.decrease_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_decrease}}} = Activity.get_by_id(id) - assert expected_decrease == actual_decrease - end end From 4c289e924e2ef7863a2c95b74f71fd83969b7827 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 16:35:01 +0300 Subject: [PATCH 48/63] Fix delete-by_ap_id to expect not only embeded objects --- lib/pleroma/activity.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 478d16356..9043530c9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -230,6 +230,7 @@ def delete_by_ap_id(id) when is_binary(id) do |> Repo.delete_all() |> elem(1) |> Enum.find(fn + %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id _ -> nil end) From d1eb578a5763638cb9959834814f8632a810f8b2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 17:03:35 +0300 Subject: [PATCH 49/63] Refactor tests that used ActivityPub.fetch_object_from_id --- test/web/activity_pub/activity_pub_test.exs | 46 ------------------- test/web/activity_pub/transmogrifier_test.exs | 5 +- 2 files changed, 3 insertions(+), 48 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 122690184..291f3df4b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -647,43 +647,6 @@ test "fetches the latest Follow activity" do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end - end - describe "following / unfollowing" do test "creates a follow activity" do follower = insert(:user) @@ -900,15 +863,6 @@ test "it creates an update activity with the new user data" do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - test "returned pinned statuses" do Pleroma.Config.put([:instance, :max_pinned_statuses], 3) user = insert(:user) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b062b273f..34ae3a20e 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -790,7 +791,7 @@ test "it rejects activities without a valid ID" do test "it remaps video URLs as attachments if necessary" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) @@ -1185,7 +1186,7 @@ test "users cannot be collided through fake direction spoofing attempts" do test "all objects with fake directions are rejected by the object fetcher" do {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( + Fetcher.fetch_and_contain_remote_object_from_id( "https://info.pleroma.site/activity4.json" ) end From 35ac672b8d4a6711754a5f88ad65e52d356c4c67 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Apr 2019 17:59:15 +0300 Subject: [PATCH 50/63] Remove containment tests from transmogrifier and fix thread visibility solver --- lib/pleroma/object/containment.ex | 3 - lib/pleroma/web/activity_pub/visibility.ex | 21 ++++--- test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/activity_pub/transmogrifier_test.exs | 56 ------------------- 4 files changed, 15 insertions(+), 69 deletions(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 27e89d87f..25bd911fb 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -9,9 +9,6 @@ defmodule Pleroma.Object.Containment do Object containment is an important step in validating remote objects to prevent spoofing, therefore removal of object containment functions is NOT recommended. """ - - require Logger - def get_actor(%{"actor" => actor}) when is_binary(actor) do actor end diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index db52fe933..3da709b3d 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -41,16 +41,19 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, _user), do: false - # child + # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop def entire_thread_visible_for_user?( - %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, + %Activity{} = tail, + # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, user - ) - when is_binary(parent_id) do - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - end + ) do + case Object.normalize(tail) do + %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> + parent = Activity.get_in_reply_to_activity(tail) + visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - # root - def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user) + _ -> + visible_for_user?(tail, user) + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 291f3df4b..4a9acae69 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -832,7 +832,9 @@ test "it filters broken threads" do activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + assert [public_activity, private_activity_1, private_activity_3] == + activities + assert length(activities) == 3 activities = ActivityPub.contain_timeline(activities, user1) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 34ae3a20e..6bb81a054 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1136,62 +1136,6 @@ test "it rejects activities which reference objects that have an incorrect attri end end - describe "general origin containment" do - test "contain_origin_from_id() catches obvious spoofing attempts" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :error = - Transmogrifier.contain_origin_from_id( - "http://example.org/~alyssa/activities/1234.json", - data - ) - end - - test "contain_origin_from_id() allows alternate IDs within the same origin domain" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234", - data - ) - end - - test "contain_origin_from_id() allows matching IDs" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234.json", - data - ) - end - - test "users cannot be collided through fake direction spoofing attempts" do - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") - end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - Fetcher.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end - end - describe "reserialization" do test "successfully reserializes a message with inReplyTo == nil" do user = insert(:user) From 2abc09570f40352e949f0142d11778f89a70c920 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 01:37:04 +0300 Subject: [PATCH 51/63] Use the preloaded object in tag queries --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 ++++++++-------- test/web/activity_pub/activity_pub_test.exs | 9 +++++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 28fca6116..6b2fb17a4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -574,8 +574,8 @@ defp restrict_since(query, _), do: query defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) when is_list(tag_reject) and tag_reject != [] do from( - activity in query, - where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject) + [_activity, object] in query, + where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end @@ -584,8 +584,8 @@ defp restrict_tag_reject(query, _), do: query defp restrict_tag_all(query, %{"tag_all" => tag_all}) when is_list(tag_all) and tag_all != [] do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end @@ -593,15 +593,15 @@ defp restrict_tag_all(query, _), do: query defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data) + [_activity, object] in query, + where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) ) end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4a9acae69..aacafc60a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -84,17 +84,21 @@ test "it fetches the appropriate tag-restricted posts" do {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"}) {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"}) - fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"}) - fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]}) + fetch_one = ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => "test"}) + + fetch_two = + ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => ["test", "essais"]}) fetch_three = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test", "essais"], "tag_reject" => ["reject"] }) fetch_four = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test"], "tag_all" => ["test", "reject"] }) @@ -832,6 +836,7 @@ test "it filters broken threads" do activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) + assert [public_activity, private_activity_1, private_activity_3] == activities From e31a22043bcf74d1d85f6fe007bd4606291d41e9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 08:31:08 +0300 Subject: [PATCH 52/63] Fix media timeline depending on embeded object and add some guards --- lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6b2fb17a4..03be8b06f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -571,6 +571,10 @@ defp restrict_since(query, %{"since_id" => since_id}) do defp restrict_since(query, _), do: query + defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) when is_list(tag_reject) and tag_reject != [] do from( @@ -581,6 +585,10 @@ defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) defp restrict_tag_reject(query, _), do: query + defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_all(query, %{"tag_all" => tag_all}) when is_list(tag_all) and tag_all != [] do from( @@ -591,6 +599,10 @@ defp restrict_tag_all(query, %{"tag_all" => tag_all}) defp restrict_tag_all(query, _), do: query + defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( [_activity, object] in query, @@ -666,10 +678,14 @@ defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do defp restrict_favorited_by(query, _), do: query + defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do from( - activity in query, - where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[]) + [_activity, object] in query, + where: fragment("not (?)->'attachment' = (?)", object.data, ^[]) ) end From 6069d0fd361a9971f2297d3babc596d60634eb43 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 09:28:20 +0300 Subject: [PATCH 53/63] Fix object search depending on embeded object --- lib/pleroma/web/activity_pub/visibility.ex | 4 +++- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 +++--- test/user_test.exs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 3da709b3d..6dee61dd6 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -41,7 +41,9 @@ def visible_for_user?(activity, user) do # guard def entire_thread_visible_for_user?(nil, _user), do: false - # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop + # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def entire_thread_visible_for_user?( %Activity{} = tail, # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 4cec26c9b..3916d7c41 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1012,13 +1012,13 @@ def status_search(user, query) do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: 20, diff --git a/test/user_test.exs b/test/user_test.exs index 6ce5b9cf5..eee6881eb 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -4,8 +4,8 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity - alias Pleroma.Object alias Pleroma.Builders.UserBuilder + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI From ac04311b3f0a611b5008747037d6cd5874fa3ae9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 10:24:06 +0300 Subject: [PATCH 54/63] Fix search in TwitterAPI --- lib/pleroma/web/twitter_api/twitter_api.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index d6ce0a7c6..8e44dbeb8 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -266,6 +266,7 @@ defp parse_int(string, default) when is_binary(string) do defp parse_int(_, default), do: default + # TODO: unify the search query with MastoAPI one and do only pagination here def search(_user, %{"q" => query} = params) do limit = parse_int(params["rpp"], 20) page = parse_int(params["page"], 1) @@ -273,13 +274,13 @@ def search(_user, %{"q" => query} = params) do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: ^limit, From a11ca87f40fd85341afa4445d3b7303ae8e92b76 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 13:10:38 +0300 Subject: [PATCH 55/63] Add a migration to remove embeded objects --- lib/mix/tasks/compact_database.ex | 57 ------------------- .../20190418072951_remove_embeded_objects.exs | 10 ++++ 2 files changed, 10 insertions(+), 57 deletions(-) delete mode 100644 lib/mix/tasks/compact_database.ex create mode 100644 priv/repo/migrations/20190418072951_remove_embeded_objects.exs diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex deleted file mode 100644 index 17b9721f7..000000000 --- a/lib/mix/tasks/compact_database.ex +++ /dev/null @@ -1,57 +0,0 @@ -defmodule Mix.Tasks.CompactDatabase do - @moduledoc """ - Compact the database by flattening the object graph. - """ - - require Logger - - use Mix.Task - import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Repo - - defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do - data = - activity.data - |> Map.put("object", object_id) - - {:ok, activity} = - Activity.change(activity, %{data: data}) - |> Repo.update() - - {:ok, activity} - end - - defp maybe_compact(%Activity{} = activity), do: {:ok, activity} - - defp activity_query(min_id, max_id) do - from( - a in Activity, - where: fragment("?->>'type' = 'Create'", a.data), - where: a.id >= ^min_id, - where: a.id < ^max_id - ) - end - - def run(_args) do - Application.ensure_all_started(:pleroma) - - max = Repo.aggregate(Activity, :max, :id) - Logger.info("Considering #{max} activities") - - chunks = 0..round(max / 100) - - Enum.each(chunks, fn i -> - min = i * 100 - max = min + 100 - - activity_query(min, max) - |> Repo.all() - |> Enum.each(&maybe_compact/1) - - IO.write(".") - end) - - Logger.info("Finished.") - end -end diff --git a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs new file mode 100644 index 000000000..128094278 --- /dev/null +++ b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.RemoveEmbededObjects do + use Ecto.Migration + + # TODO: bench on a real DB and add clippy if it takes too long + def change do + execute """ + update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->>'type' = 'Create' and data->'object'->>'id' is not null; + """ + end +end From 4ef237f26a41652014105b8a7ad3a9d5c7b41202 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 14:37:57 +0300 Subject: [PATCH 56/63] Fix my incorrect search test fix --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 245887ff8..786af2088 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1879,7 +1879,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do capture_log(fn -> conn = conn - |> get("/api/v1/search", %{"q" => Object.normalize(activity).id}) + |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]}) assert results = json_response(conn, 200) From 9238dccac1310fc2e281b242768a2de79f405f35 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 21:40:40 +0300 Subject: [PATCH 57/63] Add a guard to fetching reply activity --- lib/pleroma/activity.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 9043530c9..4a2ded518 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -204,12 +204,14 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do def create_by_object_ap_id_with_object(_), do: nil - def get_create_by_object_ap_id_with_object(ap_id) do + def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id |> create_by_object_ap_id_with_object() |> Repo.one() end + def get_create_by_object_ap_id_with_object(_), do: nil + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_by_object_ap_id_with_object(ap_id) end From 099f89367efaf4032b8e937258b2c1a90f16b047 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 23:34:01 +0300 Subject: [PATCH 58/63] Replace embedded object migration with a mix task --- lib/mix/tasks/pleroma/database.ex | 45 +++++++++++++++++++ .../20190418072951_remove_embeded_objects.exs | 10 ----- 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 lib/mix/tasks/pleroma/database.ex delete mode 100644 priv/repo/migrations/20190418072951_remove_embeded_objects.exs diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex new file mode 100644 index 000000000..ce3252af5 --- /dev/null +++ b/lib/mix/tasks/pleroma/database.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Database do + alias Mix.Tasks.Pleroma.Common + use Mix.Task + + @shortdoc "A collection of database related tasks" + @moduledoc """ + A collection of database related tasks + + ## Replace embedded objects with their references + + Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration. + + mix pleroma.database remove_embedded_objects + + Options: + - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + """ + def run(["remove_embedded_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;" + ) + + if Keyword.get(options, :vacuum) do + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "vacuum full;" + ) + end + end +end diff --git a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs b/priv/repo/migrations/20190418072951_remove_embeded_objects.exs deleted file mode 100644 index 128094278..000000000 --- a/priv/repo/migrations/20190418072951_remove_embeded_objects.exs +++ /dev/null @@ -1,10 +0,0 @@ -defmodule Pleroma.Repo.Migrations.RemoveEmbededObjects do - use Ecto.Migration - - # TODO: bench on a real DB and add clippy if it takes too long - def change do - execute """ - update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->>'type' = 'Create' and data->'object'->>'id' is not null; - """ - end -end From 945325013af6dde3f1da2417753bb97f55911a84 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 18 Apr 2019 23:58:59 +0300 Subject: [PATCH 59/63] remove query timeouts --- lib/mix/tasks/pleroma/database.ex | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ce3252af5..d657c1ef0 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -30,15 +30,17 @@ def run(["remove_embedded_objects" | args]) do Common.start_pleroma() - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;" + Pleroma.Repo.query!( + "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", + [], + timeout: :infinity ) if Keyword.get(options, :vacuum) do - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "vacuum full;" + Pleroma.Repo.query!( + "vacuum full;", + [], + timeout: :infinity ) end end From 73b8c5387b25caaf27734f7018dc4702d49af7de Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:17:37 +0300 Subject: [PATCH 60/63] Add some logging --- lib/mix/tasks/pleroma/database.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index d657c1ef0..ab9a3a7ff 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + require Logger use Mix.Task @shortdoc "A collection of database related tasks" @@ -29,6 +30,7 @@ def run(["remove_embedded_objects" | args]) do ) Common.start_pleroma() + Logger.info("Removing embedded objects") Pleroma.Repo.query!( "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", @@ -37,6 +39,8 @@ def run(["remove_embedded_objects" | args]) do ) if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + Pleroma.Repo.query!( "vacuum full;", [], From 17d94ae2676efad3a78ddfcd9dbb86d52e26b929 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:26:46 +0300 Subject: [PATCH 61/63] Add a changelog entry for removing embded objects mix task --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ad83a01..648dceabe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database remove_embedded_objects` - Federation: Support for reports - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option From f9865cf9439e2e9273f55d3b82c1e68166178b07 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 19 Apr 2019 00:47:02 +0300 Subject: [PATCH 62/63] Stream out deletes, mistakingly removed when resolving merge conflicts --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9e2574419..0b99a169c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -167,7 +167,7 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do def stream_out(activity) do public = "https://www.w3.org/ns/activitystreams#Public" - if activity.data["type"] in ["Create", "Announce"] do + if activity.data["type"] in ["Create", "Announce", "Delete"] do object = Object.normalize(activity.data["object"]) Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) From 8166cefa2103eecdf4af0b52b9e69d3b9b4e9cdb Mon Sep 17 00:00:00 2001 From: "witches.live" Date: Sat, 20 Apr 2019 07:11:06 +0000 Subject: [PATCH 63/63] Include explicit warning for emerge config line as it will vary more widely based on current version of postgres --- docs/installation/gentoo_en.md | 296 +++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 docs/installation/gentoo_en.md diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md new file mode 100644 index 000000000..fccaad378 --- /dev/null +++ b/docs/installation/gentoo_en.md @@ -0,0 +1,296 @@ +# Installing on Gentoo GNU/Linux +## Installation + +This guide will assume that you have administrative rights, either as root or a user with [sudo permissions](https://wiki.gentoo.org/wiki/Sudo). Lines that begin with `#` indicate that they should be run as the superuser. Lines using `$` should be run as the indicated user, e.g. `pleroma$` should be run as the `pleroma` user. + +### Configuring your hostname (optional) + +If you would like your prompt to permanently include your host/domain, change `/etc/conf.d/hostname` to your hostname. You can reboot or use the `hostname` command to make immediate changes. + +### Your make.conf, package.use, and USE flags + +The only specific USE flag you should need is the `uuid` flag for `dev-db/postgresql`. Add the following line to any new file in `/etc/portage/package.use`. If you would like a suggested name for the file, either `postgresql` or `pleroma` would do fine, depending on how you like to arrange your package.use flags. + +```text +dev-db/postgresql uuid +``` + +You could opt to add `USE="uuid"` to `/etc/portage/make.conf` if you'd rather set this as a global USE flags, but this flags does unrelated things in other packages, so keep that in mind if you elect to do so. + +Double check your compiler flags in `/etc/portage/make.conf`. If you require any special compilation flags or would like to set up remote builds, now is the time to do so. Be sure that your CFLAGS and MAKEOPTS make sense for the platform you are using. It is not recommended to use above `-O2` or risky optimization flags for a production server. + +### Installing a cron daemon + +Gentoo quite pointedly does not come with a cron daemon installed, and as such it is recommended you install one to automate certbot renewals and to allow other system administration tasks to be run automatically. Gentoo has [a whole wide world of cron options](https://wiki.gentoo.org/wiki/Cron) but if you just want A Cron That Works, `emerge --ask virtual/cron` will install the default cron implementation (probably cronie) which will work just fine. For the purpouses of this guide, we will be doing just that. + +### Required ebuilds + +* `dev-db/postgresql` +* `dev-lang/elixir` +* `dev-vcs/git` + +#### Optional ebuilds used in this guide + +* `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo) +* `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates) +* `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot) + +### Prepare the system + +* First ensure that you have the latest copy of the portage ebuilds if you have not synced them yet: + +```shell + # emaint sync -a +``` + +* Emerge all required the required and suggested software in one go: + +```shell + # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx +``` + +If you would not like to install the optional packages, remove them from this line. + +If you're running this from a low-powered virtual machine, it should work though it will take some time. There were no issues on a VPS with a single core and 1GB of RAM; if you are using an even more limited device and run into issues, you can try creating a swapfile or use a more powerful machine running Gentoo to [cross build](https://wiki.gentoo.org/wiki/Cross_build_environment). If you have a wait ahead of you, now would be a good time to take a break, strech a bit, refresh your beverage of choice and/or get a snack, and reply to Arch users' posts with "I use Gentoo btw" as we do. + +### Install PostgreSQL + +[Gentoo Wiki article](https://wiki.gentoo.org/wiki/PostgreSQL) as well as [PostgreSQL QuickStart](https://wiki.gentoo.org/wiki/PostgreSQL/QuickStart) might be worth a quick glance, as the way Gentoo handles postgres is slightly unusual, with built in capability to have two different databases running for testing and live or whatever other purpouse. While it is still straightforward to install, it does mean that the version numbers used in this guide might change for future updates, so keep an eye out for the output you get from `emerge` to ensure you are using the correct ones. + +* Install postgresql if you have not done so already: + +```shell + # emerge --ask dev-db/postgresql +``` + +Ensure that `/etc/conf.d/postgresql-11` has the encoding you want (it defaults to UTF8 which is probably what you want) and make any adjustments to the data directory if you find it necessary. Be sure to adjust the number at the end depending on what version of postgres you actually installed. + +* Initialize the database cluster + +The output from emerging postgresql should give you a command for initializing the postgres database. The default slot should be indicated in this command, ensure that it matches the command below. + +```shell + # emerge --config dev-db/postgresql:11 +``` + +* Start postgres and enable the system service + +```shell + # /etc/init.d/postgresql-11 start + # rc-update add postgresql-11 default + ``` + +### A note on licenses, the AGPL, and deployment procedures + +If you do not plan to make any modifications to your Pleroma instance, cloning directly from the main repo will get you what you need. However, if you plan on doing any contributions to upstream development, making changes or modifications to your instance, making custom themes, or want to play around--and let's be honest here, if you're using Gentoo that is most likely you--you will save yourself a lot of headache later if you take the time right now to fork the Pleroma repo and use that in the following section. + +Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure. + +### Install PleromaBE + +* Add a new system user for the Pleroma service and set up default directories: + +Remove `,wheel` if you do not want this user to be able to use `sudo`, however note that being able to `sudo` as the `pleroma` user will make finishing the insallation and common maintenence tasks somewhat easier: + +```shell + # useradd -m -G users,wheel -s /bin/bash pleroma +``` + +Optional: If you are using sudo, review your sudo setup to ensure it works for you. The `/etc/sudoers` file has a lot of options and examples to help you, and [the Gentoo sudo guide](https://wiki.gentoo.org/wiki/Sudo) has more information. Finishing this installation will be somewhat easier if you have a way to sudo from the `pleroma` user, but it might be best to not allow that user to sudo during normal operation, and as such there will be a reminder at the end of this guide to double check if you would like to lock down the `pleroma` user after initial setup. + +**Note**: To execute a single command as the Pleroma system user, use `sudo -Hu pleroma command`. You can also switch to a shell by using `sudo -Hu pleroma $SHELL`. If you don't have or want `sudo` or would like to use the system as the `pleroma` user for instance maintenance tasks, you can simply use `su - pleroma` to switch to the `pleroma` user. + +* Git clone the PleromaBE repository and make the Pleroma user the owner of the directory: + +It is highly recommended you use your own fork for the `https://path/to/repo` part below, however if you foolishly decide to forego using your own fork, the primary repo `https://git.pleroma.social/pleroma/pleroma` will work here. + +```shell + pleroma$ cd ~ + pleroma$ git clone https://path/to/repo +``` + +* Change to the new directory: + +```shell +pleroma$ cd ~/pleroma +``` + +* Install the dependencies for Pleroma and answer with `yes` if it asks you to install `Hex`: + +```shell +pleroma$ mix deps.get +``` + +* Generate the configuration: + +```shell +pleroma$ mix pleroma.instance gen +``` + + * Answer with `yes` if it asks you to install `rebar3`. + + * This part precompiles some parts of Pleroma, so it might take a few moments + + * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. + + * Spend some time with `generated_config.exs` to ensure that everything is in order. If you plan on using an S3-compatible service to store your local media, that can be done here. You will likely mostly be using `prod.secret.exs` for a production instance, however if you would like to set up a development environment, make a copy to `dev.secret.exs` and adjust settings as needed as well. + +```shell +pleroma$ mv config/generated_config.exs config/prod.secret.exs +``` + +* The previous command creates also the file `config/setup_db.psql`, with which you can create the database. Ensure that it is using the correct database name on the `CREATE DATABASE` and the `\c` lines, then run the postgres script: + +```shell +pleroma$ sudo -Hu postgres psql -f config/setup_db.psql +``` + +* Now run the database migration: + +```shell +pleroma$ MIX_ENV=prod mix ecto.migrate +``` + +* Now you can start Pleroma already + +```shell +pleroma$ MIX_ENV=prod mix phx.server +``` + +It probably won't work over the public internet quite yet, however, as we still need to set up a web servere to proxy to the pleroma application, as well as configure SSL. + +### Finalize installation + +Assuming you want to open your newly installed federated social network to, well, the federation, you should run nginx or some other webserver/proxy in front of Pleroma. It is also a good idea to set up Pleroma to run as a system service. + +#### Nginx + +* Install nginx, if not already done: + +```shell + # emerge --ask www-servers/nginx +``` + +* Create directories for available and enabled sites: + +```shell + # mkdir -p /etc/nginx/sites-{available,enabled} +``` + +* Append the following line at the end of the `http` block in `/etc/nginx/nginx.conf`: + +```Nginx +include sites-enabled/*; +``` + +* Setup your SSL cert, using your method of choice or certbot. If using certbot, install it if you haven't already: + +```shell + # emerge --ask app-crypt/certbot app-crypt/certbot-nginx +``` + +and then set it up: + +```shell + # mkdir -p /var/lib/letsencrypt/ + # certbot certonly --email -d --standalone +``` + +If that doesn't work the first time, add `--dry-run` to further attempts to avoid being ratelimited as you identify the issue, and do not remove it until the dry run succeeds. If that doesn’t work, make sure, that nginx is not already running. If it still doesn’t work, try setting up nginx first (change ssl “on” to “off” and try again). Often the answer to issues with certbot is to use the `--nginx` flag once you have nginx up and running. + +If you are using any additional subdomains, such as for a media proxy, you can re-run the same command with the subdomain in question. When it comes time to renew later, you will not need to run multiple times for each domain, one renew will handle it. + +--- + +* Copy the example nginx configuration and activate it: + +```shell + # cp /home/pleroma/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/ + # ln -s /etc/nginx/sites-available/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx +``` + +* Take some time to ensure that your nginx config is correct + +Replace all instances of `example.tld` with your instance's public URL. If for whatever reason you made changes to the port that your pleroma app runs on, be sure that is reflected in your configuration. + +Pay special attention to the line that begins with `ssl_ecdh_curve`. It is stongly advised to comment that line out so that OpenSSL will use its full capabilities, and it is also possible you are running OpenSSL 1.0.2 necessitating that you do this. + +* Enable and start nginx: + +```shell + # rc-update add nginx default + # /etc/init.d/nginx start +``` + +If you are using certbot, it is HIGHLY recommend you set up a cron job that renews your certificate, and that you install the suggested `certbot-nginx` plugin. If you don't do these things, you only have yourself to blame when your instance breaks suddenly because you forgot about it. + +First, ensure that the command you will be installing into your crontab works. + +```shell + # /usr/bin/certbot renew --nginx +``` + +Assuming not much time has passed since you got certbot working a few steps ago, you should get a message for all domains you installed certificates for saying `Cert not yet due for renewal`. + +Now, run crontab as a superuser with `crontab -e` or `sudo crontab -e` as appropriate, and add the following line to your cron: + +```cron +0 0 1 * * /usr/bin/certbot renew --nginx +``` + +This will run certbot on the first of the month at midnight. If you'd rather run more frequently, it's not a bad idea, feel free to go for it. + +#### Other webserver/proxies + +If you would like to use other webservers or proxies, there are example configurations for some popular alternatives in `/home/pleroma/pleroma/installation/`. You can, of course, check out [the Gentoo wiki](https://wiki.gentoo.org) for more information on installing and configuring said alternatives. + +#### Create the uploads folder + +Even if you are using S3, Pleroma needs someplace to store media posted on your instance. If you are using the `/home/pleroma/pleroma` root folder suggested by this guide, simply: + +```shell + pleroma$ mkdir -p ~/pleroma/uploads + ``` + +#### init.d service + +* Copy example service file + +```shell + # cp /home/pleroma/pleroma/installation/init.d/pleroma /etc/init.d/ +``` + +* Be sure to take a look at this service file and make sure that all paths fit your installation + +* Enable and start `pleroma`: + +```shell + # rc-update add pleroma default + # /etc/init.d/pleroma start +``` + +#### Create your first user + +If your instance is up and running, you can create your first user with administrative rights with the following task: + +```shell +pleroma$ MIX_ENV=prod mix pleroma.user new --admin +``` + +#### Privilege cleanup + +If you opted to allow sudo for the `pleroma` user but would like to remove the ability for greater security, now might be a good time to edit `/etc/sudoers` and/or change the groups the `pleroma` user belongs to. Be sure to restart the pleroma service afterwards to ensure it picks up on the changes. + +#### Further reading + +* [Admin tasks](Admin tasks) +* [Backup your instance](Backup-your-instance) +* [Configuration tips](General tips for customizing pleroma fe) +* [Hardening your instance](Hardening-your-instance) +* [How to activate mediaproxy](How-to-activate-mediaproxy) +* [Small Pleroma-FE customizations](Small customizations) +* [Updating your instance](Updating-your-instance) + +## Questions + +Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.