From 763756f8790809d593c34dc78196d241d230658a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 4 Aug 2017 16:57:38 +0200 Subject: [PATCH] Also fetch atom links. --- .../web/ostatus/handlers/note_handler.ex | 2 +- lib/pleroma/web/ostatus/ostatus.ex | 99 +++++++++++-------- test/fixtures/httpoison_mock/sakamoto.atom | 1 + .../httpoison_mock/sakamoto_eal_feed.atom | 1 + test/support/httpoison_mock.ex | 22 +++++ test/web/ostatus/ostatus_test.exs | 9 +- 6 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 test/fixtures/httpoison_mock/sakamoto.atom create mode 100644 test/fixtures/httpoison_mock/sakamoto_eal_feed.atom diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index e55f972b2..e67f67b37 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -10,7 +10,7 @@ def fetch_replied_to_activity(entry, inReplyTo) do if inReplyTo && !Object.get_cached_by_ap_id(inReplyTo) do inReplyToHref = XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry) if inReplyToHref do - OStatus.fetch_activity_from_html_url(inReplyToHref) + OStatus.fetch_activity_from_url(inReplyToHref) else Logger.debug("Couldn't find a href link to #{inReplyTo}") end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 469d4031c..02a0996b0 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -24,45 +24,48 @@ def salmon_path(user) do end def handle_incoming(xml_string) do - doc = parse_document(xml_string) - entries = :xmerl_xpath.string('//entry', doc) + with doc when doc != :error <- parse_document(xml_string) do + entries = :xmerl_xpath.string('//entry', doc) - activities = Enum.map(entries, fn (entry) -> - {:xmlObj, :string, object_type} = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) - {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) - Logger.debug("Handling #{verb}") + activities = Enum.map(entries, fn (entry) -> + {:xmlObj, :string, object_type} = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) + {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) + Logger.debug("Handling #{verb}") - try do - case verb do - 'http://activitystrea.ms/schema/1.0/delete' -> - with {:ok, activity} <- DeleteHandler.handle_delete(entry, doc), do: activity - 'http://activitystrea.ms/schema/1.0/follow' -> - with {:ok, activity} <- FollowHandler.handle(entry, doc), do: activity - 'http://activitystrea.ms/schema/1.0/share' -> - with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity] - 'http://activitystrea.ms/schema/1.0/favorite' -> - with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc), do: [activity, favorited_activity] - _ -> - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity - 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity - _ -> - Logger.error("Couldn't parse incoming document") - nil - end - end - rescue - e -> - Logger.error("Error occured while handling activity") + try do + case verb do + 'http://activitystrea.ms/schema/1.0/delete' -> + with {:ok, activity} <- DeleteHandler.handle_delete(entry, doc), do: activity + 'http://activitystrea.ms/schema/1.0/follow' -> + with {:ok, activity} <- FollowHandler.handle(entry, doc), do: activity + 'http://activitystrea.ms/schema/1.0/share' -> + with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity] + 'http://activitystrea.ms/schema/1.0/favorite' -> + with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc), do: [activity, favorited_activity] + _ -> + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + 'http://activitystrea.ms/schema/1.0/comment' -> + with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + _ -> + Logger.error("Couldn't parse incoming document") + nil + end + end + rescue + e -> + Logger.error("Error occured while handling activity") Logger.error(inspect(e)) nil - end - end) - |> Enum.filter(&(&1)) + end + end) + |> Enum.filter(&(&1)) - {:ok, activities} + {:ok, activities} + else + _e -> {:error, []} + end end def make_share(entry, doc, retweeted_activity) do @@ -111,7 +114,7 @@ def get_or_try_fetching(entry) do else e -> Logger.debug("Couldn't get, will try to fetch") with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), - {:ok, [favorited_activity]} <- fetch_activity_from_html_url(href) do + {:ok, [favorited_activity]} <- fetch_activity_from_url(href) do {:ok, favorited_activity} else e -> Logger.debug("Couldn't find href: #{inspect(e)}") end @@ -278,14 +281,30 @@ def get_atom_url(body) do end end - def fetch_activity_from_html_url(url) do - Logger.debug("Trying to fetch #{url}") - with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000), - {:ok, atom_url} <- get_atom_url(body), - {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do + def fetch_activity_from_atom_url(url) do + with {:ok, %{body: body, status_code: code}} when code in 200..299 <- @httpoison.get(url, [Accept: "application/atom+xml"], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") end end + + def fetch_activity_from_html_url(url) do + Logger.debug("Trying to fetch #{url}") + with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000), + {:ok, atom_url} <- get_atom_url(body) do + fetch_activity_from_atom_url(atom_url) + else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") + end + end + + def fetch_activity_from_url(url) do + with {:ok, activities} <- fetch_activity_from_atom_url(url) do + {:ok, activities} + else + _e -> with {:ok, activities} <- fetch_activity_from_html_url(url) do + {:ok, activities} + end + end + end end diff --git a/test/fixtures/httpoison_mock/sakamoto.atom b/test/fixtures/httpoison_mock/sakamoto.atom new file mode 100644 index 000000000..648946795 --- /dev/null +++ b/test/fixtures/httpoison_mock/sakamoto.atom @@ -0,0 +1 @@ +http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056New note by eal<a href='https://shitposter.club/user/5381'>@shpuld</a> <a href='https://pleroma.hjkos.com/users/hj'>@hj</a> IM NOT GAY DAD2017-08-04T12:51:26.130592Z2017-08-04T12:51:26.130592Zhttps://pleroma.hjkos.com/contexts/53093c74-2100-4bf4-aac6-66d1973d03efhttps://social.sakamoto.gq/users/ealhttp://activitystrea.ms/schema/1.0/personhttps://social.sakamoto.gq/users/ealeal坂本（・ヮ・）eal \ No newline at end of file diff --git a/test/fixtures/httpoison_mock/sakamoto_eal_feed.atom b/test/fixtures/httpoison_mock/sakamoto_eal_feed.atom new file mode 100644 index 000000000..9340d9038 --- /dev/null +++ b/test/fixtures/httpoison_mock/sakamoto_eal_feed.atom @@ -0,0 +1 @@ +https://social.sakamoto.gq/users/eal/feed.atomeal's timeline2017-08-04T14:19:12.683854https://social.sakamoto.gq/users/ealhttp://activitystrea.ms/schema/1.0/personhttps://social.sakamoto.gq/users/ealeal坂本（・ヮ・）ealhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/b79a1721-23f3-45a5-9610-adb08c2afae5New note by ealHonestly, I like all smileys that are not emoji.2017-08-04T14:19:12.675999Z2017-08-04T14:19:12.675999Zhttps://social.sakamoto.gq/contexts/e05ede92-8db9-4963-8b8e-e71a5797d68fhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/45475bf3-2dfc-4d9e-8eae-1f4f86f48982New note by ealThen again, I like all smileys/emoticons that are not emoji.<br>2017-08-04T14:19:10.113373Z2017-08-04T14:19:10.113373Zhttps://social.sakamoto.gq/contexts/852d1605-4dcb-4ba7-9ba4-dfc37ed62fbchttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/8f8fd6d6-cc63-40c6-a5d0-1c0e4f919368New note by ealI love the russian-style smiley.2017-08-04T14:18:30.478552Z2017-08-04T14:18:30.478552Zhttps://social.sakamoto.gq/contexts/852d1605-4dcb-4ba7-9ba4-dfc37ed62fbchttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/6e69df95-f2ad-4b8e-af4a-e93ff93d64e1eal started following https://cybre.space/users/0x3Feal started following https://cybre.space/users/0x3F2017-08-04T14:17:24.942193Z2017-08-04T14:17:24.942193Zhttp://activitystrea.ms/schema/1.0/personhttps://cybre.space/users/0x3Fhttps://cybre.space/users/0x3Fhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/54c5e260-0185-4267-a2a6-f5dd9c76c2c9eal started following https://niu.moe/users/ryeeal started following https://niu.moe/users/rye2017-08-04T14:16:35.604739Z2017-08-04T14:16:35.604739Zhttp://activitystrea.ms/schema/1.0/personhttps://niu.moe/users/ryehttps://niu.moe/users/ryehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/092ca863-19a8-416c-85d7-d3f23b3c0203eal started following https://mastodon.xyz/users/rafudesueal started following https://mastodon.xyz/users/rafudesu2017-08-04T14:16:10.993429Z2017-08-04T14:16:10.993429Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.xyz/users/rafudesuhttps://mastodon.xyz/users/rafudesuhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/be5cf702-b127-423b-a6be-5f78f01a4289eal started following https://gs.kawa-kun.com/user/2eal started following https://gs.kawa-kun.com/user/22017-08-04T14:15:41.804611Z2017-08-04T14:15:41.804611Zhttp://activitystrea.ms/schema/1.0/personhttps://gs.kawa-kun.com/user/2https://gs.kawa-kun.com/user/2http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/4951e2a1-9bae-4e87-8e98-e6d2f8a52338eal started following https://gs.kawa-kun.com/user/4885eal started following https://gs.kawa-kun.com/user/48852017-08-04T14:15:00.135352Z2017-08-04T14:15:00.135352Zhttp://activitystrea.ms/schema/1.0/personhttps://gs.kawa-kun.com/user/4885https://gs.kawa-kun.com/user/4885http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/cadf8745-b9ee-4f6c-af32-bfddb70e4607eal started following https://mastodon.social/users/Murassaeal started following https://mastodon.social/users/Murassa2017-08-04T14:14:36.339560Z2017-08-04T14:14:36.339560Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/Murassahttps://mastodon.social/users/Murassahttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/a52c9aab-f0e6-4ccb-8dd3-9f417e72a41ceal started following https://mastodon.social/users/rysiekeal started following https://mastodon.social/users/rysiek2017-08-04T14:13:04.061572Z2017-08-04T14:13:04.061572Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/rysiekhttps://mastodon.social/users/rysiekhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/738bc887-4cca-4b36-8c86-2b54d4c54732eal started following https://mastodon.hasameli.com/users/munineal started following https://mastodon.hasameli.com/users/munin2017-08-04T14:12:10.514155Z2017-08-04T14:12:10.514155Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.hasameli.com/users/muninhttps://mastodon.hasameli.com/users/muninhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/dc66ad5a-b776-4180-a8aa-e4c1bf7cb703eal started following https://cybre.space/users/nightpooleal started following https://cybre.space/users/nightpool2017-08-04T14:11:16.046148Z2017-08-04T14:11:16.046148Zhttp://activitystrea.ms/schema/1.0/personhttps://cybre.space/users/nightpoolhttps://cybre.space/users/nightpoolhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/9c5c00d7-3ce4-4c11-b965-dc5c2bda86c5New note by eal<a href='https://mastodon.zombocloud.com/users/staticsafe'>@staticsafe</a> privet )))2017-08-04T14:10:08.812247Z2017-08-04T14:10:08.812247Zhttps://social.sakamoto.gq/contexts/12a33823-0327-4c1c-a591-850ea79331b5http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/49798053-1f40-4a71-ad33-106e90630863eal started following https://social.homunyan.com/users/animeirleal started following https://social.homunyan.com/users/animeirl2017-08-04T14:09:44.904792Z2017-08-04T14:09:44.904792Zhttp://activitystrea.ms/schema/1.0/personhttps://social.homunyan.com/users/animeirlhttps://social.homunyan.com/users/animeirlhttp://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/2d83a1c5-70a6-45d3-9b84-59d6a70fbb17New favorite by ealeal favorited something2017-08-04T14:07:27.210044Z2017-08-04T14:07:27.210044Zhttp://activitystrea.ms/schema/1.0/notehttps://pleroma.soykaf.com/objects/b831e52f-4ed4-438e-95b4-888897f64f09https://pleroma.hjkos.com/contexts/3ed48205-1e72-4e19-a618-89a0d2ca811ehttp://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/06d28bed-544a-496b-8414-1c6d439273b5New favorite by ealeal favorited something2017-08-04T14:05:37.280200Z2017-08-04T14:05:37.280200Zhttp://activitystrea.ms/schema/1.0/notetag:toot-lab.reclaim.technology,2017-08-04:objectId=1166030:objectType=Statustag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/72bf19d4-9ad4-4b2f-9cd0-f0d70f4e931beal started following https://mstdn.jp/users/nullkaleal started following https://mstdn.jp/users/nullkal2017-08-04T14:05:04.148904Z2017-08-04T14:05:04.148904Zhttp://activitystrea.ms/schema/1.0/personhttps://mstdn.jp/users/nullkalhttps://mstdn.jp/users/nullkalhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/b0e89515-7621-4e09-b23d-83e192324107New note by eal<a href='https://p2px.me/user/1'>@stitchxd</a> test also2017-08-04T14:04:38.699051Z2017-08-04T14:04:38.699051Ztag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/d8d2006b-6b23-45d6-ba27-39d27587777dNew favorite by ealeal favorited something2017-08-04T14:04:32.106626Z2017-08-04T14:04:32.106626Zhttp://activitystrea.ms/schema/1.0/notetag:p2px.me,2017-08-04:noticeId=222109:objectType=notetag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/cb9db95d-ec27-41fa-bebd-5375fc13acb9eal started following https://mastodon.social/users/Gargroneal started following https://mastodon.social/users/Gargron2017-08-04T14:04:04.325531Z2017-08-04T14:04:04.325531Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/Gargronhttps://mastodon.social/users/Gargron \ No newline at end of file diff --git a/test/support/httpoison_mock.ex b/test/support/httpoison_mock.ex index 420d42a0d..08b566fcd 100644 --- a/test/support/httpoison_mock.ex +++ b/test/support/httpoison_mock.ex @@ -87,6 +87,13 @@ def get("https://pleroma.soykaf.com/users/lain/feed.atom", _body, _headers) do }} end + def get("https://social.sakamoto.gq/users/eal/feed.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/sakamoto_eal_feed.atom") + }} + end + def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _body, _headers) do {:ok, %Response{ status_code: 200, @@ -142,6 +149,13 @@ def get("https://mamot.fr/.well-known/webfinger", [Accept: "application/xrd+xml" }} end + def get("https://social.sakamoto.gq/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://social.sakamoto.gq/users/eal"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/eal_sakamoto.xml") + }} + end + def get("https://mamot.fr/users/Skruyb.atom", _, _) do {:ok, %Response{ status_code: 200, @@ -149,6 +163,14 @@ def get("https://mamot.fr/users/Skruyb.atom", _, _) do }} end + def get("https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", [Accept: "application/atom+xml"], _) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/sakamoto.atom") + }} + + end + def get(url, body, headers) do {:error, "Not implemented the mock response for get #{inspect(url)}, #{inspect(body)}, #{inspect(headers)}"} end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 959b744af..44d687d8e 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -326,11 +326,18 @@ test "it works with the uri" do describe "fetching a status by it's HTML url" do test "it builds a missing status from an html url" do url = "https://shitposter.club/notice/2827873" - {:ok, [activity] } = OStatus.fetch_activity_from_html_url(url) + {:ok, [activity] } = OStatus.fetch_activity_from_url(url) assert activity.data["actor"] == "https://shitposter.club/user/1" assert activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" end + + test "it works for atom notes, too" do + url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" + {:ok, [activity] } = OStatus.fetch_activity_from_url(url) + assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal" + assert activity.data["object"]["id"] == url + end end test "insert or update a user from given data" do