From 8141024259ee4bebd58d6ecd963f181aad420846 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 14:26:49 +0200 Subject: [PATCH] Attachment parsing, better magic key fetching. --- lib/pleroma/web/ostatus/ostatus.ex | 35 ++++++++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/salmon/salmon.ex | 15 ++--- lib/pleroma/web/web.ex | 22 +------ lib/pleroma/web/web_finger/web_finger.ex | 12 ++-- lib/pleroma/web/websub/websub.ex | 12 ++-- .../incoming_websub_gnusocial_attachments.xml | 59 +++++++++++++++++++ test/user_test.exs | 10 ++++ test/web/ostatus/ostatus_test.exs | 54 ++++++++++------- test/web/salmon/salmon_test.exs | 2 +- test/web/web_finger/web_finger_test.exs | 10 ++-- test/web/websub/websub_test.exs | 12 ++-- 12 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 test/fixtures/incoming_websub_gnusocial_attachments.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 7aa1ac4ac..f81751a25 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -39,6 +39,24 @@ def handle_incoming(xml_string) do {:ok, activities} end + def get_attachments(entry) do + :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) + |> Enum.map(fn (enclosure) -> + with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure), + type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do + %{ + "type" => "Attachment", + "url" => [%{ + "type" => "Link", + "mediaType" => type, + "href" => href + }] + } + end + end) + |> Enum.filter(&(&1)) + end + def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) @@ -48,6 +66,8 @@ def handle_note(entry, doc \\ nil) do context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim + attachments = get_attachments(entry) + context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do context else _e -> @@ -77,7 +97,8 @@ def handle_note(entry, doc \\ nil) do "content" => content_html, "published" => date, "context" => context, - "actor" => actor.ap_id + "actor" => actor.ap_id, + "attachment" => attachments } object = if inReplyTo do @@ -124,11 +145,11 @@ def make_user(uri) do with {:ok, info} <- gather_user_info(uri) do data = %{ local: false, - name: info.name, - nickname: info.nickname <> "@" <> info.host, - ap_id: info.uri, + name: info["name"], + nickname: info["nickname"] <> "@" <> info["host"], + ap_id: info["uri"], info: info, - avatar: info.avatar + avatar: info["avatar"] } # TODO: Make remote user changeset # SHould enforce fqn nickname @@ -158,8 +179,8 @@ def make_avatar_object(author_doc) do def gather_user_info(username) do with {:ok, webfinger_data} <- WebFinger.finger(username), - {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data.topic) do - {:ok, Map.merge(webfinger_data, feed_data) |> Map.put(:fqn, username)} + {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do + {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)} else e -> Logger.debug("Couldn't gather info for #{username}") {:error, e} diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index c6700ae78..1f2dedd30 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -33,7 +33,7 @@ def feed(conn, %{"nickname" => nickname}) do def salmon_incoming(conn, params) do {:ok, body, _conn} = read_body(conn) - magic_key = Pleroma.Web.Salmon.fetch_magic_key(body) + {:ok, magic_key} = Pleroma.Web.Salmon.fetch_magic_key(body) {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) Pleroma.Web.OStatus.handle_incoming(doc) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index b4f214d46..f02cb11dc 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -24,16 +24,13 @@ def decode(salmon) do [data, type, encoding, alg, sig] end - # TODO rewrite in with-stile - # Make it fetch the key from the saved user if there is one def fetch_magic_key(salmon) do - [data, _, _, _, _] = decode(salmon) - doc = XML.parse_document(data) - uri = XML.string_from_xpath("/entry/author[1]/uri", doc) - - {:ok, info} = Pleroma.Web.OStatus.gather_user_info(uri) - - info.magic_key + with [data, _, _, _, _] <- decode(salmon), + doc <- XML.parse_document(data), + uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc), + {:ok, %{info: %{"magic_key" => magic_key}}} <- Pleroma.Web.OStatus.find_or_make_user(uri) do + {:ok, magic_key} + end end def decode_and_validate(magickey, salmon) do diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index a81e3e6e1..ee7ee78e9 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -61,27 +61,7 @@ defmacro __using__(which) when is_atom(which) do apply(__MODULE__, which, []) end - def host do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - settings - |> Keyword.fetch!(:url) - |> Keyword.fetch!(:host) - end - def base_url do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - - host = host() - - protocol = settings |> Keyword.fetch!(:protocol) - - port_fragment = with {:ok, protocol_info} <- settings |> Keyword.fetch(String.to_atom(protocol)), - {:ok, port} <- protocol_info |> Keyword.fetch(:port) - do - ":#{port}" - else _e -> - "" - end - "#{protocol}://#{host}#{port_fragment}" + Pleroma.Web.Endpoint.url end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index d16bdd982..5fa69c2c8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -16,7 +16,7 @@ def host_meta() do end def webfinger(resource) do - host = Pleroma.Web.host + host = Pleroma.Web.Endpoint.host regex = ~r/(acct:)?(?\w+)@#{host}/ with %{"username" => username} <- Regex.named_captures(regex, resource) do user = User.get_by_nickname(username) @@ -37,7 +37,7 @@ def represent_user(user) do { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, + {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, @@ -72,10 +72,10 @@ defp webfinger_from_xml(doc) do subject = XML.string_from_xpath("//Subject", doc) salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc) data = %{ - magic_key: magic_key, - topic: topic, - subject: subject, - salmon: salmon + "magic_key" => magic_key, + "topic" => topic, + "subject" => subject, + "salmon" => salmon } {:ok, data} end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 546bfb5a4..e32fc8817 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -146,12 +146,12 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do avatar = OStatus.make_avatar_object(doc) {:ok, %{ - uri: uri, - hub: hub, - nickname: preferredUsername || name, - name: displayName || name, - host: URI.parse(uri).host, - avatar: avatar + "uri" => uri, + "hub" => hub, + "nickname" => preferredUsername || name, + "name" => displayName || name, + "host" => URI.parse(uri).host, + "avatar" => avatar }} else e -> {:error, e} diff --git a/test/fixtures/incoming_websub_gnusocial_attachments.xml b/test/fixtures/incoming_websub_gnusocial_attachments.xml new file mode 100644 index 000000000..9d331ef32 --- /dev/null +++ b/test/fixtures/incoming_websub_gnusocial_attachments.xml @@ -0,0 +1,59 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-02T20:29:35+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2020923:objectType=note + New note by lambadalambda + Okay gonna stream some cool games!! <a href="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" title="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-423842">https://social.heldscal.la/attachment/423842</a> <a href="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" title="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" rel="nofollow external noreferrer" class="attachment" id="attachment-423843">https://social.heldscal.la/attachment/423843</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T20:29:35+00:00 + 2017-05-02T20:29:35+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=26c7afdcbcf4ebd4 + + + + + + + + diff --git a/test/user_test.exs b/test/user_test.exs index 1331ac971..7435e30e0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -111,5 +111,15 @@ test "returns nil for nonexistant local user" do assert fetched_user == nil end end + + test "returns an ap_id for a user" do + user = insert(:user) + assert User.ap_id(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) + end + + test "returns an ap_followers link for a user" do + user = insert(:user) + assert User.ap_followers(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) <> "/followers" + end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index e39952807..94a735337 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -33,6 +33,16 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes with attachments - GS, subscription" do + incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["attachment"] |> length == 2 + end + test "handle incoming notes - Mastodon, salmon, reply" do # It uses the context of the replied to object Repo.insert!(%Object{ @@ -118,17 +128,17 @@ test "it returns user info in a hash" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "acct:shp@social.heldscal.la", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "acct:shp@social.heldscal.la", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end @@ -140,17 +150,17 @@ test "it works with the uri" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "https://social.heldscal.la/user/29191", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "https://social.heldscal.la/user/29191", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 77dacc1c0..ed26ccf83 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -55,7 +55,7 @@ test "encodes an xml payload with a private key" do test "it gets a magic key" do # TODO: Make test local salmon = File.read!("test/fixtures/salmon2.xml") - key = Salmon.fetch_magic_key(salmon) + {:ok, key} = Salmon.fetch_magic_key(salmon) assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index b48fdd0aa..495d3d50b 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -15,7 +15,7 @@ test "returns a link to the xml lrdd" do test "works for fqns" do user = insert(:user) - {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.host}") + {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host}") assert is_binary(result) end @@ -37,10 +37,10 @@ test "returns the info for a user" do {:ok, data} = WebFinger.finger(user, getter) - assert data.magic_key == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" - assert data.topic == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" - assert data.subject == "acct:shp@social.heldscal.la" - assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" + assert data["magic_key"] == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" + assert data["topic"] == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" + assert data["subject"] == "acct:shp@social.heldscal.la" + assert data["salmon"] == "https://social.heldscal.la/main/salmon/user/29191" end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 63acb3c43..065fb250a 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -115,12 +115,12 @@ test "discovers the hub and canonical url" do {:ok, discovered} = Websub.gather_feed_data(topic, getter) expected = %{ - hub: "https://mastodon.social/api/push", - uri: "https://mastodon.social/users/lambadalambda", - nickname: "lambadalambda", - name: "Critical Value", - host: "mastodon.social", - avatar: %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} + "hub" => "https://mastodon.social/api/push", + "uri" => "https://mastodon.social/users/lambadalambda", + "nickname" => "lambadalambda", + "name" => "Critical Value", + "host" => "mastodon.social", + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } assert expected == discovered