From 21afdf6d9966265de95df860d90d250c76bdbe08 Mon Sep 17 00:00:00 2001 From: raeno Date: Mon, 10 Dec 2018 16:25:33 +0400 Subject: [PATCH 01/61] Insert meta tags into static index.html on the fly for OStatus#notice --- lib/pleroma/web/oembed/oembed_controller.ex | 11 +++++++++++ lib/pleroma/web/ostatus/ostatus.ex | 15 +++++++++++++++ lib/pleroma/web/ostatus/ostatus_controller.ex | 14 +++++++++++--- lib/pleroma/web/router.ex | 10 ++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 lib/pleroma/web/oembed/oembed_controller.ex diff --git a/lib/pleroma/web/oembed/oembed_controller.ex b/lib/pleroma/web/oembed/oembed_controller.ex new file mode 100644 index 000000000..e9030049e --- /dev/null +++ b/lib/pleroma/web/oembed/oembed_controller.ex @@ -0,0 +1,11 @@ +defmodule Pleroma.Web.OEmbed.OEmbedController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + + def url(conn, %{ "url" => uri} ) do + conn + |> put_resp_content_type("application/json") + |> json(%{ status: "success"} ) + end +end \ No newline at end of file diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index c6440c20e..8ec92d5f1 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -26,6 +26,16 @@ def is_representable?(%Activity{data: data}) do end end + def metadata(url), do: oembed_links(url) + + def oembed_links(url) do + Enum.map(["xml", "json"], fn format -> + href = oembed_path(url, format) + " Enum.join("\r\n") + end + def feed_path(user) do "#{user.ap_id}/feed.atom" end @@ -42,6 +52,11 @@ def remote_follow_path do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end + def oembed_path(url, format) do + query = URI.encode_query(%{url: url, format: format}) + "#{Web.base_url()}/oembed?#{query}" + end + def handle_incoming(xml_string) do with doc when doc != :error <- parse_document(xml_string) do entries = :xmerl_xpath.string('//entry', doc) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 9dfcf0f95..635189619 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.ActivityPubController alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Router.Helpers, as: Routes plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) action_fallback(:errors) @@ -134,9 +135,7 @@ def notice(conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> - conn - |> put_resp_content_type("text/html") - |> send_file(200, Application.app_dir(:pleroma, "priv/static/index.html")) + serve_static_with_meta(conn, activity) _ -> represent_activity(conn, format, activity, user) @@ -153,6 +152,15 @@ def notice(conn, %{"id" => id}) do end end + defp serve_static_with_meta(conn, activity) do + {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) + links = OStatus.metadata(request_url(conn)) + response = String.replace(index_content, "", links) + conn + |> put_resp_content_type("text/html") + |> send_resp(200, response) + end + defp represent_activity( conn, "activity+json", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9c06fac4f..3239249f9 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -354,6 +354,10 @@ defmodule Pleroma.Web.Router do plug(:accepts, ["xml", "atom", "html", "activity+json"]) end + pipeline :oembed do + plug(:accepts, ["json", "xml"]) + end + scope "/", Pleroma.Web do pipe_through(:ostatus) @@ -369,6 +373,12 @@ defmodule Pleroma.Web.Router do post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming) end + scope "/", Pleroma.Web do + pipe_through(:oembed) + + get("/oembed", OEmbed.OEmbedController, :url) + end + pipeline :activitypub do plug(:accepts, ["activity+json"]) plug(Pleroma.Web.Plugs.HTTPSignaturePlug) From 8902942128e3aee814c215d700e2eaee21b491e9 Mon Sep 17 00:00:00 2001 From: raeno Date: Mon, 10 Dec 2018 23:08:02 +0400 Subject: [PATCH 02/61] WIP. Implement oembed route and handle both json/xml for "Note" type activity --- lib/pleroma/formatter.ex | 16 ++++++++++++ .../web/oembed/activity_representer.ex | 25 ++++++++++++++++++ lib/pleroma/web/oembed/oembed.ex | 23 ++++++++++++++++ lib/pleroma/web/oembed/oembed_controller.ex | 26 ++++++++++++++++--- lib/pleroma/web/oembed/views/note_view.ex | 21 +++++++++++++++ lib/pleroma/web/ostatus/ostatus.ex | 2 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 1 - .../web/templates/o_embed/note.xml.eex | 10 +++++++ test/web/oembed/oembed_test.exs | 23 ++++++++++++++++ 9 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 lib/pleroma/web/oembed/activity_representer.ex create mode 100644 lib/pleroma/web/oembed/oembed.ex create mode 100644 lib/pleroma/web/oembed/views/note_view.ex create mode 100644 lib/pleroma/web/templates/o_embed/note.xml.eex create mode 100644 test/web/oembed/oembed_test.exs diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 133683794..b63f592fb 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -163,4 +163,20 @@ def finalize({subs, text}) do String.replace(result_text, uuid, replacement) end) end + + def truncate(text, opts \\ []) do + max_length = opts[:max_length] || 200 + omission = opts[:omission] || "..." + + cond do + not String.valid?(text) -> + text + String.length(text) < max_length -> + text + true -> + length_with_omission = max_length - String.length(omission) + + "#{String.slice(text, 0, length_with_omission)}#{omission}" + end + end end diff --git a/lib/pleroma/web/oembed/activity_representer.ex b/lib/pleroma/web/oembed/activity_representer.ex new file mode 100644 index 000000000..0e65090ee --- /dev/null +++ b/lib/pleroma/web/oembed/activity_representer.ex @@ -0,0 +1,25 @@ +defmodule Pleroma.Web.OEmbed.ActivityRepresenter do + alias Pleroma.{Activity, User, Object} + alias Pleroma.Web.OStatus.UserRepresenter + + def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do + h = fn str -> [to_charlist(str)] end + + content = if activity.data["object"]["content"] do + [{:content, [], h.(activity.data["object"]["content"])}] + else + [] + end + + [ + {:version, ["1.0"]}, + {:type, ["link"]}, + ] ++ content + + end + + def wrap_with_entry(simple_form) do + [ { :oembed, [], simple_form } ] + end + +end diff --git a/lib/pleroma/web/oembed/oembed.ex b/lib/pleroma/web/oembed/oembed.ex new file mode 100644 index 000000000..a52aa6004 --- /dev/null +++ b/lib/pleroma/web/oembed/oembed.ex @@ -0,0 +1,23 @@ +defmodule Pleroma.Web.OEmbed do + alias Pleroma.{Repo, Object, Activity, User} + alias Pleroma.Formatter + + def recognize_path(url) do + details = Regex.named_captures(~r/.+\/(?.+)\/(?\w+).*$/, url) + + case details do + %{ "route" => "notice", "id" => id } -> + %{type: :activity, entity: Repo.get(Activity, id) } + %{ "route" => "users", "id" => nickname } -> + %{type: :user, entity: User.get_by_nickname(nickname) } + _ -> + { :error, "no matching route"} + end + end + + def truncated_content(activity) do + content = activity.data['object']['content'] + IO.puts(content) + Formatter.truncate(content) + end +end diff --git a/lib/pleroma/web/oembed/oembed_controller.ex b/lib/pleroma/web/oembed/oembed_controller.ex index e9030049e..d63d3c58c 100644 --- a/lib/pleroma/web/oembed/oembed_controller.ex +++ b/lib/pleroma/web/oembed/oembed_controller.ex @@ -1,11 +1,31 @@ defmodule Pleroma.Web.OEmbed.OEmbedController do use Pleroma.Web, :controller + alias Pleroma.Web.OEmbed + alias Pleroma.Web.OEmbed.{NoteView, ActivityRepresenter} + alias Pleroma.Web.MediaProxy alias Pleroma.Repo + alias Pleroma.User - def url(conn, %{ "url" => uri} ) do + def url(conn, %{ "url" => url} ) do + case format = get_format(conn) do + _ -> + result = OEmbed.recognize_path(url) + render_oembed(conn, format, result) + end + end + + def render_oembed(conn, format \\ "json", result) + def render_oembed(conn, "json", result) do conn |> put_resp_content_type("application/json") - |> json(%{ status: "success"} ) + |> json(NoteView.render("note.json", result)) end -end \ No newline at end of file + + def render_oembed(conn, "xml", result) do + conn + |> put_resp_content_type("application/xml") + |> NoteView.render("note.json", result) + + end +end diff --git a/lib/pleroma/web/oembed/views/note_view.ex b/lib/pleroma/web/oembed/views/note_view.ex new file mode 100644 index 000000000..ecdabc04b --- /dev/null +++ b/lib/pleroma/web/oembed/views/note_view.ex @@ -0,0 +1,21 @@ +defmodule Pleroma.Web.OEmbed.NoteView do + use Pleroma.Web, :view + alias Pleroma.{User, Activity} + alias Pleroma.Web.OEmbed + + def render("note.json", %{type: type, entity: activity }) do + oembed_data(activity) + end + + def oembed_data(activity) do + with %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]), + image = User.avatar_url(user) |> MediaProxy.url() do + %{ + version: "1.0", + type: "link", + title: OEmbed.truncated_content(activity), + provider_url: "https://pleroma.site", + thumbnail_url: image, + } + end +end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 8ec92d5f1..0f6756d16 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -31,7 +31,7 @@ def metadata(url), do: oembed_links(url) def oembed_links(url) do Enum.map(["xml", "json"], fn format -> href = oembed_path(url, format) - "" end) |> Enum.join("\r\n") end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 635189619..27ec24f57 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.ActivityPubController alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Router.Helpers, as: Routes plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) action_fallback(:errors) diff --git a/lib/pleroma/web/templates/o_embed/note.xml.eex b/lib/pleroma/web/templates/o_embed/note.xml.eex new file mode 100644 index 000000000..e814a4cb3 --- /dev/null +++ b/lib/pleroma/web/templates/o_embed/note.xml.eex @@ -0,0 +1,10 @@ + + + 1.0 + link + raeno + http://iamcal.com/ + 86400 + iamcal.com + http://iamcal.com/ + \ No newline at end of file diff --git a/test/web/oembed/oembed_test.exs b/test/web/oembed/oembed_test.exs new file mode 100644 index 000000000..16c2c3598 --- /dev/null +++ b/test/web/oembed/oembed_test.exs @@ -0,0 +1,23 @@ +defmodule Pleroma.Web.OEmbedTest do + use Pleroma.DataCase + alias Pleroma.Web.OEmbed + alias Pleroma.Web.XML + alias Pleroma.{Object, Repo, User, Activity} + import Pleroma.Factory + import ExUnit.CaptureLog + + setup_all do + :ok + end + + test 'recognizes notices in given url' do + url = "https://pleroma.site/notice/5" + assert { :activity, _ } = OEmbed.recognize_path(url) + end + + test 'recognizes user card in given url' do + url = "https://pleroma.site/users/raeno" + assert { :user, _ } = OEmbed.recognize_path(url) + end + +end From d903e34cacd18157a2a60e7b112eb05af2766266 Mon Sep 17 00:00:00 2001 From: raeno Date: Wed, 12 Dec 2018 22:37:00 +0300 Subject: [PATCH 03/61] Add opengraph/twitter_card:summary support. Add config to toggle on/off specific metadata --- config/config.exs | 4 ++ lib/pleroma/web/ostatus/ostatus.ex | 58 +++++++++++++++++-- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 +- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1401b0a3d..ad4b79f54 100644 --- a/config/config.exs +++ b/config/config.exs @@ -171,6 +171,10 @@ ip: {0, 0, 0, 0}, port: 9999 +config :pleroma, :metadata, + oembed: false, + opengraph: true + config :pleroma, :suggestions, enabled: false, third_party_engine: diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 0f6756d16..73cdd3837 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -5,11 +5,12 @@ defmodule Pleroma.Web.OStatus do import Pleroma.Web.XML require Logger - alias Pleroma.{Repo, User, Web, Object, Activity} + alias Pleroma.{Repo, User, Web, Object, Activity, Formatter} alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.{WebFinger, Websub} + alias Pleroma.Web.{WebFinger, Websub, MediaProxy} alias Pleroma.Web.OStatus.{FollowHandler, UnfollowHandler, NoteHandler, DeleteHandler} alias Pleroma.Web.ActivityPub.Transmogrifier + alias Phoenix.HTML def is_representable?(%Activity{data: data}) do object = Object.normalize(data["object"]) @@ -26,14 +27,59 @@ def is_representable?(%Activity{data: data}) do end end - def metadata(url), do: oembed_links(url) + def metadata(activity, user, url) do + Enum.concat([ + if(meta_enabled?(:opengraph), do: opengraph_tags(activity, user), else: []), + if(meta_enabled?(:oembed), do: oembed_links(url), else: []) + ]) + |> Enum.map(&to_tag/1) + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.join("\n") + end + + def meta_enabled?(type) do + config = Pleroma.Config.get(:metadata, []) + Keyword.get(config, type, false) + end + + def to_tag(data) do + with {name, attrs, _content = []} <- data do + HTML.Tag.tag(name, attrs) + else + {name, attrs, content} -> + HTML.Tag.content_tag(name, content, attrs) + + _ -> + raise ArgumentError, message: "make_tag invalid args" + end + end def oembed_links(url) do Enum.map(["xml", "json"], fn format -> - href = oembed_path(url, format) - "" + href = HTML.raw(oembed_path(url, format)) + { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } end) - |> Enum.join("\r\n") + end + + def opengraph_tags(activity, user) do + with image = User.avatar_url(user) |> MediaProxy.url(), + truncated_content = Formatter.truncate(activity.data["object"]["content"]), + domain = Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") do + [ + {:meta, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{domain}) post ##{activity.id}" + ], []}, + {:meta, [property: "og:url", content: activity.data["id"]], []}, + {:meta, [property: "og:description", content: truncated_content], + []}, + {:meta, [property: "og:image", content: image], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end end def feed_path(user) do diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 27ec24f57..bfc36c0c4 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -134,7 +134,7 @@ def notice(conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> - serve_static_with_meta(conn, activity) + serve_static_with_meta(conn, activity, user) _ -> represent_activity(conn, format, activity, user) @@ -151,9 +151,9 @@ def notice(conn, %{"id" => id}) do end end - defp serve_static_with_meta(conn, activity) do + defp serve_static_with_meta(conn, activity, user) do {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) - links = OStatus.metadata(request_url(conn)) + links = OStatus.metadata(activity, user, request_url(conn)) response = String.replace(index_content, "", links) conn |> put_resp_content_type("text/html") From 9b3a6cdb07383631c9ac11ce5de51306f6c4a464 Mon Sep 17 00:00:00 2001 From: raeno Date: Wed, 12 Dec 2018 22:55:01 +0300 Subject: [PATCH 04/61] Extract opengraph/oembed into separate module --- lib/pleroma/web/ostatus/metadata.ex | 66 +++++++++++++++++++ lib/pleroma/web/ostatus/ostatus.ex | 63 +----------------- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 +- 3 files changed, 70 insertions(+), 65 deletions(-) create mode 100644 lib/pleroma/web/ostatus/metadata.ex diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/ostatus/metadata.ex new file mode 100644 index 000000000..dd099e2ad --- /dev/null +++ b/lib/pleroma/web/ostatus/metadata.ex @@ -0,0 +1,66 @@ +defmodule Pleroma.Web.Metadata do + alias Phoenix.HTML + alias Pleroma.{Web, Formatter} + alias Pleroma.{User, Activity} + alias Pleroma.Web.MediaProxy + + def build_tags(activity, user, url) do + Enum.concat([ + if(meta_enabled?(:opengraph), do: opengraph_tags(activity, user), else: []), + if(meta_enabled?(:oembed), do: oembed_links(url), else: []) + ]) + |> Enum.map(&to_tag/1) + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.join("\n") + end + + def meta_enabled?(type) do + config = Pleroma.Config.get(:metadata, []) + Keyword.get(config, type, false) + end + + def to_tag(data) do + with {name, attrs, _content = []} <- data do + HTML.Tag.tag(name, attrs) + else + {name, attrs, content} -> + HTML.Tag.content_tag(name, content, attrs) + + _ -> + raise ArgumentError, message: "make_tag invalid args" + end + end + + defp oembed_links(url) do + Enum.map(["xml", "json"], fn format -> + href = HTML.raw(oembed_path(url, format)) + { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } + end) + end + + defp opengraph_tags(activity, user) do + with image = User.avatar_url(user) |> MediaProxy.url(), + truncated_content = Formatter.truncate(activity.data["object"]["content"]), + domain = Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") do + [ + {:meta, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{domain}) post ##{activity.id}" + ], []}, + {:meta, [property: "og:url", content: activity.data["id"]], []}, + {:meta, [property: "og:description", content: truncated_content], + []}, + {:meta, [property: "og:image", content: image], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + end + + defp oembed_path(url, format) do + query = URI.encode_query(%{url: url, format: format}) + "#{Web.base_url()}/oembed?#{query}" + end +end \ No newline at end of file diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 73cdd3837..ba44b2e99 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -7,10 +7,9 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.{Repo, User, Web, Object, Activity, Formatter} alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.{WebFinger, Websub, MediaProxy} + alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.OStatus.{FollowHandler, UnfollowHandler, NoteHandler, DeleteHandler} alias Pleroma.Web.ActivityPub.Transmogrifier - alias Phoenix.HTML def is_representable?(%Activity{data: data}) do object = Object.normalize(data["object"]) @@ -27,61 +26,6 @@ def is_representable?(%Activity{data: data}) do end end - def metadata(activity, user, url) do - Enum.concat([ - if(meta_enabled?(:opengraph), do: opengraph_tags(activity, user), else: []), - if(meta_enabled?(:oembed), do: oembed_links(url), else: []) - ]) - |> Enum.map(&to_tag/1) - |> Enum.map(&HTML.safe_to_string/1) - |> Enum.join("\n") - end - - def meta_enabled?(type) do - config = Pleroma.Config.get(:metadata, []) - Keyword.get(config, type, false) - end - - def to_tag(data) do - with {name, attrs, _content = []} <- data do - HTML.Tag.tag(name, attrs) - else - {name, attrs, content} -> - HTML.Tag.content_tag(name, content, attrs) - - _ -> - raise ArgumentError, message: "make_tag invalid args" - end - end - - def oembed_links(url) do - Enum.map(["xml", "json"], fn format -> - href = HTML.raw(oembed_path(url, format)) - { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } - end) - end - - def opengraph_tags(activity, user) do - with image = User.avatar_url(user) |> MediaProxy.url(), - truncated_content = Formatter.truncate(activity.data["object"]["content"]), - domain = Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") do - [ - {:meta, - [ - property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{domain}) post ##{activity.id}" - ], []}, - {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], - []}, - {:meta, [property: "og:image", content: image], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, - {:meta, [property: "twitter:card", content: "summary"], []} - ] - end - end - def feed_path(user) do "#{user.ap_id}/feed.atom" end @@ -98,11 +42,6 @@ def remote_follow_path do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end - def oembed_path(url, format) do - query = URI.encode_query(%{url: url, format: format}) - "#{Web.base_url()}/oembed?#{query}" - end - def handle_incoming(xml_string) do with doc when doc != :error <- parse_document(xml_string) do entries = :xmerl_xpath.string('//entry', doc) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index bfc36c0c4..23ca8e088 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity, Object} alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} alias Pleroma.Repo - alias Pleroma.Web.{OStatus, Federator} + alias Pleroma.Web.{OStatus, Federator, Metadata} alias Pleroma.Web.XML alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.ActivityPubController @@ -153,8 +153,8 @@ def notice(conn, %{"id" => id}) do defp serve_static_with_meta(conn, activity, user) do {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) - links = OStatus.metadata(activity, user, request_url(conn)) - response = String.replace(index_content, "", links) + tags = Metadata.build_tags(activity, user, request_url(conn)) + response = String.replace(index_content, "", tags) conn |> put_resp_content_type("text/html") |> send_resp(200, response) From 49c4f7d604be921bf5c6fb966b279fbb037b6cfa Mon Sep 17 00:00:00 2001 From: raeno Date: Wed, 12 Dec 2018 22:57:01 +0300 Subject: [PATCH 05/61] Set Pleroma.instance.domain when config is generated --- lib/mix/tasks/pleroma/sample_config.eex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mix/tasks/pleroma/sample_config.eex b/lib/mix/tasks/pleroma/sample_config.eex index 0cd572797..8eacefab2 100644 --- a/lib/mix/tasks/pleroma/sample_config.eex +++ b/lib/mix/tasks/pleroma/sample_config.eex @@ -12,6 +12,7 @@ config :pleroma, Pleroma.Web.Endpoint, config :pleroma, :instance, name: "<%= name %>", email: "<%= email %>", + domain: "<%= domain %>, limit: 5000, registrations_open: true, dedupe_media: false From 018516d3f3fcc6e902a5382517792832841695d3 Mon Sep 17 00:00:00 2001 From: raeno Date: Thu, 13 Dec 2018 22:13:02 +0100 Subject: [PATCH 06/61] Refactor ostatus_controller, extract metatags redirection to Redirector itself. Set 'html' as default type for ostatus links --- lib/pleroma/web/ostatus/metadata.ex | 88 ++++++++++++------- lib/pleroma/web/ostatus/ostatus_controller.ex | 21 ++--- lib/pleroma/web/router.ex | 13 ++- 3 files changed, 77 insertions(+), 45 deletions(-) diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/ostatus/metadata.ex index dd099e2ad..4f319b360 100644 --- a/lib/pleroma/web/ostatus/metadata.ex +++ b/lib/pleroma/web/ostatus/metadata.ex @@ -4,10 +4,10 @@ defmodule Pleroma.Web.Metadata do alias Pleroma.{User, Activity} alias Pleroma.Web.MediaProxy - def build_tags(activity, user, url) do + def build_tags(request_url, params) do Enum.concat([ - if(meta_enabled?(:opengraph), do: opengraph_tags(activity, user), else: []), - if(meta_enabled?(:oembed), do: oembed_links(url), else: []) + if(meta_enabled?(:opengraph), do: opengraph_tags(params), else: []), + if(meta_enabled?(:oembed), do: oembed_links(request_url), else: []) ]) |> Enum.map(&to_tag/1) |> Enum.map(&HTML.safe_to_string/1) @@ -19,6 +19,52 @@ def meta_enabled?(type) do Keyword.get(config, type, false) end + # opengraph for single status + defp opengraph_tags(%{activity: activity, user: user}) do + with truncated_content = Formatter.truncate(activity.data["object"]["content"]) do + [ + {:meta, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) post ##{activity.id}" + ], []}, + {:meta, [property: "og:url", content: activity.data["id"]], []}, + {:meta, [property: "og:description", content: truncated_content], + []}, + {:meta, [property: "og:image", content: user_avatar_url(user)], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + end + + # opengraph for user card + defp opengraph_tags(%{user: user}) do + with truncated_bio = Formatter.truncate(user.bio) do + [ + {:meta, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) profile" + ], []}, + {:meta, [property: "og:url", content: User.profile_url(user)], []}, + {:meta, [property: "og:description", content: truncated_bio], []}, + {:meta, [property: "og:image", content: user_avatar_url(user)], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + end + + defp oembed_links(url) do + Enum.map(["xml", "json"], fn format -> + href = HTML.raw(oembed_path(url, format)) + { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } + end) + end + def to_tag(data) do with {name, attrs, _content = []} <- data do HTML.Tag.tag(name, attrs) @@ -31,36 +77,16 @@ def to_tag(data) do end end - defp oembed_links(url) do - Enum.map(["xml", "json"], fn format -> - href = HTML.raw(oembed_path(url, format)) - { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } - end) - end - - defp opengraph_tags(activity, user) do - with image = User.avatar_url(user) |> MediaProxy.url(), - truncated_content = Formatter.truncate(activity.data["object"]["content"]), - domain = Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") do - [ - {:meta, - [ - property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{domain}) post ##{activity.id}" - ], []}, - {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], - []}, - {:meta, [property: "og:image", content: image], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, - {:meta, [property: "twitter:card", content: "summary"], []} - ] - end - end - defp oembed_path(url, format) do query = URI.encode_query(%{url: url, format: format}) "#{Web.base_url()}/oembed?#{query}" end + + defp user_avatar_url(user) do + User.avatar_url(user) |> MediaProxy.url() + end + + def pleroma_domain do + Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") + end end \ No newline at end of file diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 23ca8e088..fbd25c5f5 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity, Object} alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} alias Pleroma.Repo - alias Pleroma.Web.{OStatus, Federator, Metadata} + alias Pleroma.Web.{OStatus, Federator} alias Pleroma.Web.XML alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.ActivityPubController @@ -14,9 +14,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do action_fallback(:errors) def feed_redirect(conn, %{"nickname" => nickname}) do - case get_format(conn) do + format = get_format(conn) + IO.puts(format) + case format do "html" -> - Fallback.RedirectController.redirector(conn, nil) + with %User{} = user <- User.get_cached_by_nickname(nickname) do + Fallback.RedirectController.redirector_with_meta(conn, %{user: user}) + end "activity+json" -> ActivityPubController.call(conn, :user) @@ -134,7 +138,7 @@ def notice(conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> - serve_static_with_meta(conn, activity, user) + Fallback.RedirectController.redirector_with_meta(conn, %{activity: activity, user: user}) _ -> represent_activity(conn, format, activity, user) @@ -151,15 +155,6 @@ def notice(conn, %{"id" => id}) do end end - defp serve_static_with_meta(conn, activity, user) do - {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) - tags = Metadata.build_tags(activity, user, request_url(conn)) - response = String.replace(index_content, "", tags) - conn - |> put_resp_content_type("text/html") - |> send_resp(200, response) - end - defp represent_activity( conn, "activity+json", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3239249f9..ad26093ac 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -351,7 +351,7 @@ defmodule Pleroma.Web.Router do end pipeline :ostatus do - plug(:accepts, ["xml", "atom", "html", "activity+json"]) + plug(:accepts, ["html", "xml", "atom", "activity+json"]) end pipeline :oembed do @@ -444,6 +444,7 @@ defmodule Pleroma.Web.Router do defmodule Fallback.RedirectController do use Pleroma.Web, :controller + alias Pleroma.Web.Metadata def redirector(conn, _params) do conn @@ -451,6 +452,16 @@ def redirector(conn, _params) do |> send_file(200, Application.app_dir(:pleroma, "priv/static/index.html")) end + def redirector_with_meta(conn, params) do + {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) + tags = Metadata.build_tags(request_url(conn), params) + response = String.replace(index_content, "", tags) + + conn + |> put_resp_content_type("text/html") + |> send_resp(200, response) + end + def registration_page(conn, params) do redirector(conn, params) end From b5de7c4c4d90d1049b59381953cbd76e79b77e66 Mon Sep 17 00:00:00 2001 From: raeno Date: Thu, 13 Dec 2018 22:16:54 +0100 Subject: [PATCH 07/61] Remove oembed for now, will submit it in another MR. Fix warnings --- config/config.exs | 4 +- lib/pleroma/formatter.ex | 6 ++- .../web/oembed/activity_representer.ex | 25 ----------- lib/pleroma/web/oembed/oembed.ex | 23 ---------- lib/pleroma/web/oembed/oembed_controller.ex | 31 ------------- lib/pleroma/web/oembed/views/note_view.ex | 21 --------- lib/pleroma/web/ostatus/metadata.ex | 43 ++++++------------- lib/pleroma/web/ostatus/ostatus.ex | 2 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 4 +- lib/pleroma/web/router.ex | 4 +- test/web/oembed/oembed_test.exs | 23 ---------- 11 files changed, 22 insertions(+), 164 deletions(-) delete mode 100644 lib/pleroma/web/oembed/activity_representer.ex delete mode 100644 lib/pleroma/web/oembed/oembed.ex delete mode 100644 lib/pleroma/web/oembed/oembed_controller.ex delete mode 100644 lib/pleroma/web/oembed/views/note_view.ex delete mode 100644 test/web/oembed/oembed_test.exs diff --git a/config/config.exs b/config/config.exs index ad4b79f54..5477c0c0e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -171,9 +171,7 @@ ip: {0, 0, 0, 0}, port: 9999 -config :pleroma, :metadata, - oembed: false, - opengraph: true +config :pleroma, :metadata, opengraph: true config :pleroma, :suggestions, enabled: false, diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index b63f592fb..1d5dc0c99 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -165,14 +165,16 @@ def finalize({subs, text}) do end def truncate(text, opts \\ []) do - max_length = opts[:max_length] || 200 - omission = opts[:omission] || "..." + max_length = opts[:max_length] || 200 + omission = opts[:omission] || "..." cond do not String.valid?(text) -> text + String.length(text) < max_length -> text + true -> length_with_omission = max_length - String.length(omission) diff --git a/lib/pleroma/web/oembed/activity_representer.ex b/lib/pleroma/web/oembed/activity_representer.ex deleted file mode 100644 index 0e65090ee..000000000 --- a/lib/pleroma/web/oembed/activity_representer.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Pleroma.Web.OEmbed.ActivityRepresenter do - alias Pleroma.{Activity, User, Object} - alias Pleroma.Web.OStatus.UserRepresenter - - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - content = if activity.data["object"]["content"] do - [{:content, [], h.(activity.data["object"]["content"])}] - else - [] - end - - [ - {:version, ["1.0"]}, - {:type, ["link"]}, - ] ++ content - - end - - def wrap_with_entry(simple_form) do - [ { :oembed, [], simple_form } ] - end - -end diff --git a/lib/pleroma/web/oembed/oembed.ex b/lib/pleroma/web/oembed/oembed.ex deleted file mode 100644 index a52aa6004..000000000 --- a/lib/pleroma/web/oembed/oembed.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Pleroma.Web.OEmbed do - alias Pleroma.{Repo, Object, Activity, User} - alias Pleroma.Formatter - - def recognize_path(url) do - details = Regex.named_captures(~r/.+\/(?.+)\/(?\w+).*$/, url) - - case details do - %{ "route" => "notice", "id" => id } -> - %{type: :activity, entity: Repo.get(Activity, id) } - %{ "route" => "users", "id" => nickname } -> - %{type: :user, entity: User.get_by_nickname(nickname) } - _ -> - { :error, "no matching route"} - end - end - - def truncated_content(activity) do - content = activity.data['object']['content'] - IO.puts(content) - Formatter.truncate(content) - end -end diff --git a/lib/pleroma/web/oembed/oembed_controller.ex b/lib/pleroma/web/oembed/oembed_controller.ex deleted file mode 100644 index d63d3c58c..000000000 --- a/lib/pleroma/web/oembed/oembed_controller.ex +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Pleroma.Web.OEmbed.OEmbedController do - use Pleroma.Web, :controller - - alias Pleroma.Web.OEmbed - alias Pleroma.Web.OEmbed.{NoteView, ActivityRepresenter} - alias Pleroma.Web.MediaProxy - alias Pleroma.Repo - alias Pleroma.User - - def url(conn, %{ "url" => url} ) do - case format = get_format(conn) do - _ -> - result = OEmbed.recognize_path(url) - render_oembed(conn, format, result) - end - end - - def render_oembed(conn, format \\ "json", result) - def render_oembed(conn, "json", result) do - conn - |> put_resp_content_type("application/json") - |> json(NoteView.render("note.json", result)) - end - - def render_oembed(conn, "xml", result) do - conn - |> put_resp_content_type("application/xml") - |> NoteView.render("note.json", result) - - end -end diff --git a/lib/pleroma/web/oembed/views/note_view.ex b/lib/pleroma/web/oembed/views/note_view.ex deleted file mode 100644 index ecdabc04b..000000000 --- a/lib/pleroma/web/oembed/views/note_view.ex +++ /dev/null @@ -1,21 +0,0 @@ -defmodule Pleroma.Web.OEmbed.NoteView do - use Pleroma.Web, :view - alias Pleroma.{User, Activity} - alias Pleroma.Web.OEmbed - - def render("note.json", %{type: type, entity: activity }) do - oembed_data(activity) - end - - def oembed_data(activity) do - with %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]), - image = User.avatar_url(user) |> MediaProxy.url() do - %{ - version: "1.0", - type: "link", - title: OEmbed.truncated_content(activity), - provider_url: "https://pleroma.site", - thumbnail_url: image, - } - end -end diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/ostatus/metadata.ex index 4f319b360..792b4a4bd 100644 --- a/lib/pleroma/web/ostatus/metadata.ex +++ b/lib/pleroma/web/ostatus/metadata.ex @@ -1,14 +1,10 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML - alias Pleroma.{Web, Formatter} - alias Pleroma.{User, Activity} + alias Pleroma.{Formatter, User} alias Pleroma.Web.MediaProxy - def build_tags(request_url, params) do - Enum.concat([ - if(meta_enabled?(:opengraph), do: opengraph_tags(params), else: []), - if(meta_enabled?(:oembed), do: oembed_links(request_url), else: []) - ]) + def build_tags(params) do + if(meta_enabled?(:opengraph), do: opengraph_tags(params), else: []) |> Enum.map(&to_tag/1) |> Enum.map(&HTML.safe_to_string/1) |> Enum.join("\n") @@ -24,13 +20,12 @@ defp opengraph_tags(%{activity: activity, user: user}) do with truncated_content = Formatter.truncate(activity.data["object"]["content"]) do [ {:meta, - [ - property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) post ##{activity.id}" - ], []}, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) post ##{activity.id}" + ], []}, {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], - []}, + {:meta, [property: "og:description", content: truncated_content], []}, {:meta, [property: "og:image", content: user_avatar_url(user)], []}, {:meta, [property: "og:image:width", content: 120], []}, {:meta, [property: "og:image:height", content: 120], []}, @@ -44,10 +39,10 @@ defp opengraph_tags(%{user: user}) do with truncated_bio = Formatter.truncate(user.bio) do [ {:meta, - [ - property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) profile" - ], []}, + [ + property: "og:title", + content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) profile" + ], []}, {:meta, [property: "og:url", content: User.profile_url(user)], []}, {:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:image", content: user_avatar_url(user)], []}, @@ -58,13 +53,6 @@ defp opengraph_tags(%{user: user}) do end end - defp oembed_links(url) do - Enum.map(["xml", "json"], fn format -> - href = HTML.raw(oembed_path(url, format)) - { :link, [ type: ["application/#{format}+oembed"], href: href, rel: 'alternate'], [] } - end) - end - def to_tag(data) do with {name, attrs, _content = []} <- data do HTML.Tag.tag(name, attrs) @@ -77,11 +65,6 @@ def to_tag(data) do end end - defp oembed_path(url, format) do - query = URI.encode_query(%{url: url, format: format}) - "#{Web.base_url()}/oembed?#{query}" - end - defp user_avatar_url(user) do User.avatar_url(user) |> MediaProxy.url() end @@ -89,4 +72,4 @@ defp user_avatar_url(user) do def pleroma_domain do Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") end -end \ No newline at end of file +end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index ba44b2e99..c6440c20e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.OStatus do import Pleroma.Web.XML require Logger - alias Pleroma.{Repo, User, Web, Object, Activity, Formatter} + alias Pleroma.{Repo, User, Web, Object, Activity} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.{WebFinger, Websub} alias Pleroma.Web.OStatus.{FollowHandler, UnfollowHandler, NoteHandler, DeleteHandler} diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index fbd25c5f5..55dbcab93 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -14,9 +14,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do action_fallback(:errors) def feed_redirect(conn, %{"nickname" => nickname}) do - format = get_format(conn) - IO.puts(format) - case format do + case get_format(conn) do "html" -> with %User{} = user <- User.get_cached_by_nickname(nickname) do Fallback.RedirectController.redirector_with_meta(conn, %{user: user}) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ad26093ac..914cd6a6d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -453,8 +453,8 @@ def redirector(conn, _params) do end def redirector_with_meta(conn, params) do - {:ok, index_content } = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) - tags = Metadata.build_tags(request_url(conn), params) + {:ok, index_content} = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) + tags = Metadata.build_tags(params) response = String.replace(index_content, "", tags) conn diff --git a/test/web/oembed/oembed_test.exs b/test/web/oembed/oembed_test.exs deleted file mode 100644 index 16c2c3598..000000000 --- a/test/web/oembed/oembed_test.exs +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Pleroma.Web.OEmbedTest do - use Pleroma.DataCase - alias Pleroma.Web.OEmbed - alias Pleroma.Web.XML - alias Pleroma.{Object, Repo, User, Activity} - import Pleroma.Factory - import ExUnit.CaptureLog - - setup_all do - :ok - end - - test 'recognizes notices in given url' do - url = "https://pleroma.site/notice/5" - assert { :activity, _ } = OEmbed.recognize_path(url) - end - - test 'recognizes user card in given url' do - url = "https://pleroma.site/users/raeno" - assert { :user, _ } = OEmbed.recognize_path(url) - end - -end From c5c3ad90d00f1ea599e489aa03cfecada44c998c Mon Sep 17 00:00:00 2001 From: raeno Date: Fri, 14 Dec 2018 02:59:33 +0100 Subject: [PATCH 08/61] Fix tests. Remove oembed template --- .../web/templates/o_embed/note.xml.eex | 10 --------- test/web/ostatus/ostatus_controller_test.exs | 21 +++++++++++++++++-- 2 files changed, 19 insertions(+), 12 deletions(-) delete mode 100644 lib/pleroma/web/templates/o_embed/note.xml.eex diff --git a/lib/pleroma/web/templates/o_embed/note.xml.eex b/lib/pleroma/web/templates/o_embed/note.xml.eex deleted file mode 100644 index e814a4cb3..000000000 --- a/lib/pleroma/web/templates/o_embed/note.xml.eex +++ /dev/null @@ -1,10 +0,0 @@ - - - 1.0 - link - raeno - http://iamcal.com/ - 86400 - iamcal.com - http://iamcal.com/ - \ No newline at end of file diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 560305c15..747e30154 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -84,6 +84,7 @@ test "gets an object", %{conn: conn} do conn = conn + |> put_req_header("accept", "application/xml") |> get(url) expected = @@ -110,11 +111,12 @@ test "404s on nonexisting objects", %{conn: conn} do |> response(404) end - test "gets an activity", %{conn: conn} do + test "gets an activity in xml format", %{conn: conn} do note_activity = insert(:note_activity) [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) conn + |> put_req_header("accept", "application/xml") |> get("/activities/#{uuid}") |> response(200) end @@ -134,7 +136,22 @@ test "404s on nonexistent activities", %{conn: conn} do |> response(404) end - test "gets a notice", %{conn: conn} do + test "renders notice metatags in html format" do + note_activity = insert(:note_activity) + + conn = get(conn, "/notice/#{note_activity.id}") + + twitter_card_summary = "" + + description_content = + "" + + body = html_response(conn, 200) + assert body =~ twitter_card_summary + assert body =~ description_content + end + + test "gets a notice in xml format", %{conn: conn} do note_activity = insert(:note_activity) conn From 46486595ff96e0af0c3193e60879a8e67f77b933 Mon Sep 17 00:00:00 2001 From: raeno Date: Fri, 14 Dec 2018 19:55:40 +0100 Subject: [PATCH 09/61] Handle "users/:id" links as well. Fix comments in MR. --- lib/pleroma/user.ex | 13 +++++++++++++ lib/pleroma/web/ostatus/metadata.ex | 5 ++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 4 +++- lib/pleroma/web/router.ex | 8 ++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 49928bc13..28ff08a39 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -294,6 +294,10 @@ def locked?(%User{} = user) do user.info.locked || false end + def get_by_id(id) do + Repo.get_by(User, id: id) + end + def get_by_ap_id(ap_id) do Repo.get_by(User, ap_id: ap_id) end @@ -320,11 +324,20 @@ def get_cached_by_ap_id(ap_id) do Cachex.fetch!(:user_cache, key, fn _ -> get_by_ap_id(ap_id) end) end + def get_cached_by_id(id) do + key = "id:#{id}" + Cachex.fetch!(:user_cache, key, fn _ -> get_by_id(id) end) + end + def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end) end + def get_cached_by_nickname_or_id(nickname_or_id) do + get_cached_by_nickname(nickname_or_id) || get_cached_by_id(nickname_or_id) + end + def get_by_nickname(nickname) do Repo.get_by(User, nickname: nickname) end diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/ostatus/metadata.ex index 792b4a4bd..120a89a8b 100644 --- a/lib/pleroma/web/ostatus/metadata.ex +++ b/lib/pleroma/web/ostatus/metadata.ex @@ -11,8 +11,7 @@ def build_tags(params) do end def meta_enabled?(type) do - config = Pleroma.Config.get(:metadata, []) - Keyword.get(config, type, false) + Pleroma.Config.get([:metadata, type], false) end # opengraph for single status @@ -70,6 +69,6 @@ defp user_avatar_url(user) do end def pleroma_domain do - Pleroma.Config.get([:instance, :domain], "UNKNOWN_DOMAIN") + Pleroma.Web.Endpoint.host() end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 55dbcab93..5dbee20e1 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -16,8 +16,10 @@ defmodule Pleroma.Web.OStatus.OStatusController do def feed_redirect(conn, %{"nickname" => nickname}) do case get_format(conn) do "html" -> - with %User{} = user <- User.get_cached_by_nickname(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do Fallback.RedirectController.redirector_with_meta(conn, %{user: user}) + else + nil -> {:error, :not_found} end "activity+json" -> diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 914cd6a6d..ad97698dd 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -449,11 +449,11 @@ defmodule Fallback.RedirectController do def redirector(conn, _params) do conn |> put_resp_content_type("text/html") - |> send_file(200, Application.app_dir(:pleroma, "priv/static/index.html")) + |> send_file(200, index_file_path()) end def redirector_with_meta(conn, params) do - {:ok, index_content} = File.read(Application.app_dir(:pleroma, "priv/static/index.html")) + {:ok, index_content} = File.read(index_file_path()) tags = Metadata.build_tags(params) response = String.replace(index_content, "", tags) @@ -462,6 +462,10 @@ def redirector_with_meta(conn, params) do |> send_resp(200, response) end + def index_file_path do + Application.app_dir(:pleroma, "priv/static/index.html") + end + def registration_page(conn, params) do redirector(conn, params) end From 9ff61ed793b7fd968b51c5f6e4b72958adeae977 Mon Sep 17 00:00:00 2001 From: raeno Date: Fri, 14 Dec 2018 20:23:51 +0100 Subject: [PATCH 10/61] Fix tests Notice test has been failing due to missing placeholder in index.html I've tried to use fixtures to substitute that file in test environment but it became too much hassle. Fixtures are not copied to _build directory so I'd need to change file fetching logic. IMO it doesn't worth it since pleroma-fe already has this placeholder merged and all future updated of index.html will include it. --- priv/static/index.html | 2 +- test/web/ostatus/ostatus_controller_test.exs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/priv/static/index.html b/priv/static/index.html index 0a2a56035..4ae1afc82 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -1 +1 @@ -Pleroma
\ No newline at end of file +Pleroma
\ No newline at end of file diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 747e30154..e9e9bdb16 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -136,17 +136,15 @@ test "404s on nonexistent activities", %{conn: conn} do |> response(404) end - test "renders notice metatags in html format" do + test "renders notice metatags in html format", %{conn: conn} do note_activity = insert(:note_activity) - conn = get(conn, "/notice/#{note_activity.id}") - + body = html_response(conn, 200) twitter_card_summary = "" description_content = "" - body = html_response(conn, 200) assert body =~ twitter_card_summary assert body =~ description_content end From b0c3211984c0aa807ea58897308f04ad42651cf1 Mon Sep 17 00:00:00 2001 From: raeno Date: Fri, 14 Dec 2018 21:07:06 +0100 Subject: [PATCH 11/61] Scrub html from activity.content or user.bio for opengraph meta --- lib/pleroma/web/ostatus/metadata.ex | 12 ++++++++++-- mix.exs | 1 + mix.lock | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/ostatus/metadata.ex index 120a89a8b..9935726fc 100644 --- a/lib/pleroma/web/ostatus/metadata.ex +++ b/lib/pleroma/web/ostatus/metadata.ex @@ -16,7 +16,7 @@ def meta_enabled?(type) do # opengraph for single status defp opengraph_tags(%{activity: activity, user: user}) do - with truncated_content = Formatter.truncate(activity.data["object"]["content"]) do + with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do [ {:meta, [ @@ -35,7 +35,7 @@ defp opengraph_tags(%{activity: activity, user: user}) do # opengraph for user card defp opengraph_tags(%{user: user}) do - with truncated_bio = Formatter.truncate(user.bio) do + with truncated_bio = scrub_html_and_truncate(user.bio) do [ {:meta, [ @@ -64,6 +64,14 @@ def to_tag(data) do end end + defp scrub_html_and_truncate(content) do + content + # html content comes from DB already encoded, decode first and scrub after + |> HtmlEntities.decode() + |> Pleroma.HTML.strip_tags() + |> Formatter.truncate() + end + defp user_avatar_url(user) do User.avatar_url(user) |> MediaProxy.url() end diff --git a/mix.exs b/mix.exs index 0fb40e07b..e5fb654dd 100644 --- a/mix.exs +++ b/mix.exs @@ -58,6 +58,7 @@ defp deps do {:pbkdf2_elixir, "~> 0.12.3"}, {:trailing_format_plug, "~> 0.0.7"}, {:html_sanitize_ex, "~> 1.3.0"}, + {:html_entities, "~> 0.4"}, {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, diff --git a/mix.lock b/mix.lock index d9ae9a83b..007079652 100644 --- a/mix.lock +++ b/mix.lock @@ -23,6 +23,7 @@ "gen_smtp": {:hex, :gen_smtp, "0.13.0", "11f08504c4bdd831dc520b8f84a1dce5ce624474a797394e7aafd3c29f5dcd25", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, From 30812f84518f1b2fcc70a416d1a6b9f81264f3a5 Mon Sep 17 00:00:00 2001 From: raeno Date: Fri, 14 Dec 2018 21:08:25 +0100 Subject: [PATCH 12/61] Remove domain from sample_config template --- lib/mix/tasks/pleroma/sample_config.eex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/sample_config.eex b/lib/mix/tasks/pleroma/sample_config.eex index 8eacefab2..0cd572797 100644 --- a/lib/mix/tasks/pleroma/sample_config.eex +++ b/lib/mix/tasks/pleroma/sample_config.eex @@ -12,7 +12,6 @@ config :pleroma, Pleroma.Web.Endpoint, config :pleroma, :instance, name: "<%= name %>", email: "<%= email %>", - domain: "<%= domain %>, limit: 5000, registrations_open: true, dedupe_media: false From 652f49d176b12618c7f537885c0cb1767121acf0 Mon Sep 17 00:00:00 2001 From: raeno Date: Wed, 19 Dec 2018 23:06:10 +0400 Subject: [PATCH 13/61] Remove extra bracket --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e28055d2a..59784549d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -485,7 +485,7 @@ def redirector_with_meta(conn, params) do end def index_file_path do - Pleroma.Plugs.InstanceStatic.file_path("index.html")) + Pleroma.Plugs.InstanceStatic.file_path("index.html") end def registration_page(conn, params) do From 00ddfbe1c2311ea23927d0692353ae6295c1640c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 13 Jan 2019 12:58:49 +0300 Subject: [PATCH 14/61] Remove .idea --- .idea/libraries/gen_smtp.xml | 6 ------ .idea/libraries/html_entities.xml | 6 ------ .idea/libraries/plug_cowboy.xml | 8 -------- .idea/libraries/plug_crypto.xml | 8 -------- .idea/libraries/swoosh.xml | 6 ------ 5 files changed, 34 deletions(-) delete mode 100644 .idea/libraries/gen_smtp.xml delete mode 100644 .idea/libraries/html_entities.xml delete mode 100644 .idea/libraries/plug_cowboy.xml delete mode 100644 .idea/libraries/plug_crypto.xml delete mode 100644 .idea/libraries/swoosh.xml diff --git a/.idea/libraries/gen_smtp.xml b/.idea/libraries/gen_smtp.xml deleted file mode 100644 index e02c2c68c..000000000 --- a/.idea/libraries/gen_smtp.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/libraries/html_entities.xml b/.idea/libraries/html_entities.xml deleted file mode 100644 index 996a8d55c..000000000 --- a/.idea/libraries/html_entities.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/libraries/plug_cowboy.xml b/.idea/libraries/plug_cowboy.xml deleted file mode 100644 index 30a8b7257..000000000 --- a/.idea/libraries/plug_cowboy.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/plug_crypto.xml b/.idea/libraries/plug_crypto.xml deleted file mode 100644 index 69d4a35b3..000000000 --- a/.idea/libraries/plug_crypto.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/swoosh.xml b/.idea/libraries/swoosh.xml deleted file mode 100644 index 546fdb7cd..000000000 --- a/.idea/libraries/swoosh.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 9a6236e69d2709c0df374574fd77c4b2bc3d68fb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 13 Jan 2019 13:00:53 +0300 Subject: [PATCH 15/61] Reset index.html to develop --- priv/static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/static/index.html b/priv/static/index.html index e897a336d..83c11118c 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -1 +1 @@ -Pleroma
+Pleroma
\ No newline at end of file From 5f9786288d0ebc13240f55caa7ae4578d386d843 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Jan 2019 09:52:52 +0300 Subject: [PATCH 16/61] Prefer ids to usernames --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3120b13b6..26cef53ee 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -454,7 +454,7 @@ def get_cached_by_nickname(nickname) do end def get_cached_by_nickname_or_id(nickname_or_id) do - get_cached_by_nickname(nickname_or_id) || get_cached_by_id(nickname_or_id) + get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) end def get_by_nickname(nickname) do From 4656f433f94f132449df019beae1013f71728d0e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 11:00:17 +0300 Subject: [PATCH 17/61] Move metadata.ex out of ostatus --- lib/pleroma/web/{ostatus => }/metadata.ex | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/pleroma/web/{ostatus => }/metadata.ex (100%) diff --git a/lib/pleroma/web/ostatus/metadata.ex b/lib/pleroma/web/metadata.ex similarity index 100% rename from lib/pleroma/web/ostatus/metadata.ex rename to lib/pleroma/web/metadata.ex From ce15e0659e63217d482ef0448ef7629c76755c46 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 11:56:35 +0300 Subject: [PATCH 18/61] Fix some edge cases [nervous laughter] --- lib/pleroma/user.ex | 15 ++++++++++++++- lib/pleroma/web/metadata.ex | 10 ++++++++-- lib/pleroma/web/router.ex | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 26cef53ee..0e828a99e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -454,7 +454,20 @@ def get_cached_by_nickname(nickname) do end def get_cached_by_nickname_or_id(nickname_or_id) do - get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) + try do + # TODO: convert to UUIDs when !654 is merged + maybe_id = String.to_integer(nickname_or_id) + user = get_cached_by_id(maybe_id) + + if user == nil do + raise ArgumentError, message: "invalid argument foo" + else + user + end + rescue + _ in ArgumentError -> + get_cached_by_nickname(nickname_or_id) + end end def get_by_nickname(nickname) do diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 9935726fc..b7052eec1 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -21,7 +21,13 @@ defp opengraph_tags(%{activity: activity, user: user}) do {:meta, [ property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) post ##{activity.id}" + content: + "#{user.name}" <> + if user.local do + "(@#{user.nickname}@{pleroma_domain})" + else + "(@#{user.nickname})" + end ], []}, {:meta, [property: "og:url", content: activity.data["id"]], []}, {:meta, [property: "og:description", content: truncated_content], []}, @@ -35,7 +41,7 @@ defp opengraph_tags(%{activity: activity, user: user}) do # opengraph for user card defp opengraph_tags(%{user: user}) do - with truncated_bio = scrub_html_and_truncate(user.bio) do + with truncated_bio = scrub_html_and_truncate(user.bio || "") do [ {:meta, [ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5ef99bec5..6cf689ce2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -505,7 +505,7 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) - get("/*path", RedirectController, :redirector) + get("/*path", RedirectController, :redirector_with_meta) options("/*path", RedirectController, :empty) end From 4587a5712adcd5d71aaca70de1d895d2e6505382 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 11:59:05 +0300 Subject: [PATCH 19/61] cringe --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0e828a99e..19759acec 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -460,7 +460,7 @@ def get_cached_by_nickname_or_id(nickname_or_id) do user = get_cached_by_id(maybe_id) if user == nil do - raise ArgumentError, message: "invalid argument foo" + raise ArgumentError, message: "No such user id" else user end From 6f23139864936c8e61cdc278c55b3d8f725e892d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 12:02:55 +0300 Subject: [PATCH 20/61] please don't bully me for this --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 19759acec..5707ba7a2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -465,7 +465,7 @@ def get_cached_by_nickname_or_id(nickname_or_id) do user end rescue - _ in ArgumentError -> + ArgumentError -> get_cached_by_nickname(nickname_or_id) end end From 2858fd2da217e172dcc0f664f21bfc8d81dce452 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 12:11:07 +0300 Subject: [PATCH 21/61] add a fallback function --- lib/pleroma/web/metadata.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index b7052eec1..8591db6b5 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -57,7 +57,9 @@ defp opengraph_tags(%{user: user}) do ] end end - + defp opengraph_tags(_) do + [] + end def to_tag(data) do with {name, attrs, _content = []} <- data do HTML.Tag.tag(name, attrs) From 8745c8c9908ab21ee68e176ab085219df0d49d7b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 18:34:47 +0300 Subject: [PATCH 22/61] remove useless newlines after every tag. Make domain.com/username provide opengraph too --- lib/pleroma/web/metadata.ex | 4 +++- lib/pleroma/web/router.ex | 13 ++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 8591db6b5..f8906e03f 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -7,7 +7,7 @@ def build_tags(params) do if(meta_enabled?(:opengraph), do: opengraph_tags(params), else: []) |> Enum.map(&to_tag/1) |> Enum.map(&HTML.safe_to_string/1) - |> Enum.join("\n") + |> Enum.join() end def meta_enabled?(type) do @@ -57,9 +57,11 @@ defp opengraph_tags(%{user: user}) do ] end end + defp opengraph_tags(_) do [] end + def to_tag(data) do with {name, attrs, _content = []} <- data do HTML.Tag.tag(name, attrs) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6cf689ce2..25e866c48 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -505,7 +505,8 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) - get("/*path", RedirectController, :redirector_with_meta) + get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/*path", RedirectController, :redirector) options("/*path", RedirectController, :empty) end @@ -514,6 +515,7 @@ defmodule Pleroma.Web.Router do defmodule Fallback.RedirectController do use Pleroma.Web, :controller alias Pleroma.Web.Metadata + alias Pleroma.User def redirector(conn, _params) do conn @@ -521,6 +523,15 @@ def redirector(conn, _params) do |> send_file(200, index_file_path()) end + def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do + with %User{} = user <- User.get_cached_by_nickname_or_id(maybe_nickname_or_id) do + redirector_with_meta(conn, %{user: user}) + else + nil -> + redirector(conn, params) + end + end + def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) tags = Metadata.build_tags(params) From 850912b06b64818c069b8f169242f0106b73bbfe Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 19:43:52 +0300 Subject: [PATCH 23/61] refactor opengraph to a different module with a behaviour --- lib/pleroma/web/metadata.ex | 83 ++++----------------------- lib/pleroma/web/metadata/opengraph.ex | 66 +++++++++++++++++++++ lib/pleroma/web/metadata/provider.ex | 3 + 3 files changed, 79 insertions(+), 73 deletions(-) create mode 100644 lib/pleroma/web/metadata/opengraph.ex create mode 100644 lib/pleroma/web/metadata/provider.ex diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index f8906e03f..cf2b86aaa 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -1,65 +1,18 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML - alias Pleroma.{Formatter, User} - alias Pleroma.Web.MediaProxy + @parsers Pleroma.Config.get([:metadata, :providers], []) def build_tags(params) do - if(meta_enabled?(:opengraph), do: opengraph_tags(params), else: []) - |> Enum.map(&to_tag/1) - |> Enum.map(&HTML.safe_to_string/1) - |> Enum.join() - end + Enum.reduce(@parsers, "", fn parser, acc -> + rendered_html = + params + |> parser.build_tags() + |> Enum.map(&to_tag/1) + |> Enum.map(&HTML.safe_to_string/1) + |> Enum.join() - def meta_enabled?(type) do - Pleroma.Config.get([:metadata, type], false) - end - - # opengraph for single status - defp opengraph_tags(%{activity: activity, user: user}) do - with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do - [ - {:meta, - [ - property: "og:title", - content: - "#{user.name}" <> - if user.local do - "(@#{user.nickname}@{pleroma_domain})" - else - "(@#{user.nickname})" - end - ], []}, - {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], []}, - {:meta, [property: "og:image", content: user_avatar_url(user)], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, - {:meta, [property: "twitter:card", content: "summary"], []} - ] - end - end - - # opengraph for user card - defp opengraph_tags(%{user: user}) do - with truncated_bio = scrub_html_and_truncate(user.bio || "") do - [ - {:meta, - [ - property: "og:title", - content: "#{user.name} (@#{user.nickname}@#{pleroma_domain()}) profile" - ], []}, - {:meta, [property: "og:url", content: User.profile_url(user)], []}, - {:meta, [property: "og:description", content: truncated_bio], []}, - {:meta, [property: "og:image", content: user_avatar_url(user)], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, - {:meta, [property: "twitter:card", content: "summary"], []} - ] - end - end - - defp opengraph_tags(_) do - [] + acc <> rendered_html + end) end def to_tag(data) do @@ -73,20 +26,4 @@ def to_tag(data) do raise ArgumentError, message: "make_tag invalid args" end end - - defp scrub_html_and_truncate(content) do - content - # html content comes from DB already encoded, decode first and scrub after - |> HtmlEntities.decode() - |> Pleroma.HTML.strip_tags() - |> Formatter.truncate() - end - - defp user_avatar_url(user) do - User.avatar_url(user) |> MediaProxy.url() - end - - def pleroma_domain do - Pleroma.Web.Endpoint.host() - end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex new file mode 100644 index 000000000..5f8bed2fb --- /dev/null +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -0,0 +1,66 @@ +defmodule Pleroma.Web.Metadata.Providers.OpenGraph do + alias Pleroma.Web.Metadata.Providers.Provider + alias Pleroma.{HTML, Formatter, User} + alias Pleroma.Web.MediaProxy + + @behaviour Provider + + @impl Provider + def build_tags(%{activity: activity, user: user}) do + with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do + [ + {:meta, + [ + property: "og:title", + content: user_name_string(user) + ], []}, + {:meta, [property: "og:url", content: activity.data["id"]], []}, + {:meta, [property: "og:description", content: truncated_content], []}, + {:meta, [property: "og:image", content: user_avatar_url(user)], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + end + + @impl Provider + def build_tags(%{user: user}) do + with truncated_bio = scrub_html_and_truncate(user.bio || "") do + [ + {:meta, + [ + property: "og:title", + content: user_name_string(user) + ], []}, + {:meta, [property: "og:url", content: User.profile_url(user)], []}, + {:meta, [property: "og:description", content: truncated_bio], []}, + {:meta, [property: "og:image", content: user_avatar_url(user)], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] + end + end + + defp scrub_html_and_truncate(content) do + content + # html content comes from DB already encoded, decode first and scrub after + |> HtmlEntities.decode() + |> HTML.strip_tags() + |> Formatter.truncate() + end + + defp user_avatar_url(user) do + User.avatar_url(user) |> MediaProxy.url() + end + + defp user_name_string(user) do + "#{user.name}" <> + if user.local do + "(@#{user.nickname}@#{Pleroma.Web.Endpoint.host()})" + else + "(@#{user.nickname})" + end + end +end diff --git a/lib/pleroma/web/metadata/provider.ex b/lib/pleroma/web/metadata/provider.ex new file mode 100644 index 000000000..f64810fad --- /dev/null +++ b/lib/pleroma/web/metadata/provider.ex @@ -0,0 +1,3 @@ +defmodule Pleroma.Web.Metadata.Providers.Provider do + @callback build_tags(map()) :: list() +end From ff6c9a5c961647b64c3c9fcc05932093595e9588 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 20:00:21 +0300 Subject: [PATCH 24/61] Introduce get_by_id in Activity, replace newlines with spaces --- config/config.exs | 2 +- lib/pleroma/activity.ex | 5 ++++- lib/pleroma/web/metadata/opengraph.ex | 1 + lib/pleroma/web/ostatus/ostatus_controller.ex | 3 +-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 895dbb3ab..b8c208cb4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -207,7 +207,7 @@ ip: {0, 0, 0, 0}, port: 9999 -config :pleroma, :metadata, opengraph: true +config :pleroma, :metadata, providers: [] config :pleroma, :suggestions, enabled: false, diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 353f9f6cd..47562306d 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -35,7 +35,10 @@ def get_by_ap_id(ap_id) do ) ) end - + + def get_by_id(id) do + Repo.get(Activity, id) + end # TODO: # Go through these and fix them everywhere. # Wrong name, only returns create activities diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 5f8bed2fb..6d86c0ee6 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -47,6 +47,7 @@ defp scrub_html_and_truncate(content) do content # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() + |> String.replace(~r//, " ") |> HTML.strip_tags() |> Formatter.truncate() end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index be648a6ee..2a47519d1 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity, Object} alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} - alias Pleroma.Repo alias Pleroma.Web.{OStatus, Federator} alias Pleroma.Web.XML alias Pleroma.Web.ActivityPub.ObjectView @@ -141,7 +140,7 @@ def activity(conn, %{"uuid" => uuid}) do end def notice(conn, %{"id" => id}) do - with {_, %Activity{} = activity} <- {:activity, Repo.get(Activity, id)}, + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)}, {_, true} <- {:public?, ActivityPub.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do From 410fd9d774fb3437a38adfe405ff45d4950b51a9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 21:17:56 +0300 Subject: [PATCH 25/61] Attach attachments --- lib/pleroma/activity.ex | 3 ++- lib/pleroma/web/metadata/opengraph.ex | 36 +++++++++++++++++++++------ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 47562306d..8fd0311d2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -35,10 +35,11 @@ def get_by_ap_id(ap_id) do ) ) end - + def get_by_id(id) do Repo.get(Activity, id) end + # TODO: # Go through these and fix them everywhere. # Wrong name, only returns create activities diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 6d86c0ee6..2eac04ae7 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{activity: activity, user: user}) do with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do + attachments = build_attachments(activity) [ {:meta, [ @@ -16,11 +17,11 @@ def build_tags(%{activity: activity, user: user}) do ], []}, {:meta, [property: "og:url", content: activity.data["id"]], []}, {:meta, [property: "og:description", content: truncated_content], []}, - {:meta, [property: "og:image", content: user_avatar_url(user)], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, {:meta, [property: "twitter:card", content: "summary"], []} - ] + ] ++ if attachments == [] do [ + {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []} ] else attachments end end end @@ -35,7 +36,7 @@ def build_tags(%{user: user}) do ], []}, {:meta, [property: "og:url", content: User.profile_url(user)], []}, {:meta, [property: "og:description", content: truncated_bio], []}, - {:meta, [property: "og:image", content: user_avatar_url(user)], []}, + {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 120], []}, {:meta, [property: "og:image:height", content: 120], []}, {:meta, [property: "twitter:card", content: "summary"], []} @@ -43,6 +44,27 @@ def build_tags(%{user: user}) do end end + defp build_attachments(activity) do + Enum.reduce(activity.data["object"]["attachment"], [], fn attachment, acc -> + rendered_tags = + Enum.map(attachment["url"], fn url -> + media_type = + Enum.find(["image", "audio", "video"], fn media_type -> + String.starts_with?(url["mediaType"], media_type) + end) + + if media_type do + {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} + else + nil + end + end) + + Enum.reject(rendered_tags, &is_nil/1) + acc ++ rendered_tags + end) + end + defp scrub_html_and_truncate(content) do content # html content comes from DB already encoded, decode first and scrub after @@ -52,8 +74,8 @@ defp scrub_html_and_truncate(content) do |> Formatter.truncate() end - defp user_avatar_url(user) do - User.avatar_url(user) |> MediaProxy.url() + defp attachment_url(url) do + MediaProxy.url(url) end defp user_name_string(user) do From e8eecd61b4714406501fa0c5335ba53e19ee0bae Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 21:20:27 +0300 Subject: [PATCH 26/61] Formating --- lib/pleroma/web/metadata/opengraph.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 2eac04ae7..be7b155d5 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do def build_tags(%{activity: activity, user: user}) do with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do attachments = build_attachments(activity) + [ {:meta, [ @@ -18,10 +19,16 @@ def build_tags(%{activity: activity, user: user}) do {:meta, [property: "og:url", content: activity.data["id"]], []}, {:meta, [property: "og:description", content: truncated_content], []}, {:meta, [property: "twitter:card", content: "summary"], []} - ] ++ if attachments == [] do [ - {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []} ] else attachments end + ] ++ + if attachments == [] do + [ + {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []} + ] + else + attachments + end end end From 70f140681f90d92169b9774ad048100a003fd5a0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 21:47:45 +0300 Subject: [PATCH 27/61] Add space between name and nickname --- lib/pleroma/web/metadata/opengraph.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index be7b155d5..8046c13ba 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -86,7 +86,7 @@ defp attachment_url(url) do end defp user_name_string(user) do - "#{user.name}" <> + "#{user.name} " <> if user.local do "(@#{user.nickname}@#{Pleroma.Web.Endpoint.host()})" else From 2e630bea0da6452d3342a335da3ca642dc61c1b3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 23:00:22 +0300 Subject: [PATCH 28/61] Add twitter card, filter nsfw --- lib/pleroma/web/metadata/opengraph.ex | 8 ++---- lib/pleroma/web/metadata/twitter_card.ex | 36 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 lib/pleroma/web/metadata/twitter_card.ex diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 8046c13ba..fcc960311 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -17,10 +17,9 @@ def build_tags(%{activity: activity, user: user}) do content: user_name_string(user) ], []}, {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], []}, - {:meta, [property: "twitter:card", content: "summary"], []} + {:meta, [property: "og:description", content: truncated_content], []} ] ++ - if attachments == [] do + if attachments == [] or Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do [ {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 120], []}, @@ -45,8 +44,7 @@ def build_tags(%{user: user}) do {:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []}, - {:meta, [property: "twitter:card", content: "summary"], []} + {:meta, [property: "og:image:height", content: 120], []} ] end end diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex new file mode 100644 index 000000000..6424f84b3 --- /dev/null +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -0,0 +1,36 @@ +defmodule Pleroma.Web.Metadata.Providers.TwitterCard do + def build_tags(%{activity: activity}) do + if Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) or + activity.data["object"]["attachment"] == [] do + build_tags(nil) + else + case find_first_acceptable_media_type(activity) do + "image" -> + [{:meta, [property: "twitter:card", content: "summary_large_image"], []}] + + "audio" -> + [{:meta, [property: "twitter:card", content: "player"], []}] + + "video" -> + [{:meta, [property: "twitter:card", content: "player"], []}] + + _ -> + build_tags(nil) + end + end + end + + def build_tags(_) do + [{:meta, [property: "twitter:card", content: "summary"], []}] + end + + def find_first_acceptable_media_type(%{data: %{"object" => %{"attachment" => attachment}}}) do + Enum.find_value(attachment, fn attachment -> + Enum.find_value(attachment["url"], fn url -> + Enum.find(["image", "audio", "video"], fn media_type -> + String.starts_with?(url["mediaType"], media_type) + end) + end) + end) + end +end From 9aa69e12b87a892d33d1bf4f0d556752391b465a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 15 Jan 2019 23:25:28 +0300 Subject: [PATCH 29/61] Add behaviours to TwitterCard, remove some dumb stuff in Formatter.truncate --- lib/pleroma/formatter.ex | 21 ++++++--------------- lib/pleroma/web/metadata/opengraph.ex | 3 ++- lib/pleroma/web/metadata/twitter_card.ex | 6 ++++++ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 49f7075e6..63e0acb21 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -184,21 +184,12 @@ def finalize({subs, text}) do end) end - def truncate(text, opts \\ []) do - max_length = opts[:max_length] || 200 - omission = opts[:omission] || "..." - - cond do - not String.valid?(text) -> - text - - String.length(text) < max_length -> - text - - true -> - length_with_omission = max_length - String.length(omission) - - "#{String.slice(text, 0, length_with_omission)}#{omission}" + def truncate(text, max_length \\ 200, omission \\ "...") do + if String.length(text) < max_length do + text + else + length_with_omission = max_length - String.length(omission) + String.slice(text, 0, length_with_omission) <> omission end end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index fcc960311..1e3af947d 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -19,7 +19,8 @@ def build_tags(%{activity: activity, user: user}) do {:meta, [property: "og:url", content: activity.data["id"]], []}, {:meta, [property: "og:description", content: truncated_content], []} ] ++ - if attachments == [] or Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do + if attachments == [] or + Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do [ {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 120], []}, diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 6424f84b3..3094e61fd 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -1,4 +1,9 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do + alias Pleroma.Web.Metadata.Providers.Provider + + @behaviour Provider + + @impl Provider def build_tags(%{activity: activity}) do if Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) or activity.data["object"]["attachment"] == [] do @@ -20,6 +25,7 @@ def build_tags(%{activity: activity}) do end end + @impl Provider def build_tags(_) do [{:meta, [property: "twitter:card", content: "summary"], []}] end From 565caff3f4f8b21b4bae9fb20732688389b4d829 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 00:07:28 +0300 Subject: [PATCH 30/61] cache HTML in OGP --- lib/pleroma/web/metadata/opengraph.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 1e3af947d..33ff075c6 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{activity: activity, user: user}) do - with truncated_content = scrub_html_and_truncate(activity.data["object"]["content"]) do + with truncated_content = scrub_html_and_truncate(activity) do attachments = build_attachments(activity) [ @@ -71,6 +71,15 @@ defp build_attachments(activity) do end) end + defp scrub_html_and_truncate(%{data: %{ "object" => %{ "content" => content}}} = activity) do + content + # html content comes from DB already encoded, decode first and scrub after + |> HtmlEntities.decode() + |> String.replace(~r//, " ") + |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + |> Formatter.truncate() + end + defp scrub_html_and_truncate(content) do content # html content comes from DB already encoded, decode first and scrub after @@ -79,7 +88,6 @@ defp scrub_html_and_truncate(content) do |> HTML.strip_tags() |> Formatter.truncate() end - defp attachment_url(url) do MediaProxy.url(url) end From dd1432d6955a72b8483717978d61a505e0608bbc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 09:42:24 +0300 Subject: [PATCH 31/61] Disable previews for any activity, but create --- lib/pleroma/web/metadata/opengraph.ex | 5 +++-- lib/pleroma/web/ostatus/ostatus_controller.ex | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 33ff075c6..b15856974 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -71,7 +71,7 @@ defp build_attachments(activity) do end) end - defp scrub_html_and_truncate(%{data: %{ "object" => %{ "content" => content}}} = activity) do + defp scrub_html_and_truncate(%{data: %{"object" => %{"content" => content}}} = activity) do content # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() @@ -80,7 +80,7 @@ defp scrub_html_and_truncate(%{data: %{ "object" => %{ "content" => content}}} = |> Formatter.truncate() end - defp scrub_html_and_truncate(content) do + defp scrub_html_and_truncate(content) when is_binary(content) do content # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() @@ -88,6 +88,7 @@ defp scrub_html_and_truncate(content) do |> HTML.strip_tags() |> Formatter.truncate() end + defp attachment_url(url) do MediaProxy.url(url) end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 2a47519d1..f7ba57389 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -145,7 +145,15 @@ def notice(conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> - Fallback.RedirectController.redirector_with_meta(conn, %{activity: activity, user: user}) + # Only Create actvities have a map at object + if is_map(activity.data["object"]) do + Fallback.RedirectController.redirector_with_meta(conn, %{ + activity: activity, + user: user + }) + else + Fallback.RedirectController.redirector(conn, nil) + end _ -> represent_activity(conn, format, activity, user) From 70b2bb6eded5597bd768d7dc8754c2c994561aee Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:26:01 +0300 Subject: [PATCH 32/61] add caching --- lib/pleroma/application.ex | 11 +++++++++++ lib/pleroma/web/metadata.ex | 10 ++++++++++ lib/pleroma/web/router.ex | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ad2797209..5f9518914 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -84,6 +84,17 @@ def start(_type, _args) do ], id: :cachex_scrubber ), + worker( + Cachex, + [ + :metadata_cache, + [ + limit: 2500, + default_ttl: :timer.minutes(15) + ] + ], + id: :cachex_metadata + ), worker( Cachex, [ diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index cf2b86aaa..a5a706b8f 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -2,6 +2,16 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML @parsers Pleroma.Config.get([:metadata, :providers], []) + + def get_cached_tags(params) do + # I am unsure how well ETS works with big keys + key = :erlang.term_to_binary(params) + + Cachex.fetch!(:metadata_cache, key, fn _key -> + {:commit, build_tags(params)} + end) + end + def build_tags(params) do Enum.reduce(@parsers, "", fn parser, acc -> rendered_html = diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 25e866c48..1ecd4aee1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -534,7 +534,7 @@ def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = Metadata.build_tags(params) + tags = Metadata.get_cached_tags(params) response = String.replace(index_content, "", tags) conn From 0039d45b5b695a28660dd56ff74fa45414c92e47 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:30:47 +0300 Subject: [PATCH 33/61] No need to use activity in the key --- lib/pleroma/web/metadata.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index a5a706b8f..890367152 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -3,9 +3,10 @@ defmodule Pleroma.Web.Metadata do @parsers Pleroma.Config.get([:metadata, :providers], []) - def get_cached_tags(params) do + def get_cached_tags(%{user: user} = params) do # I am unsure how well ETS works with big keys - key = :erlang.term_to_binary(params) + # We don't need to use the both activity and a user since the object can't change it's content + key = :erlang.term_to_binary(user) Cachex.fetch!(:metadata_cache, key, fn _key -> {:commit, build_tags(params)} From a76793006bb64723b4fe0aa054765edadc65905c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:35:06 +0300 Subject: [PATCH 34/61] Thanks you rinpatch, very cool --- lib/pleroma/web/metadata.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 890367152..bc0f3beed 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -2,10 +2,17 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML @parsers Pleroma.Config.get([:metadata, :providers], []) + def get_cached_tags(%{activity: activity, user: user} = params) do + # We don't need to use the both activity and a user since the object can't change it's content + key = "#{:erlang.term_to_binary(user)}#{activity.data["id"]}" + + Cachex.fetch!(:metadata_cache, key, fn _key -> + {:commit, build_tags(params)} + end) + end def get_cached_tags(%{user: user} = params) do # I am unsure how well ETS works with big keys - # We don't need to use the both activity and a user since the object can't change it's content key = :erlang.term_to_binary(user) Cachex.fetch!(:metadata_cache, key, fn _key -> From a9c27e137d992c991827417af40fe49fafa34be2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:38:42 +0300 Subject: [PATCH 35/61] Add a fallback function to handle generic params, just in case --- lib/pleroma/web/metadata.ex | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index bc0f3beed..23f152e06 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -20,6 +20,14 @@ def get_cached_tags(%{user: user} = params) do end) end + def get_cached_tags(params) do + key = :erlang.term_to_binary(params) + + Cachex.fetch!(:metadata_cache, key, fn _key -> + {:commit, build_tags(params)} + end) + end + def build_tags(params) do Enum.reduce(@parsers, "", fn parser, acc -> rendered_html = From bfe2a11a6b64b868d18727359b83edc51ce5bb80 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:45:56 +0300 Subject: [PATCH 36/61] Add config doc --- docs/config.md | 5 +++++ lib/pleroma/web/metadata.ex | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index e3738271b..c2104a9e8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -210,3 +210,8 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando * `max_jobs`: The maximum amount of parallel federation jobs running at the same time. * `initial_timeout`: The initial timeout in seconds * `max_retries`: The maximum number of times a federation job is retried + +## Pleroma.Web.Metadata +* `providers`: a list of metadata providers to enable. Providers avalible: + * Pleroma.Web.Metadata.Providers.OpenGraph + * Pleroma.Web.Metadata.Providers.TwitterCard diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 23f152e06..ddc74fb0d 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML - @parsers Pleroma.Config.get([:metadata, :providers], []) + @providers Pleroma.Config.get([__MODULE__, :providers], []) def get_cached_tags(%{activity: activity, user: user} = params) do # We don't need to use the both activity and a user since the object can't change it's content key = "#{:erlang.term_to_binary(user)}#{activity.data["id"]}" @@ -29,7 +29,7 @@ def get_cached_tags(params) do end def build_tags(params) do - Enum.reduce(@parsers, "", fn parser, acc -> + Enum.reduce(@providers, "", fn parser, acc -> rendered_html = params |> parser.build_tags() From 8cea00e1dc2bf3c2e807b09fac886dfd7b4ea3e9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 10:53:57 +0300 Subject: [PATCH 37/61] Remove OpenGraph test from Ostatus tests --- test/web/ostatus/ostatus_controller_test.exs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 8e9d2b69a..5a94cb3b3 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -156,19 +156,6 @@ test "404s on nonexistent activities", %{conn: conn} do |> response(404) end - test "renders notice metatags in html format", %{conn: conn} do - note_activity = insert(:note_activity) - conn = get(conn, "/notice/#{note_activity.id}") - body = html_response(conn, 200) - twitter_card_summary = "" - - description_content = - "" - - assert body =~ twitter_card_summary - assert body =~ description_content - end - test "gets a notice in xml format", %{conn: conn} do note_activity = insert(:note_activity) From ff01fd3c4fe1edb45098a7730b97b9424b1222cc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 11:18:25 +0300 Subject: [PATCH 38/61] Remove caching because it does not affect performance and may be even worse in some cases --- lib/pleroma/web/metadata.ex | 25 ------------------------- lib/pleroma/web/router.ex | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index ddc74fb0d..d859dfd8b 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -2,31 +2,6 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML @providers Pleroma.Config.get([__MODULE__, :providers], []) - def get_cached_tags(%{activity: activity, user: user} = params) do - # We don't need to use the both activity and a user since the object can't change it's content - key = "#{:erlang.term_to_binary(user)}#{activity.data["id"]}" - - Cachex.fetch!(:metadata_cache, key, fn _key -> - {:commit, build_tags(params)} - end) - end - - def get_cached_tags(%{user: user} = params) do - # I am unsure how well ETS works with big keys - key = :erlang.term_to_binary(user) - - Cachex.fetch!(:metadata_cache, key, fn _key -> - {:commit, build_tags(params)} - end) - end - - def get_cached_tags(params) do - key = :erlang.term_to_binary(params) - - Cachex.fetch!(:metadata_cache, key, fn _key -> - {:commit, build_tags(params)} - end) - end def build_tags(params) do Enum.reduce(@providers, "", fn parser, acc -> diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1ecd4aee1..25e866c48 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -534,7 +534,7 @@ def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = Metadata.get_cached_tags(params) + tags = Metadata.build_tags(params) response = String.replace(index_content, "", tags) conn From b44995866ba49d2c9ea53082deadb6b72acf53e3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 16:52:01 +0300 Subject: [PATCH 39/61] Replace map with reduce to remove nils --- lib/pleroma/web/metadata/opengraph.ex | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index b15856974..f6a1f3023 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -50,23 +50,25 @@ def build_tags(%{user: user}) do end end - defp build_attachments(activity) do - Enum.reduce(activity.data["object"]["attachment"], [], fn attachment, acc -> + defp build_attachments(%{data: %{"object" => %{"attachment" => attachments}}} = _activity) do + Enum.reduce(attachments, [], fn attachment, acc -> rendered_tags = - Enum.map(attachment["url"], fn url -> + Enum.reduce(attachment["url"], [], fn url, acc -> media_type = Enum.find(["image", "audio", "video"], fn media_type -> String.starts_with?(url["mediaType"], media_type) end) if media_type do - {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} + [ + {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} + | acc + ] else - nil + acc end end) - Enum.reject(rendered_tags, &is_nil/1) acc ++ rendered_tags end) end From 293f6a8b712246d0580f9eb113c798ae1ea3b634 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 17:15:13 +0300 Subject: [PATCH 40/61] join us now and share the software~. Also tests --- lib/pleroma/web/metadata.ex | 3 + lib/pleroma/web/metadata/opengraph.ex | 3 + lib/pleroma/web/metadata/provider.ex | 3 + lib/pleroma/web/metadata/twitter_card.ex | 3 + test/web/metadata/opengraph_test.exs | 113 +++++++++++++++++++++++ 5 files changed, 125 insertions(+) create mode 100644 test/web/metadata/opengraph_test.exs diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index d859dfd8b..8793fc265 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -1,3 +1,6 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata do alias Phoenix.HTML diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index f6a1f3023..b7c5dc64e 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -1,3 +1,6 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Providers.Provider alias Pleroma.{HTML, Formatter, User} diff --git a/lib/pleroma/web/metadata/provider.ex b/lib/pleroma/web/metadata/provider.ex index f64810fad..a39008bcc 100644 --- a/lib/pleroma/web/metadata/provider.ex +++ b/lib/pleroma/web/metadata/provider.ex @@ -1,3 +1,6 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.Provider do @callback build_tags(map()) :: list() end diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 3094e61fd..853776611 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -1,3 +1,6 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.TwitterCard do alias Pleroma.Web.Metadata.Providers.Provider diff --git a/test/web/metadata/opengraph_test.exs b/test/web/metadata/opengraph_test.exs new file mode 100644 index 000000000..be3f89364 --- /dev/null +++ b/test/web/metadata/opengraph_test.exs @@ -0,0 +1,113 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Web.Metadata.Providers.OpenGraphTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Web.Metadata.Providers.OpenGraph + + test "it renders all supported types of attachments and skips unknown types" do + user = insert(:user) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "tag" => [], + "content" => "pleroma in a nutshell", + "attachment" => [ + %{ + "url" => [ + %{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"} + ] + }, + %{ + "url" => [ + %{ + "mediaType" => "application/octet-stream", + "href" => "https://pleroma.gov/fqa/badapple.sfc" + } + ] + }, + %{ + "url" => [ + %{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"} + ] + }, + %{ + "url" => [ + %{ + "mediaType" => "audio/basic", + "href" => "http://www.gnu.org/music/free-software-song.au" + } + ] + } + ] + } + }) + + note_activity = + insert(:note_activity, %{ + data: %{ + "actor" => note.data["actor"], + "to" => note.data["to"], + "object" => note.data, + "context" => note.data["context"] + }, + actor: note.data["actor"], + recipients: note.data["to"] + }) + + result = OpenGraph.build_tags(%{activity: note_activity, user: user}) + + assert Enum.all?( + [ + {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []}, + {:meta, + [property: "og:audio", content: "http://www.gnu.org/music/free-software-song.au"], + []}, + {:meta, [property: "og:video", content: "https://pleroma.gov/about/juche.webm"], + []} + ], + fn element -> element in result end + ) + end + + test "it does not render attachments if post is nsfw" do + user = insert(:user, avatar: %{"url" => [%{"href" => "https://pleroma.gov/tenshi.png"}]}) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "content" => "#cuteposting #nsfw #hambaga", + "tag" => ["cuteposting", "nsfw", "hambaga"], + "attachment" => [ + %{ + "url" => [ + %{"mediaType" => "image/png", "href" => "https://misskey.microsoft/corndog.png"} + ] + } + ] + } + }) + + note_activity = + insert(:note_activity, %{ + data: %{ + "actor" => note.data["actor"], + "to" => note.data["to"], + "object" => note.data, + "context" => note.data["context"] + }, + actor: note.data["actor"], + recipients: note.data["to"] + }) + + result = OpenGraph.build_tags(%{activity: note_activity, user: user}) + + assert {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []} in result + + refute {:meta, [property: "og:image", content: "https://misskey.microsoft/corndog.png"], []} in result + end +end From 82cf9b97510acb261c805f098043471d9a588e2b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 17:16:28 +0300 Subject: [PATCH 41/61] forgot to remove this thing --- lib/pleroma/application.ex | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 5f9518914..ad2797209 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -84,17 +84,6 @@ def start(_type, _args) do ], id: :cachex_scrubber ), - worker( - Cachex, - [ - :metadata_cache, - [ - limit: 2500, - default_ttl: :timer.minutes(15) - ] - ], - id: :cachex_metadata - ), worker( Cachex, [ From 76f80940da3979871923d647c95d7693b1c01fd5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 17:19:01 +0300 Subject: [PATCH 42/61] oops --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index b8c208cb4..3448e31ca 100644 --- a/config/config.exs +++ b/config/config.exs @@ -207,7 +207,7 @@ ip: {0, 0, 0, 0}, port: 9999 -config :pleroma, :metadata, providers: [] +config :pleroma, Pleroma.Web.Metadata, providers: [] config :pleroma, :suggestions, enabled: false, From 5a08dee37905c576da3289d7d69e9b114f202634 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 17:44:08 +0300 Subject: [PATCH 43/61] cache ap_id in id instead of caching user two times --- lib/pleroma/user.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5707ba7a2..3becd545c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -445,7 +445,15 @@ def get_cached_by_ap_id(ap_id) do def get_cached_by_id(id) do key = "id:#{id}" - Cachex.fetch!(:user_cache, key, fn _ -> get_by_id(id) end) + + ap_id = + Cachex.fetch!(:user_cache, key, fn _ -> + user = get_by_id(id) + Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) + {:commit, user.ap_id} + end) + + get_cached_by_ap_id(ap_id) end def get_cached_by_nickname(nickname) do From 8a9f089812295ef5087864e852ad9550ee00ce76 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 17:58:11 +0300 Subject: [PATCH 44/61] remove id cast --- lib/pleroma/user.ex | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3becd545c..fdc552f75 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -462,19 +462,7 @@ def get_cached_by_nickname(nickname) do end def get_cached_by_nickname_or_id(nickname_or_id) do - try do - # TODO: convert to UUIDs when !654 is merged - maybe_id = String.to_integer(nickname_or_id) - user = get_cached_by_id(maybe_id) - - if user == nil do - raise ArgumentError, message: "No such user id" - else - user - end - rescue - ArgumentError -> - get_cached_by_nickname(nickname_or_id) + get_cached_by_id(maybe_id) || get_cached_by_nickname(nickname_or_id) end end From 9a90b5d91a7a5a40d0e90d3112faea65b6e332ad Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 18:01:17 +0300 Subject: [PATCH 45/61] oof --- lib/pleroma/user.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fdc552f75..96160094a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -463,7 +463,6 @@ def get_cached_by_nickname(nickname) do def get_cached_by_nickname_or_id(nickname_or_id) do get_cached_by_id(maybe_id) || get_cached_by_nickname(nickname_or_id) - end end def get_by_nickname(nickname) do From bb43f4cee94dfd6a71105fb31f10c4b0188a4bf2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 18:02:46 +0300 Subject: [PATCH 46/61] Remove useless with in opengraph.ex --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/metadata/opengraph.ex | 42 +++++++++++++-------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 96160094a..f6c0f36e4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -462,7 +462,7 @@ def get_cached_by_nickname(nickname) do end def get_cached_by_nickname_or_id(nickname_or_id) do - get_cached_by_id(maybe_id) || get_cached_by_nickname(nickname_or_id) + get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) end def get_by_nickname(nickname) do diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index b7c5dc64e..6f88e9bb4 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -10,29 +10,27 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{activity: activity, user: user}) do - with truncated_content = scrub_html_and_truncate(activity) do - attachments = build_attachments(activity) + attachments = build_attachments(activity) - [ - {:meta, - [ - property: "og:title", - content: user_name_string(user) - ], []}, - {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: truncated_content], []} - ] ++ - if attachments == [] or - Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do - [ - {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []} - ] - else - attachments - end - end + [ + {:meta, + [ + property: "og:title", + content: user_name_string(user) + ], []}, + {:meta, [property: "og:url", content: activity.data["id"]], []}, + {:meta, [property: "og:description", content: scrub_html_and_truncate(activity)], []} + ] ++ + if attachments == [] or + Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do + [ + {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, + {:meta, [property: "og:image:width", content: 120], []}, + {:meta, [property: "og:image:height", content: 120], []} + ] + else + attachments + end end @impl Provider From 5fa508cc2be6a8e5b4d6522687708c6c269803a3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Jan 2019 18:04:41 +0300 Subject: [PATCH 47/61] Remove @providers and call Pleroma.config on runtime --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/metadata.ex | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f6c0f36e4..180ef180c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -462,7 +462,7 @@ def get_cached_by_nickname(nickname) do end def get_cached_by_nickname_or_id(nickname_or_id) do - get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) + get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) end def get_by_nickname(nickname) do diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 8793fc265..2164b0fe8 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -4,10 +4,8 @@ defmodule Pleroma.Web.Metadata do alias Phoenix.HTML - @providers Pleroma.Config.get([__MODULE__, :providers], []) - def build_tags(params) do - Enum.reduce(@providers, "", fn parser, acc -> + Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc -> rendered_html = params |> parser.build_tags() From fd3a558230c48ee291b956f52e3b07492cbad36c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 09:18:46 +0300 Subject: [PATCH 48/61] Apply feld's patch --- lib/pleroma/web/metadata/opengraph.ex | 73 ++++++++++++++++++++------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 6f88e9bb4..f95e2442a 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Providers.Provider - alias Pleroma.{HTML, Formatter, User} + alias Pleroma.{HTML, Formatter, User, Web} alias Pleroma.Web.MediaProxy @behaviour Provider @@ -12,21 +12,39 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do def build_tags(%{activity: activity, user: user}) do attachments = build_attachments(activity) + # Most previews only show og:title which is inconvenient. Instagram + # hacks this by putting the description in the title and making the + # description longer prefixed by how many likes and shares the post + # has. Here we use the descriptive nickname in the title, and expand + # the full account & nickname in the description. We also use the cute^Wevil + # smart quotes around the status text like Instagram, too. [ {:meta, [ property: "og:title", - content: user_name_string(user) + content: + "#{user.name}: " <> + "“" <> + scrub_html_and_truncate(activity) <> + "”" ], []}, - {:meta, [property: "og:url", content: activity.data["id"]], []}, - {:meta, [property: "og:description", content: scrub_html_and_truncate(activity)], []} + {:meta, [property: "og:url", content: "#{Web.base_url()}/notice/#{activity.id}"], []}, + {:meta, + [ + property: "og:description", + content: + "#{user_name_string(user)}: " <> + "“" <> + scrub_html_and_truncate(activity) <> + "”" + ], []}, + {:meta, [property: "og:type", content: "website"], []} ] ++ - if attachments == [] or - Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) do + if attachments == [] do [ {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []} + {:meta, [property: "og:image:width", content: 150], []}, + {:meta, [property: "og:image:height", content: 150], []} ] else attachments @@ -44,9 +62,10 @@ def build_tags(%{user: user}) do ], []}, {:meta, [property: "og:url", content: User.profile_url(user)], []}, {:meta, [property: "og:description", content: truncated_bio], []}, + {:meta, [property: "og:type", content: "website"], []}, {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, - {:meta, [property: "og:image:width", content: 120], []}, - {:meta, [property: "og:image:height", content: 120], []} + {:meta, [property: "og:image:width", content: 150], []}, + {:meta, [property: "og:image:height", content: 150], []} ] end end @@ -60,13 +79,33 @@ defp build_attachments(%{data: %{"object" => %{"attachment" => attachments}}} = String.starts_with?(url["mediaType"], media_type) end) - if media_type do - [ - {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} - | acc - ] - else - acc + # TODO: Add additional properties to objects when we have the data available. + # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image + # object when a Video or GIF is attached it will display that in the Whatsapp Rich Preview. + case media_type do + "audio" -> + [ + {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} + | acc + ] + + "image" -> + [ + {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], + []}, + {:meta, [property: "og:image:width", content: 150], []}, + {:meta, [property: "og:image:height", content: 150], []} + | acc + ] + + "video" -> + [ + {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []} + | acc + ] + + _ -> + acc end end) From 0256bd2f1dfb121a4d751906a202e3db482500a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 10:34:19 +0300 Subject: [PATCH 49/61] Use object url instead of a hack --- lib/pleroma/web/metadata/opengraph.ex | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index f95e2442a..a48788969 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -3,13 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Providers.Provider - alias Pleroma.{HTML, Formatter, User, Web} + alias Pleroma.{HTML, Formatter, User} alias Pleroma.Web.MediaProxy @behaviour Provider @impl Provider - def build_tags(%{activity: activity, user: user}) do + def build_tags(%{activity: %{data: %{"object" => %{"id" => object_id}}} = activity, user: user}) do attachments = build_attachments(activity) # Most previews only show og:title which is inconvenient. Instagram @@ -22,21 +22,13 @@ def build_tags(%{activity: activity, user: user}) do {:meta, [ property: "og:title", - content: - "#{user.name}: " <> - "“" <> - scrub_html_and_truncate(activity) <> - "”" + content: "#{user.name}: " <> "“" <> scrub_html_and_truncate(activity) <> "”" ], []}, - {:meta, [property: "og:url", content: "#{Web.base_url()}/notice/#{activity.id}"], []}, + {:meta, [property: "og:url", content: object_id], []}, {:meta, [ property: "og:description", - content: - "#{user_name_string(user)}: " <> - "“" <> - scrub_html_and_truncate(activity) <> - "”" + content: "#{user_name_string(user)}: " <> "“" <> scrub_html_and_truncate(activity) <> "”" ], []}, {:meta, [property: "og:type", content: "website"], []} ] ++ From 4d5f15cd422abd3a2dce6f6022c75014c18c73cf Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 11:00:02 +0300 Subject: [PATCH 50/61] Introduce optional unfurling of nsfw content --- docs/config.md | 3 ++- lib/pleroma/web/metadata.ex | 8 ++++++++ lib/pleroma/web/metadata/opengraph.ex | 3 ++- lib/pleroma/web/metadata/twitter_card.ex | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/config.md b/docs/config.md index c2104a9e8..b9ad2aef7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -212,6 +212,7 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando * `max_retries`: The maximum number of times a federation job is retried ## Pleroma.Web.Metadata -* `providers`: a list of metadata providers to enable. Providers avalible: +* `providers`: a list of metadata providers to enable. Providers availible: * Pleroma.Web.Metadata.Providers.OpenGraph * Pleroma.Web.Metadata.Providers.TwitterCard +* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 2164b0fe8..be3c384ae 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -28,4 +28,12 @@ def to_tag(data) do raise ArgumentError, message: "make_tag invalid args" end end + + def activity_nsfw?(%{data: %{"object" => %{"tag" => tags}}}) do + if(Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false) do + Enum.any?(tags, fn tag -> tag == "nsfw" end) + else + false + end + end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index a48788969..2f27a5300 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Providers.Provider + alias Pleroma.Web.Metadata alias Pleroma.{HTML, Formatter, User} alias Pleroma.Web.MediaProxy @@ -32,7 +33,7 @@ def build_tags(%{activity: %{data: %{"object" => %{"id" => object_id}}} = activi ], []}, {:meta, [property: "og:type", content: "website"], []} ] ++ - if attachments == [] do + if attachments == [] or Metadata.activity_nsfw?(activity) do [ {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 150], []}, diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 853776611..9a1245e59 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -3,13 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Providers.TwitterCard do alias Pleroma.Web.Metadata.Providers.Provider + alias Pleroma.Web.Metadata @behaviour Provider @impl Provider def build_tags(%{activity: activity}) do - if Enum.any?(activity.data["object"]["tag"], fn tag -> tag == "nsfw" end) or - activity.data["object"]["attachment"] == [] do + if Metadata.activity_nsfw?(activity) or activity.data["object"]["attachment"] == [] do build_tags(nil) else case find_first_acceptable_media_type(activity) do From 3b187896847637d93db194897565599acd73a57c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 11:08:50 +0300 Subject: [PATCH 51/61] Fix test --- test/web/metadata/opengraph_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/web/metadata/opengraph_test.exs b/test/web/metadata/opengraph_test.exs index be3f89364..672942148 100644 --- a/test/web/metadata/opengraph_test.exs +++ b/test/web/metadata/opengraph_test.exs @@ -74,6 +74,7 @@ test "it renders all supported types of attachments and skips unknown types" do end test "it does not render attachments if post is nsfw" do + Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false) user = insert(:user, avatar: %{"url" => [%{"href" => "https://pleroma.gov/tenshi.png"}]}) note = From 388ceb6a7de4a17d695b40152b5dcbfadc2eef84 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 19:00:08 +0300 Subject: [PATCH 52/61] Fix the issue with get_by_nickname never being called --- lib/pleroma/user.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 180ef180c..13a476fd2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -449,8 +449,13 @@ def get_cached_by_id(id) do ap_id = Cachex.fetch!(:user_cache, key, fn _ -> user = get_by_id(id) - Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) - {:commit, user.ap_id} + + if user do + Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) + {:commit, user.ap_id} + else + {:ignore, ""} + end end) get_cached_by_ap_id(ap_id) From a95d5da607f92faa1bc05f9d284086071a0a9d48 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 20:18:28 +0300 Subject: [PATCH 53/61] Don't show content if empty or zero width space --- lib/pleroma/web/metadata/opengraph.ex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 2f27a5300..cbd0b7d1b 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -10,8 +10,19 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @behaviour Provider @impl Provider - def build_tags(%{activity: %{data: %{"object" => %{"id" => object_id}}} = activity, user: user}) do + def build_tags(%{ + activity: %{data: %{"object" => %{"id" => object_id}}} = activity, + user: user + }) do attachments = build_attachments(activity) + scrubbed_content = scrub_html_and_truncate(activity) + # Zero width space + content = + if scrubbed_content != "" and scrubbed_content != "\u200B" do + ": “" <> scrubbed_content <> "”" + else + "" + end # Most previews only show og:title which is inconvenient. Instagram # hacks this by putting the description in the title and making the @@ -23,13 +34,13 @@ def build_tags(%{activity: %{data: %{"object" => %{"id" => object_id}}} = activi {:meta, [ property: "og:title", - content: "#{user.name}: " <> "“" <> scrub_html_and_truncate(activity) <> "”" + content: "#{user.name}" <> content ], []}, {:meta, [property: "og:url", content: object_id], []}, {:meta, [ property: "og:description", - content: "#{user_name_string(user)}: " <> "“" <> scrub_html_and_truncate(activity) <> "”" + content: "#{user_name_string(user)}" <> content ], []}, {:meta, [property: "og:type", content: "website"], []} ] ++ From 64143d8040a5806ccf9707644a7e7eacc91d74c6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Jan 2019 22:57:24 +0300 Subject: [PATCH 54/61] Fix object id in OpenGraph test --- test/web/metadata/opengraph_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/web/metadata/opengraph_test.exs b/test/web/metadata/opengraph_test.exs index 672942148..1c4fadc46 100644 --- a/test/web/metadata/opengraph_test.exs +++ b/test/web/metadata/opengraph_test.exs @@ -14,6 +14,7 @@ test "it renders all supported types of attachments and skips unknown types" do data: %{ "actor" => user.ap_id, "tag" => [], + "id" => "https://pleroma.gov/objects/whatever", "content" => "pleroma in a nutshell", "attachment" => [ %{ @@ -81,6 +82,7 @@ test "it does not render attachments if post is nsfw" do insert(:note, %{ data: %{ "actor" => user.ap_id, + "id" => "https://pleroma.gov/objects/whatever", "content" => "#cuteposting #nsfw #hambaga", "tag" => ["cuteposting", "nsfw", "hambaga"], "attachment" => [ From 1b1af4798a74c4ab357140ef2c5928dd9ebd3221 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Jan 2019 09:32:52 +0300 Subject: [PATCH 55/61] Use object instead of activity for metadata --- lib/pleroma/web/metadata.ex | 13 ++++---- lib/pleroma/web/metadata/opengraph.ex | 17 ++++++----- lib/pleroma/web/metadata/provider.ex | 1 + lib/pleroma/web/metadata/twitter_card.ex | 9 +++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 7 +++-- test/web/metadata/opengraph_test.exs | 30 +++---------------- 6 files changed, 30 insertions(+), 47 deletions(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index be3c384ae..8761260f2 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Metadata do alias Phoenix.HTML @@ -29,11 +30,11 @@ def to_tag(data) do end end - def activity_nsfw?(%{data: %{"object" => %{"tag" => tags}}}) do - if(Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false) do - Enum.any?(tags, fn tag -> tag == "nsfw" end) - else - false - end + def activity_nsfw?(%{data: %{"sensitive" => sensitive}}) do + Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false and sensitive + end + + def activity_nsfw?(_) do + false end end diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index cbd0b7d1b..43303859c 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Metadata.Providers.OpenGraph do alias Pleroma.Web.Metadata.Providers.Provider alias Pleroma.Web.Metadata @@ -11,11 +12,11 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{ - activity: %{data: %{"object" => %{"id" => object_id}}} = activity, + object: object, user: user }) do - attachments = build_attachments(activity) - scrubbed_content = scrub_html_and_truncate(activity) + attachments = build_attachments(object) + scrubbed_content = scrub_html_and_truncate(object) # Zero width space content = if scrubbed_content != "" and scrubbed_content != "\u200B" do @@ -36,7 +37,7 @@ def build_tags(%{ property: "og:title", content: "#{user.name}" <> content ], []}, - {:meta, [property: "og:url", content: object_id], []}, + {:meta, [property: "og:url", content: object.data["id"]], []}, {:meta, [ property: "og:description", @@ -44,7 +45,7 @@ def build_tags(%{ ], []}, {:meta, [property: "og:type", content: "website"], []} ] ++ - if attachments == [] or Metadata.activity_nsfw?(activity) do + if attachments == [] or Metadata.activity_nsfw?(object) do [ {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image:width", content: 150], []}, @@ -74,7 +75,7 @@ def build_tags(%{user: user}) do end end - defp build_attachments(%{data: %{"object" => %{"attachment" => attachments}}} = _activity) do + defp build_attachments(%{data: %{"attachment" => attachments}}) do Enum.reduce(attachments, [], fn attachment, acc -> rendered_tags = Enum.reduce(attachment["url"], [], fn url, acc -> @@ -117,12 +118,12 @@ defp build_attachments(%{data: %{"object" => %{"attachment" => attachments}}} = end) end - defp scrub_html_and_truncate(%{data: %{"object" => %{"content" => content}}} = activity) do + defp scrub_html_and_truncate(%{data: %{"content" => content}} = object) do content # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() |> String.replace(~r//, " ") - |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + |> HTML.get_cached_stripped_html_for_object(object, __MODULE__) |> Formatter.truncate() end diff --git a/lib/pleroma/web/metadata/provider.ex b/lib/pleroma/web/metadata/provider.ex index a39008bcc..197fb2a77 100644 --- a/lib/pleroma/web/metadata/provider.ex +++ b/lib/pleroma/web/metadata/provider.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Metadata.Providers.Provider do @callback build_tags(map()) :: list() end diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 9a1245e59..32b979357 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Metadata.Providers.TwitterCard do alias Pleroma.Web.Metadata.Providers.Provider alias Pleroma.Web.Metadata @@ -8,11 +9,11 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do @behaviour Provider @impl Provider - def build_tags(%{activity: activity}) do - if Metadata.activity_nsfw?(activity) or activity.data["object"]["attachment"] == [] do + def build_tags(%{object: object}) do + if Metadata.activity_nsfw?(object) or object.data["attachment"] == [] do build_tags(nil) else - case find_first_acceptable_media_type(activity) do + case find_first_acceptable_media_type(object) do "image" -> [{:meta, [property: "twitter:card", content: "summary_large_image"], []}] @@ -33,7 +34,7 @@ def build_tags(_) do [{:meta, [property: "twitter:card", content: "summary"], []}] end - def find_first_acceptable_media_type(%{data: %{"object" => %{"attachment" => attachment}}}) do + def find_first_acceptable_media_type(%{data: %{"attachment" => attachment}}) do Enum.find_value(attachment, fn attachment -> Enum.find_value(attachment["url"], fn url -> Enum.find(["image", "audio", "video"], fn media_type -> diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index f7ba57389..4844b84ad 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -145,10 +145,11 @@ def notice(conn, %{"id" => id}) do %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> - # Only Create actvities have a map at object - if is_map(activity.data["object"]) do + if activity.data["type"] == "Create" do + %Object{} = object = Object.normalize(activity.data["object"]) + Fallback.RedirectController.redirector_with_meta(conn, %{ - activity: activity, + object: object, user: user }) else diff --git a/test/web/metadata/opengraph_test.exs b/test/web/metadata/opengraph_test.exs index 1c4fadc46..c3502bad3 100644 --- a/test/web/metadata/opengraph_test.exs +++ b/test/web/metadata/opengraph_test.exs @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Metadata.Providers.OpenGraphTest do use Pleroma.DataCase import Pleroma.Factory @@ -47,19 +48,7 @@ test "it renders all supported types of attachments and skips unknown types" do } }) - note_activity = - insert(:note_activity, %{ - data: %{ - "actor" => note.data["actor"], - "to" => note.data["to"], - "object" => note.data, - "context" => note.data["context"] - }, - actor: note.data["actor"], - recipients: note.data["to"] - }) - - result = OpenGraph.build_tags(%{activity: note_activity, user: user}) + result = OpenGraph.build_tags(%{object: note, user: user}) assert Enum.all?( [ @@ -85,6 +74,7 @@ test "it does not render attachments if post is nsfw" do "id" => "https://pleroma.gov/objects/whatever", "content" => "#cuteposting #nsfw #hambaga", "tag" => ["cuteposting", "nsfw", "hambaga"], + "sensitive" => true, "attachment" => [ %{ "url" => [ @@ -95,19 +85,7 @@ test "it does not render attachments if post is nsfw" do } }) - note_activity = - insert(:note_activity, %{ - data: %{ - "actor" => note.data["actor"], - "to" => note.data["to"], - "object" => note.data, - "context" => note.data["context"] - }, - actor: note.data["actor"], - recipients: note.data["to"] - }) - - result = OpenGraph.build_tags(%{activity: note_activity, user: user}) + result = OpenGraph.build_tags(%{object: note, user: user}) assert {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []} in result From 997f4a5e09f8531f95ba3f13986b9dda583d046a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Jan 2019 10:28:19 +0300 Subject: [PATCH 56/61] Remove custom emojis and trailing whitespaces from previews --- lib/pleroma/formatter.ex | 21 +++++++++++++++++---- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/metadata/opengraph.ex | 2 ++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 63e0acb21..2696f41c0 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -43,7 +43,7 @@ def emojify(text) do def emojify(text, nil), do: text - def emojify(text, emoji) do + def emojify(text, emoji, strip \\ false) do Enum.reduce(emoji, text, fn {emoji, file}, text -> emoji = HTML.strip_tags(emoji) file = HTML.strip_tags(file) @@ -51,14 +51,24 @@ def emojify(text, emoji) do String.replace( text, ":#{emoji}:", - "#{emoji}" + if not strip do + "#{emoji}" + else + "" + end ) |> HTML.filter_tags() end) end + def demojify(text) do + emojify(text, Emoji.get_all(), true) + end + + def demojify(text, nil), do: text + def get_emoji(text) when is_binary(text) do Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end) end @@ -185,6 +195,9 @@ def finalize({subs, text}) do end def truncate(text, max_length \\ 200, omission \\ "...") do + # Remove trailing whitespace + text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") + if String.length(text) < max_length do text else diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2902905fd..9a748d65e 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -103,7 +103,7 @@ def post(user, %{"status" => status} = data) do attachments, tags, get_content_type(data["content_type"]), - Enum.member?([true, "true"], data["no_attachment_links"]) + true ), context <- make_context(inReplyTo), cw <- data["spoiler_text"], diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 43303859c..1028e35c2 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -124,6 +124,7 @@ defp scrub_html_and_truncate(%{data: %{"content" => content}} = object) do |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.get_cached_stripped_html_for_object(object, __MODULE__) + |> Formatter.demojify() |> Formatter.truncate() end @@ -133,6 +134,7 @@ defp scrub_html_and_truncate(content) when is_binary(content) do |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.strip_tags() + |> Formatter.demojify() |> Formatter.truncate() end From 7bc6208b2fa53b04d0d319e62c7c9b34d18e0e44 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Jan 2019 10:35:52 +0300 Subject: [PATCH 57/61] Fix ostatus test --- test/web/ostatus/ostatus_controller_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 5a94cb3b3..954abf5fe 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -131,12 +131,14 @@ test "404s on deleted objects", %{conn: conn} do object = Object.get_by_ap_id(note_activity.data["object"]["id"]) conn + |> put_req_header("accept", "application/xml") |> get("/objects/#{uuid}") |> response(200) Object.delete(object) conn + |> put_req_header("accept", "application/xml") |> get("/objects/#{uuid}") |> response(404) end From a38f21f51dc810221cbf910556910a10f6104956 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Jan 2019 18:30:06 +0300 Subject: [PATCH 58/61] unfurl_nsfw: false by default --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3448e31ca..9c4d73465 100644 --- a/config/config.exs +++ b/config/config.exs @@ -207,7 +207,7 @@ ip: {0, 0, 0, 0}, port: 9999 -config :pleroma, Pleroma.Web.Metadata, providers: [] +config :pleroma, Pleroma.Web.Metadata, providers: [], unfurl_nsfw: false config :pleroma, :suggestions, enabled: false, From afd83db0d7be4890dd345ba1f78e46da8b74d449 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 19 Jan 2019 10:58:27 +0300 Subject: [PATCH 59/61] Provide local og:url for remote activities --- lib/pleroma/web/metadata/opengraph.ex | 3 ++- lib/pleroma/web/ostatus/ostatus_controller.ex | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 1028e35c2..30333785e 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do @impl Provider def build_tags(%{ object: object, + url: url, user: user }) do attachments = build_attachments(object) @@ -37,7 +38,7 @@ def build_tags(%{ property: "og:title", content: "#{user.name}" <> content ], []}, - {:meta, [property: "og:url", content: object.data["id"]], []}, + {:meta, [property: "og:url", content: url], []}, {:meta, [ property: "og:description", diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 4844b84ad..f04754f08 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -150,6 +150,7 @@ def notice(conn, %{"id" => id}) do Fallback.RedirectController.redirector_with_meta(conn, %{ object: object, + url: Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), user: user }) else From 44693fbf6e5c5ec5622207e263688e3af7d1a83a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 20 Jan 2019 01:28:46 +0300 Subject: [PATCH 60/61] Formating --- lib/pleroma/web/ostatus/ostatus_controller.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index f04754f08..e94c5415a 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -150,7 +150,12 @@ def notice(conn, %{"id" => id}) do Fallback.RedirectController.redirector_with_meta(conn, %{ object: object, - url: Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), + url: + Pleroma.Web.Router.Helpers.o_status_url( + Pleroma.Web.Endpoint, + :notice, + activity.id + ), user: user }) else From a45a90348046902df831d3a69f98677d8a0329db Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 25 Jan 2019 10:51:06 +0300 Subject: [PATCH 61/61] Fix OGP provider tests --- test/web/metadata/opengraph_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/metadata/opengraph_test.exs b/test/web/metadata/opengraph_test.exs index c3502bad3..4283f72cd 100644 --- a/test/web/metadata/opengraph_test.exs +++ b/test/web/metadata/opengraph_test.exs @@ -48,7 +48,7 @@ test "it renders all supported types of attachments and skips unknown types" do } }) - result = OpenGraph.build_tags(%{object: note, user: user}) + result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user}) assert Enum.all?( [ @@ -85,7 +85,7 @@ test "it does not render attachments if post is nsfw" do } }) - result = OpenGraph.build_tags(%{object: note, user: user}) + result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user}) assert {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []} in result