Merge branch 'fix/twittercard-video-dimensions' into 'develop'

Generate Twittercard/OGP metadata with correct dimensions

See merge request pleroma/pleroma!3456
This commit is contained in:
feld 2021-06-10 15:03:58 +00:00
commit 406dadb56b
6 changed files with 204 additions and 30 deletions

View file

@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising. - HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
- Email address is now returned if requesting user is the owner of the user account so it can be exposed in client and FE user settings UIs. - Email address is now returned if requesting user is the owner of the user account so it can be exposed in client and FE user settings UIs.
- Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available.
### Added ### Added

View file

@ -4,6 +4,7 @@
defmodule Pleroma.Web.Metadata.Providers.OpenGraph do defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MediaProxy
alias Pleroma.Web.Metadata alias Pleroma.Web.Metadata
alias Pleroma.Web.Metadata.Providers.Provider alias Pleroma.Web.Metadata.Providers.Provider
alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Metadata.Utils
@ -32,11 +33,11 @@ def build_tags(%{
property: "og:description", property: "og:description",
content: scrubbed_content content: scrubbed_content
], []}, ], []},
{:meta, [property: "og:type", content: "website"], []} {:meta, [property: "og:type", content: "article"], []}
] ++ ] ++
if attachments == [] or Metadata.activity_nsfw?(object) do if attachments == [] or Metadata.activity_nsfw?(object) do
[ [
{:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], {:meta, [property: "og:image", content: MediaProxy.preview_url(User.avatar_url(user))],
[]}, []},
{:meta, [property: "og:image:width", content: 150], []}, {:meta, [property: "og:image:width", content: 150], []},
{:meta, [property: "og:image:height", content: 150], []} {:meta, [property: "og:image:height", content: 150], []}
@ -57,8 +58,9 @@ def build_tags(%{user: user}) do
], []}, ], []},
{:meta, [property: "og:url", content: user.uri || user.ap_id], []}, {:meta, [property: "og:url", content: user.uri || user.ap_id], []},
{:meta, [property: "og:description", content: truncated_bio], []}, {:meta, [property: "og:description", content: truncated_bio], []},
{:meta, [property: "og:type", content: "website"], []}, {:meta, [property: "og:type", content: "article"], []},
{:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []}, {:meta, [property: "og:image", content: MediaProxy.preview_url(User.avatar_url(user))],
[]},
{:meta, [property: "og:image:width", content: 150], []}, {:meta, [property: "og:image:width", content: 150], []},
{:meta, [property: "og:image:height", content: 150], []} {:meta, [property: "og:image:height", content: 150], []}
] ]
@ -69,28 +71,35 @@ defp build_attachments(%{data: %{"attachment" => attachments}}) do
Enum.reduce(attachments, [], fn attachment, acc -> Enum.reduce(attachments, [], fn attachment, acc ->
rendered_tags = rendered_tags =
Enum.reduce(attachment["url"], [], fn url, acc -> Enum.reduce(attachment["url"], [], fn url, acc ->
# TODO: Add additional properties to objects when we have the data available. # TODO: Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
# Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
# object when a Video or GIF is attached it will display that in Whatsapp Rich Preview. # object when a Video or GIF is attached it will display that in Whatsapp Rich Preview.
case Utils.fetch_media_type(@media_types, url["mediaType"]) do case Utils.fetch_media_type(@media_types, url["mediaType"]) do
"audio" -> "audio" ->
[ [
{:meta, [property: "og:audio", content: Utils.attachment_url(url["href"])], []} {:meta, [property: "og:audio", content: MediaProxy.url(url["href"])], []}
| acc | acc
] ]
# Not using preview_url for this. It saves bandwidth, but the image dimensions will
# be wrong. We generate it on the fly and have no way to capture or analyze the
# image to get the dimensions. This can be an issue for apps/FEs rendering images
# in timelines too, but you can get clever with the aspect ratio metadata as a
# workaround.
"image" -> "image" ->
[ [
{:meta, [property: "og:image", content: Utils.attachment_url(url["href"])], []}, {:meta, [property: "og:image", content: MediaProxy.url(url["href"])], []},
{:meta, [property: "og:image:alt", content: attachment["name"]], []} {:meta, [property: "og:image:alt", content: attachment["name"]], []}
| acc | acc
] ]
|> maybe_add_dimensions(url)
"video" -> "video" ->
[ [
{:meta, [property: "og:video", content: Utils.attachment_url(url["href"])], []} {:meta, [property: "og:video", content: MediaProxy.url(url["href"])], []}
| acc | acc
] ]
|> maybe_add_dimensions(url)
|> maybe_add_video_thumbnail(url)
_ -> _ ->
acc acc
@ -102,4 +111,38 @@ defp build_attachments(%{data: %{"attachment" => attachments}}) do
end end
defp build_attachments(_), do: [] defp build_attachments(_), do: []
# We can use url["mediaType"] to dynamically fill the metadata
defp maybe_add_dimensions(metadata, url) do
type = url["mediaType"] |> String.split("/") |> List.first()
cond do
!is_nil(url["height"]) && !is_nil(url["width"]) ->
metadata ++
[
{:meta, [property: "og:#{type}:width", content: "#{url["width"]}"], []},
{:meta, [property: "og:#{type}:height", content: "#{url["height"]}"], []}
]
true ->
metadata
end
end
# Media Preview Proxy makes thumbnails of videos without resizing, so we can trust the
# width and height of the source video.
defp maybe_add_video_thumbnail(metadata, url) do
cond do
Pleroma.Config.get([:media_preview_proxy, :enabled], false) ->
metadata ++
[
{:meta, [property: "og:image:width", content: "#{url["width"]}"], []},
{:meta, [property: "og:image:height", content: "#{url["height"]}"], []},
{:meta, [property: "og:image", content: MediaProxy.preview_url(url["href"])], []}
]
true ->
metadata
end
end
end end

View file

@ -5,6 +5,7 @@
defmodule Pleroma.Web.Metadata.Providers.TwitterCard do defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.MediaProxy
alias Pleroma.Web.Metadata alias Pleroma.Web.Metadata
alias Pleroma.Web.Metadata.Providers.Provider alias Pleroma.Web.Metadata.Providers.Provider
alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Metadata.Utils
@ -48,14 +49,14 @@ defp title_tag(user) do
end end
def image_tag(user) do def image_tag(user) do
{:meta, [property: "twitter:image", content: Utils.attachment_url(User.avatar_url(user))], []} {:meta, [property: "twitter:image", content: MediaProxy.preview_url(User.avatar_url(user))],
[]}
end end
defp build_attachments(id, %{data: %{"attachment" => attachments}}) do defp build_attachments(id, %{data: %{"attachment" => attachments}}) do
Enum.reduce(attachments, [], fn attachment, acc -> Enum.reduce(attachments, [], fn attachment, acc ->
rendered_tags = rendered_tags =
Enum.reduce(attachment["url"], [], fn url, acc -> Enum.reduce(attachment["url"], [], fn url, acc ->
# TODO: Add additional properties to objects when we have the data available.
case Utils.fetch_media_type(@media_types, url["mediaType"]) do case Utils.fetch_media_type(@media_types, url["mediaType"]) do
"audio" -> "audio" ->
[ [
@ -66,26 +67,35 @@ defp build_attachments(id, %{data: %{"attachment" => attachments}}) do
| acc | acc
] ]
# Not using preview_url for this. It saves bandwidth, but the image dimensions will
# be wrong. We generate it on the fly and have no way to capture or analyze the
# image to get the dimensions. This can be an issue for apps/FEs rendering images
# in timelines too, but you can get clever with the aspect ratio metadata as a
# workaround.
"image" -> "image" ->
[ [
{:meta, [property: "twitter:card", content: "summary_large_image"], []}, {:meta, [property: "twitter:card", content: "summary_large_image"], []},
{:meta, {:meta,
[ [
property: "twitter:player", property: "twitter:player",
content: Utils.attachment_url(url["href"]) content: MediaProxy.url(url["href"])
], []} ], []}
| acc | acc
] ]
|> maybe_add_dimensions(url)
# TODO: Need the true width and height values here or Twitter renders an iFrame with
# a bad aspect ratio
"video" -> "video" ->
# fallback to old placeholder values
height = url["height"] || 480
width = url["width"] || 480
[ [
{:meta, [property: "twitter:card", content: "player"], []}, {:meta, [property: "twitter:card", content: "player"], []},
{:meta, [property: "twitter:player", content: player_url(id)], []}, {:meta, [property: "twitter:player", content: player_url(id)], []},
{:meta, [property: "twitter:player:width", content: "480"], []}, {:meta, [property: "twitter:player:width", content: "#{width}"], []},
{:meta, [property: "twitter:player:height", content: "480"], []}, {:meta, [property: "twitter:player:height", content: "#{height}"], []},
{:meta, [property: "twitter:player:stream", content: url["href"]], []}, {:meta, [property: "twitter:player:stream", content: MediaProxy.url(url["href"])],
[]},
{:meta, {:meta,
[property: "twitter:player:stream:content_type", content: url["mediaType"]], []} [property: "twitter:player:stream:content_type", content: url["mediaType"]], []}
| acc | acc
@ -105,4 +115,20 @@ defp build_attachments(_id, _object), do: []
defp player_url(id) do defp player_url(id) do
Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice_player, id) Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice_player, id)
end end
# Videos have problems without dimensions, but we used to not provide WxH for images.
# A default (read: incorrect) fallback for images is likely to cause rendering bugs.
defp maybe_add_dimensions(metadata, url) do
cond do
!is_nil(url["height"]) && !is_nil(url["width"]) ->
metadata ++
[
{:meta, [property: "twitter:player:width", content: "#{url["width"]}"], []},
{:meta, [property: "twitter:player:height", content: "#{url["height"]}"], []}
]
true ->
metadata
end
end
end end

View file

@ -7,7 +7,6 @@ defmodule Pleroma.Web.Metadata.Utils do
alias Pleroma.Emoji alias Pleroma.Emoji
alias Pleroma.Formatter alias Pleroma.Formatter
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Web.MediaProxy
def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
content content
@ -38,10 +37,6 @@ def scrub_html(content) when is_binary(content) do
def scrub_html(content), do: content def scrub_html(content), do: content
def attachment_url(url) do
MediaProxy.preview_url(url)
end
def user_name_string(user) do def user_name_string(user) do
"#{user.name} " <> "#{user.name} " <>
if user.local do if user.local do

View file

@ -22,7 +22,12 @@ test "it renders all supported types of attachments and skips unknown types" do
"attachment" => [ "attachment" => [
%{ %{
"url" => [ "url" => [
%{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"} %{
"mediaType" => "image/png",
"href" => "https://pleroma.gov/tenshi.png",
"height" => 1024,
"width" => 1280
}
] ]
}, },
%{ %{
@ -35,7 +40,12 @@ test "it renders all supported types of attachments and skips unknown types" do
}, },
%{ %{
"url" => [ "url" => [
%{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"} %{
"mediaType" => "video/webm",
"href" => "https://pleroma.gov/about/juche.webm",
"height" => 600,
"width" => 800
}
] ]
}, },
%{ %{
@ -55,11 +65,15 @@ test "it renders all supported types of attachments and skips unknown types" do
assert Enum.all?( assert Enum.all?(
[ [
{:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []}, {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []},
{:meta, [property: "og:image:width", content: "1280"], []},
{:meta, [property: "og:image:height", content: "1024"], []},
{:meta, {:meta,
[property: "og:audio", content: "http://www.gnu.org/music/free-software-song.au"], [property: "og:audio", content: "http://www.gnu.org/music/free-software-song.au"],
[]}, []},
{:meta, [property: "og:video", content: "https://pleroma.gov/about/juche.webm"], {:meta, [property: "og:video", content: "https://pleroma.gov/about/juche.webm"],
[]} []},
{:meta, [property: "og:video:width", content: "800"], []},
{:meta, [property: "og:video:height", content: "600"], []}
], ],
fn element -> element in result end fn element -> element in result end
) )
@ -93,4 +107,84 @@ test "it does not render attachments if post is nsfw" do
refute {:meta, [property: "og:image", content: "https://misskey.microsoft/corndog.png"], []} in result refute {:meta, [property: "og:image", content: "https://misskey.microsoft/corndog.png"], []} in result
end end
test "video attachments have image thumbnail with WxH metadata with Preview Proxy enabled" do
clear_config([:media_proxy, :enabled], true)
clear_config([:media_preview_proxy, :enabled], true)
user = insert(:user)
note =
insert(:note, %{
data: %{
"actor" => user.ap_id,
"id" => "https://pleroma.gov/objects/whatever",
"content" => "test video post",
"sensitive" => false,
"attachment" => [
%{
"url" => [
%{
"mediaType" => "video/webm",
"href" => "https://pleroma.gov/about/juche.webm",
"height" => 600,
"width" => 800
}
]
}
]
}
})
result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user})
assert {:meta, [property: "og:image:width", content: "800"], []} in result
assert {:meta, [property: "og:image:height", content: "600"], []} in result
assert {:meta,
[
property: "og:image",
content:
"http://localhost:4001/proxy/preview/LzAnlke-l5oZbNzWsrHfprX1rGw/aHR0cHM6Ly9wbGVyb21hLmdvdi9hYm91dC9qdWNoZS53ZWJt/juche.webm"
], []} in result
end
test "video attachments have no image thumbnail with Preview Proxy disabled" do
clear_config([:media_proxy, :enabled], true)
clear_config([:media_preview_proxy, :enabled], false)
user = insert(:user)
note =
insert(:note, %{
data: %{
"actor" => user.ap_id,
"id" => "https://pleroma.gov/objects/whatever",
"content" => "test video post",
"sensitive" => false,
"attachment" => [
%{
"url" => [
%{
"mediaType" => "video/webm",
"href" => "https://pleroma.gov/about/juche.webm",
"height" => 600,
"width" => 800
}
]
}
]
}
})
result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user})
refute {:meta, [property: "og:image:width", content: "800"], []} in result
refute {:meta, [property: "og:image:height", content: "600"], []} in result
refute {:meta,
[
property: "og:image",
content:
"http://localhost:4001/proxy/preview/LzAnlke-l5oZbNzWsrHfprX1rGw/aHR0cHM6Ly9wbGVyb21hLmdvdi9hYm91dC9qdWNoZS53ZWJt/juche.webm"
], []} in result
end
end end

View file

@ -9,6 +9,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
alias Pleroma.Web.Endpoint alias Pleroma.Web.Endpoint
alias Pleroma.Web.MediaProxy
alias Pleroma.Web.Metadata.Providers.TwitterCard alias Pleroma.Web.Metadata.Providers.TwitterCard
alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Metadata.Utils
alias Pleroma.Web.Router alias Pleroma.Web.Router
@ -17,7 +18,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCardTest do
test "it renders twitter card for user info" do test "it renders twitter card for user info" do
user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
avatar_url = Utils.attachment_url(User.avatar_url(user)) avatar_url = MediaProxy.preview_url(User.avatar_url(user))
res = TwitterCard.build_tags(%{user: user}) res = TwitterCard.build_tags(%{user: user})
assert res == [ assert res == [
@ -111,7 +112,14 @@ test "it renders supported types of attachments and skips unknown types" do
"content" => "pleroma in a nutshell", "content" => "pleroma in a nutshell",
"attachment" => [ "attachment" => [
%{ %{
"url" => [%{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"}] "url" => [
%{
"mediaType" => "image/png",
"href" => "https://pleroma.gov/tenshi.png",
"height" => 1024,
"width" => 1280
}
]
}, },
%{ %{
"url" => [ "url" => [
@ -123,7 +131,12 @@ test "it renders supported types of attachments and skips unknown types" do
}, },
%{ %{
"url" => [ "url" => [
%{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"} %{
"mediaType" => "video/webm",
"href" => "https://pleroma.gov/about/juche.webm",
"height" => 600,
"width" => 800
}
] ]
} }
] ]
@ -137,14 +150,16 @@ test "it renders supported types of attachments and skips unknown types" do
{:meta, [property: "twitter:description", content: "pleroma in a nutshell"], []}, {:meta, [property: "twitter:description", content: "pleroma in a nutshell"], []},
{:meta, [property: "twitter:card", content: "summary_large_image"], []}, {:meta, [property: "twitter:card", content: "summary_large_image"], []},
{:meta, [property: "twitter:player", content: "https://pleroma.gov/tenshi.png"], []}, {:meta, [property: "twitter:player", content: "https://pleroma.gov/tenshi.png"], []},
{:meta, [property: "twitter:player:width", content: "1280"], []},
{:meta, [property: "twitter:player:height", content: "1024"], []},
{:meta, [property: "twitter:card", content: "player"], []}, {:meta, [property: "twitter:card", content: "player"], []},
{:meta, {:meta,
[ [
property: "twitter:player", property: "twitter:player",
content: Router.Helpers.o_status_url(Endpoint, :notice_player, activity.id) content: Router.Helpers.o_status_url(Endpoint, :notice_player, activity.id)
], []}, ], []},
{:meta, [property: "twitter:player:width", content: "480"], []}, {:meta, [property: "twitter:player:width", content: "800"], []},
{:meta, [property: "twitter:player:height", content: "480"], []}, {:meta, [property: "twitter:player:height", content: "600"], []},
{:meta, {:meta,
[ [
property: "twitter:player:stream", property: "twitter:player:stream",