From 39fb7e594624957ec2e0f15ec2ec6332901587af Mon Sep 17 00:00:00 2001
From: Johann150 <johann.galle@protonmail.com>
Date: Wed, 1 Feb 2023 11:34:09 +0100
Subject: [PATCH] server: improve OpenGraph data for note attachments

With this change, not all files will be proclaimed to be image files. Only
images, videos and audio files will be represented with OpenGraph data.

More properties for these files will also be represented, e.g. image alt text.

However, if the note has a CW or any of the files are marked sensitive, none
of the files will be used.

The users profile picture will not be used any more.

Changelog: Changed
---
 packages/backend/src/server/web/index.ts      | 64 ++++++++++++++++++-
 .../backend/src/server/web/views/note.pug     |  3 +-
 2 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/packages/backend/src/server/web/index.ts b/packages/backend/src/server/web/index.ts
index 85ad59f3a..4d1726421 100644
--- a/packages/backend/src/server/web/index.ts
+++ b/packages/backend/src/server/web/index.ts
@@ -18,7 +18,7 @@ import { KoaAdapter } from '@bull-board/koa';
 import { In, IsNull } from 'typeorm';
 import { fetchMeta } from '@/misc/fetch-meta.js';
 import config from '@/config/index.js';
-import { Users, Notes, UserProfiles, Pages, Channels, Clips } from '@/models/index.js';
+import { Users, Notes, UserProfiles, Pages, Channels, Clips, DriveFiles } from '@/models/index.js';
 import * as Acct from '@/misc/acct.js';
 import { getNoteSummary } from '@/misc/get-note-summary.js';
 import { queues } from '@/queue/queues.js';
@@ -327,10 +327,70 @@ router.get('/notes/:note', async (ctx, next) => {
 			const packedNote = await Notes.pack(note);
 			const profile = await UserProfiles.findOneByOrFail({ userId: note.userId });
 			const meta = await fetchMeta();
+
+			// If the note has a CW (is sensitive as a whole) or any of the files is sensitive or there are no
+			// files, they are not used for a preview.
+			let filesOpengraph = [];
+			if (!packedNote.cw || packedNote.files.length > 0 || packedNote.files.all(file => !file.isSensitive)) {
+				let limit = 4;
+				for (const file of packedNote.files) {
+					if (file.type.startsWith('image/')) {
+						filesOpengraph.push([
+							"og:image",
+							DriveFiles.getPublicUrl(file, true),
+						]);
+						filesOpengraph.push([
+							"og:image:type",
+							file.type,
+						]);
+						if (file.properties != null) {
+							filesOpengraph.push([
+								"og:image:width",
+								file.properties?.width,
+							]);
+							filesOpengraph.push([
+								"og:image:height",
+								file.properties?.height,
+							]);
+						}
+						if (file.comment) {
+							filesOpengraph.push([
+								"og:image:alt",
+								file.comment,
+							]);
+						}
+					} else if (file.type.startsWith('audio/')) {
+						filesOpengraph.push([
+							"og:audio",
+							DriveFiles.getPublicUrl(file),
+						]);
+						filesOpengraph.push([
+							"og:audio:type",
+							file.type,
+						]);
+					} else if (file.type.startsWith('video/')) {
+						filesOpengraph.push([
+							"og:video",
+							DriveFiles.getPublicUrl(file),
+						]);
+						filesOpengraph.push([
+							"og:video:type",
+							file.type,
+						]);
+					} else {
+						// doesn't count towards the limit
+						continue;
+					}
+
+					// limit the number of presented attachments
+					if (--limit < 0) break;
+				}
+			}
+
 			await ctx.render('note', {
 				note: packedNote,
 				profile,
-				avatarUrl: await Users.getAvatarUrl(await Users.findOneByOrFail({ id: note.userId })),
+				filesOpengraph,
 				// TODO: Let locale changeable by instance setting
 				summary: getNoteSummary(packedNote),
 				instanceName: meta.name || 'FoundKey',
diff --git a/packages/backend/src/server/web/views/note.pug b/packages/backend/src/server/web/views/note.pug
index 37e8f8a41..0d9f50d89 100644
--- a/packages/backend/src/server/web/views/note.pug
+++ b/packages/backend/src/server/web/views/note.pug
@@ -17,7 +17,8 @@ block og
 	meta(property='og:title'       content= title)
 	meta(property='og:description' content= summary)
 	meta(property='og:url'         content= url)
-	meta(property='og:image'       content= avatarUrl)
+	for opengraphTag in filesOpengraph
+		meta(property=opengraphTag[0] content=opengraphTag[1])
 
 block meta
 	if user.host || isRenote || profile.noCrawle