server: refactor fetching

- The `timeout` parameter does not exist in `node-fetch`, so the timeout was not
  working properly.
- Refactor the User-Agent header to be set in a central place instead of several
  different places.
- Refactor more places to use getResult which handles the timeout and everything
  else already instead of the normal `fetch` provided by `node-fetch`.

packages/backend/src/misc/captcha.ts

@@ -1,7 +1,6 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
+import { getResponse } from '@/misc/fetch.js';
 import config from '@/config/index.js';
-import { getAgentByUrl } from './fetch.js';
 
 export async function verifyRecaptcha(secret: string, response: string): Promise<void> {
 	const result = await getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(e => {
@@ -36,15 +35,10 @@ async function getCaptchaResponse(url: string, secret: string, response: string)
 		response,
 	});
 
-	const res = await fetch(url, {
+	const res = await getResponse({
+		url,
 		method: 'POST',
 		body: params,
-		headers: {
-			'User-Agent': config.userAgent,
-		},
-		// TODO
-		//timeout: 10 * 1000,
-		agent: getAgentByUrl,
 	}).catch(e => {
 		throw new Error(`${e.message || e}`);
 	});

packages/backend/src/misc/fetch.ts

@@ -7,48 +7,45 @@ import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import { SECOND } from '@/const.js';
 import config from '@/config/index.js';
 
-export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
 	const res = await getResponse({
 		url,
 		method: 'GET',
 		headers: Object.assign({
-			'User-Agent': config.userAgent,
 			Accept: accept,
-		}, headers || {}),
+		}, headers),
 		timeout,
 	});
 
 	return await res.json();
 }
 
-export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
+export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
 	const res = await getResponse({
 		url,
 		method: 'GET',
 		headers: Object.assign({
-			'User-Agent': config.userAgent,
 			Accept: accept,
-		}, headers || {}),
+		}, headers),
 		timeout,
 	});
 
 	return await res.text();
 }
 
-export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number, redirect: 'follow' | 'manual' | 'error' = 'follow' }) {
-	const timeout = args.timeout || 10 * SECOND;
-
+export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout: number = 10 * SECOND, size?: number, redirect: 'follow' | 'manual' | 'error' = 'follow' }) {
 	const controller = new AbortController();
 	setTimeout(() => {
 		controller.abort();
-	}, timeout * 6);
+	}, timeout);
 
 	const res = await fetch(args.url, {
 		method: args.method,
-		headers: args.headers,
+		headers: Object.assign({
+			'User-Agent': config.userAgent,
+		}, args.headers),args.headers,
 		body: args.body,
 		redirect: args.redirect,
-		timeout,
 		size: args.size || 10 * 1024 * 1024, // 10 MiB
 		agent: getAgentByUrl,
 		signal: controller.signal,

packages/backend/src/remote/activitypub/kernel/update/note.ts

@@ -20,8 +20,6 @@ export async function update(actor: IRemoteUser, note: IObject, resolver: Resolv
 		try {
 			// if creating was successful...
 			const existsNow = await Notes.findOneByOrFail({ uri });
-			// set the updatedAt timestamp since the note was changed
-			await Notes.update(existsNow.id, { updatedAt: new Date() });
 			return 'ok: unknown note created and marked as updated';
 		} catch (e) {
 			return `skip: updated note unknown and creating rejected: ${e.message}`;

packages/backend/src/remote/activitypub/models/note.ts

@@ -278,6 +278,7 @@ export async function createNote(value: string | IObject, resolver: Resolver, si
 		return await post(actor, {
 			...processedContent,
 			createdAt: note.published ? new Date(note.published) : null,
+			updatedAt: note.updated,
 			reply,
 			renote: quote,
 			localOnly: false,
@@ -354,7 +355,7 @@ export async function updateNote(value: IPost, actor: User, resolver: Resolver):
 
 	// update note content itself
 	await Notes.update(exists.id, {
-		updatedAt: new Date(),
+		updatedAt: value.updated ?? new Date(),
 
 		cw: processedContent.cw,
 		fileIds: processedContent.files.map(file => file.id),

packages/backend/src/server/api/endpoints/notes/translate.ts

@@ -1,7 +1,6 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
 import config from '@/config/index.js';
-import { getAgentByUrl } from '@/misc/fetch.js';
+import { getResponse } from '@/misc/fetch.js';
 import { fetchMeta } from '@/misc/fetch-meta.js';
 import { TranslationService } from '@/models/entities/meta.js';
 import { ApiError } from '@/server/api/error.js';
@@ -154,17 +153,14 @@ export default define(meta, paramDef, async (ps, user) => {
 			? 'https://api-free.deepl.com/v2/translate'
 			: 'https://api.deepl.com/v2/translate';
 
-		const res = await fetch(endpoint, {
+		const res = await getResponse({
+			url: endpoint,
 			method: 'POST',
 			headers: {
 				'Content-Type': 'application/x-www-form-urlencoded',
-				'User-Agent': config.userAgent,
 				Accept: 'application/json, */*',
 			},
 			body: params,
-			// TODO
-			//timeout: 10000,
-			agent: getAgentByUrl,
 		});
 
 		const json = (await res.json()) as {
@@ -179,7 +175,7 @@ export default define(meta, paramDef, async (ps, user) => {
 			text: json.translations[0].text,
 		};
 	}
-	
+
 	async function translateLibreTranslate(): Promise<Translation | number> {
 		if (note.text == null || instance.libreTranslateEndpoint == null) {
 			return 204;
@@ -198,7 +194,8 @@ export default define(meta, paramDef, async (ps, user) => {
 			api_key,
 		};
 
-		const res = await fetch(endpoint, {
+		const res = await getReponse({
+			url: endpoint,
 			method: 'POST',
 			body: JSON.stringify(params),
 			headers: { 'Content-Type': 'application/json' },

packages/backend/src/services/note/create.ts

@@ -26,6 +26,7 @@ type MinimumUser = {
 
 type Option = {
 	createdAt?: Date | null;
+	updatedAt?: Date | null;
 	name?: string | null;
 	text?: string | null;
 	reply?: Note | null;
@@ -159,6 +160,7 @@ async function insertNote(user: { id: User['id']; host: User['host']; }, data: O
 	const insert = new Note({
 		id: genId(createdAt),
 		createdAt,
+		updatedAt: data.updatedAt ?? null,
 		fileIds: data.files?.map(file => file.id) ?? [],
 		replyId: data.reply?.id ?? null,
 		renoteId: data.renote?.id ?? null,