server: refactor fetching

- The `timeout` parameter does not exist in `node-fetch`, so the timeout was not
  working properly.
- Refactor the User-Agent header to be set in a central place instead of several
  different places.
- Refactor more places to use getResult which handles the timeout and everything
  else already instead of the normal `fetch` provided by `node-fetch`.

packages/backend/src/misc/captcha.ts

@@ -1,7 +1,6 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
+import { getResponse } from '@/misc/fetch.js';
 import config from '@/config/index.js';
-import { getAgentByUrl } from './fetch.js';
 
 export async function verifyRecaptcha(secret: string, response: string): Promise<void> {
 	const result = await getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(e => {
@@ -36,15 +35,10 @@ async function getCaptchaResponse(url: string, secret: string, response: string)
 		response,
 	});
 
-	const res = await fetch(url, {
+	const res = await getResponse({
+		url,
 		method: 'POST',
 		body: params,
-		headers: {
-			'User-Agent': config.userAgent,
-		},
-		// TODO
-		//timeout: 10 * 1000,
-		agent: getAgentByUrl,
 	}).catch(e => {
 		throw new Error(`${e.message || e}`);
 	});

packages/backend/src/misc/fetch.ts

@@ -7,48 +7,45 @@ import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import { SECOND } from '@/const.js';
 import config from '@/config/index.js';
 
-export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
 	const res = await getResponse({
 		url,
 		method: 'GET',
 		headers: Object.assign({
-			'User-Agent': config.userAgent,
 			Accept: accept,
-		}, headers || {}),
+		}, headers),
 		timeout,
 	});
 
 	return await res.json();
 }
 
-export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
+export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
 	const res = await getResponse({
 		url,
 		method: 'GET',
 		headers: Object.assign({
-			'User-Agent': config.userAgent,
 			Accept: accept,
-		}, headers || {}),
+		}, headers),
 		timeout,
 	});
 
 	return await res.text();
 }
 
-export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number, redirect: 'follow' | 'manual' | 'error' = 'follow' }) {
-	const timeout = args.timeout || 10 * SECOND;
-
+export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout: number = 10 * SECOND, size?: number, redirect: 'follow' | 'manual' | 'error' = 'follow' }) {
 	const controller = new AbortController();
 	setTimeout(() => {
 		controller.abort();
-	}, timeout * 6);
+	}, timeout);
 
 	const res = await fetch(args.url, {
 		method: args.method,
-		headers: args.headers,
+		headers: Object.assign({
+			'User-Agent': config.userAgent,
+		}, args.headers),args.headers,
 		body: args.body,
 		redirect: args.redirect,
-		timeout,
 		size: args.size || 10 * 1024 * 1024, // 10 MiB
 		agent: getAgentByUrl,
 		signal: controller.signal,

packages/backend/src/server/api/endpoints/notes/translate.ts

@@ -1,7 +1,6 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
 import config from '@/config/index.js';
-import { getAgentByUrl } from '@/misc/fetch.js';
+import { getResponse } from '@/misc/fetch.js';
 import { fetchMeta } from '@/misc/fetch-meta.js';
 import { TranslationService } from '@/models/entities/meta.js';
 import { ApiError } from '@/server/api/error.js';
@@ -154,17 +153,14 @@ export default define(meta, paramDef, async (ps, user) => {
 			? 'https://api-free.deepl.com/v2/translate'
 			: 'https://api.deepl.com/v2/translate';
 
-		const res = await fetch(endpoint, {
+		const res = await getResponse({
+			url: endpoint,
 			method: 'POST',
 			headers: {
 				'Content-Type': 'application/x-www-form-urlencoded',
-				'User-Agent': config.userAgent,
 				Accept: 'application/json, */*',
 			},
 			body: params,
-			// TODO
-			//timeout: 10000,
-			agent: getAgentByUrl,
 		});
 
 		const json = (await res.json()) as {
@@ -179,7 +175,7 @@ export default define(meta, paramDef, async (ps, user) => {
 			text: json.translations[0].text,
 		};
 	}
-	
+
 	async function translateLibreTranslate(): Promise<Translation | number> {
 		if (note.text == null || instance.libreTranslateEndpoint == null) {
 			return 204;
@@ -198,7 +194,8 @@ export default define(meta, paramDef, async (ps, user) => {
 			api_key,
 		};
 
-		const res = await fetch(endpoint, {
+		const res = await getReponse({
+			url: endpoint,
 			method: 'POST',
 			body: JSON.stringify(params),
 			headers: { 'Content-Type': 'application/json' },