Use node-fetch instead of request (#6228)

* requestをnode-fetchになど

* format

* fix error

* t

* Fix test

package.json

@@ -72,6 +72,7 @@
 		"@types/markdown-it": "0.0.9",
 		"@types/mocha": "7.0.2",
 		"@types/node": "13.11.0",
+		"@types/node-fetch": "2.5.5",
 		"@types/nodemailer": "6.4.0",
 		"@types/nprogress": "0.2.0",
 		"@types/oauth": "0.9.1",
@@ -84,8 +85,6 @@
 		"@types/ratelimiter": "2.1.28",
 		"@types/redis": "2.8.17",
 		"@types/rename": "1.0.1",
-		"@types/request": "2.48.4",
-		"@types/request-promise-native": "1.0.17",
 		"@types/request-stats": "3.0.0",
 		"@types/rimraf": "2.0.3",
 		"@types/seedrandom": "2.4.28",
@@ -102,7 +101,6 @@
 		"@types/websocket": "1.0.0",
 		"@types/ws": "7.2.3",
 		"@typescript-eslint/parser": "2.26.0",
-		"agentkeepalive": "4.1.0",
 		"animejs": "3.1.0",
 		"apexcharts": "3.17.1",
 		"autobind-decorator": "2.4.0",
@@ -145,6 +143,7 @@
 		"gulp-typescript": "5.0.1",
 		"hard-source-webpack-plugin": "0.13.1",
 		"html-minifier": "4.0.0",
+		"http-proxy-agent": "4.0.1",
 		"http-signature": "1.3.4",
 		"https-proxy-agent": "5.0.0",
 		"insert-text-at-cursor": "0.3.0",
@@ -205,8 +204,6 @@
 		"redis-lock": "0.1.4",
 		"reflect-metadata": "0.1.13",
 		"rename": "1.0.4",
-		"request": "2.88.2",
-		"request-promise-native": "1.0.8",
 		"request-stats": "3.0.0",
 		"require-all": "3.0.0",
 		"rimraf": "3.0.2",

src/misc/donwload-url.ts

@@ -1,5 +1,6 @@
 import * as fs from 'fs';
-import * as request from 'request';
+import fetch from 'node-fetch';
+import { httpAgent, httpsAgent } from './fetch';
 import config from '../config';
 import * as chalk from 'chalk';
 import Logger from '../services/logger';
@@ -7,11 +8,35 @@ import Logger from '../services/logger';
 export async function downloadUrl(url: string, path: string) {
 	const logger = new Logger('download');
 
-	await new Promise((res, rej) => {
-		logger.info(`Downloading ${chalk.cyan(url)} ...`);
+	logger.info(`Downloading ${chalk.cyan(url)} ...`);
 
+	const response = await fetch(new URL(url).href, {
+		headers: {
+			'User-Agent': config.userAgent
+		},
+		timeout: 10 * 1000,
+		agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+	}).then(response => {
+		if (!response.ok) {
+			logger.error(`Got ${response.status} (${url})`);
+			throw response.status;
+		} else {
+			return response;
+		}
+	});
+
+	await new Promise((res, rej) => {
 		const writable = fs.createWriteStream(path);
 
+		response.body.on('error', (error: any) => {
+			logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
+				url: url,
+				e: error
+			});
+			writable.close();
+			rej(error);
+		});
+
 		writable.on('finish', () => {
 			logger.succ(`Download finished: ${chalk.cyan(url)}`);
 			res();
@@ -25,35 +50,6 @@ export async function downloadUrl(url: string, path: string) {
 			rej(error);
 		});
 
-		const req = request({
-			url: new URL(url).href, // https://github.com/syuilo/misskey/issues/2637
-			proxy: config.proxy,
-			timeout: 10 * 1000,
-			forever: true,
-			headers: {
-				'User-Agent': config.userAgent
-			}
-		});
-
-		req.pipe(writable);
-
-		req.on('response', response => {
-			if (response.statusCode !== 200) {
-				logger.error(`Got ${response.statusCode} (${url})`);
-				writable.close();
-				rej(response.statusCode);
-			}
-		});
-
-		req.on('error', error => {
-			logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
-				url: url,
-				e: error
-			});
-			writable.close();
-			rej(error);
-		});
-
-		logger.succ(`Downloaded to: ${path}`);
+		response.body.pipe(writable);
 	});
 }

src/misc/fetch.ts

@@ -0,0 +1,43 @@
+import * as http from 'http';
+import * as https from 'https';
+import * as cache from 'lookup-dns-cache';
+import fetch, { HeadersInit } from 'node-fetch';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import config from '../config';
+
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
+	const res = await fetch(url, {
+		headers: Object.assign({
+			'User-Agent': config.userAgent,
+			Accept: accept
+		}, headers || {}),
+		timeout,
+		agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+	});
+
+	if (!res.ok) {
+		throw {
+			name: `StatusError`,
+			statusCode: res.status,
+			message: `${res.status} ${res.statusText}`,
+		};
+	}
+
+	return await res.json();
+}
+
+export const httpAgent = config.proxy
+	? new HttpProxyAgent(config.proxy)
+	: new http.Agent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+	});
+
+export const httpsAgent = config.proxy
+	? new HttpsProxyAgent(config.proxy)
+	: new https.Agent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+		lookup: cache.lookup,
+	});

src/remote/activitypub/request.ts

@@ -1,19 +1,12 @@
 import * as https from 'https';
 import { sign } from 'http-signature';
 import * as crypto from 'crypto';
-import * as cache from 'lookup-dns-cache';
 
 import config from '../../config';
 import { ILocalUser } from '../../models/entities/user';
 import { UserKeypairs } from '../../models';
 import { ensure } from '../../prelude/ensure';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-
-const agent = config.proxy
-	? new HttpsProxyAgent(config.proxy)
-	: new https.Agent({
-			lookup: cache.lookup,
-		});
+import { httpsAgent } from '../../misc/fetch';
 
 export default async (user: ILocalUser, url: string, object: any) => {
 	const timeout = 10 * 1000;
@@ -32,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
 
 	await new Promise((resolve, reject) => {
 		const req = https.request({
-			agent,
+			agent: httpsAgent,
 			protocol,
 			hostname,
 			port,

src/remote/activitypub/resolver.ts

@@ -1,10 +1,8 @@
-import * as request from 'request-promise-native';
+import { getJson } from '../../misc/fetch';
 import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type';
-import config from '../../config';
 
 export default class Resolver {
 	private history: Set<string>;
-	private timeout = 10 * 1000;
 
 	constructor() {
 		this.history = new Set();
@@ -41,24 +39,7 @@ export default class Resolver {
 
 		this.history.add(value);
 
-		const object = await request({
-			url: value,
-			proxy: config.proxy,
-			timeout: this.timeout,
-			forever: true,
-			headers: {
-				'User-Agent': config.userAgent,
-				Accept: 'application/activity+json, application/ld+json'
-			},
-			json: true
-		}).catch(e => {
-			const message = `${e.name}: ${e.message ? e.message.substr(0, 200) : undefined}, url=${value}`;
-			throw {
-				name: e.name,
-				statusCode: e.statusCode,
-				message,
-			};
-		});
+		const object = await getJson(value, 'application/activity+json, application/ld+json');
 
 		if (object == null || (
 			Array.isArray(object['@context']) ?

src/remote/webfinger.ts

@@ -1,5 +1,4 @@
-import config from '../config';
-import * as request from 'request-promise-native';
+import { getJson } from '../misc/fetch';
 import { query as urlQuery } from '../prelude/url';
 
 type ILink = {
@@ -15,17 +14,7 @@ type IWebFinger = {
 export default async function(query: string): Promise<IWebFinger> {
 	const url = genUrl(query);
 
-	return await request({
-		url,
-		proxy: config.proxy,
-		timeout: 10 * 1000,
-		forever: true,
-		headers: {
-			'User-Agent': config.userAgent,
-			Accept: 'application/jrd+json, application/json'
-		},
-		json: true
-	});
+	return await getJson(url, 'application/jrd+json, application/json');
 }
 
 function genUrl(query: string) {

src/server/api/service/discord.ts

@@ -1,6 +1,6 @@
 import * as Koa from 'koa';
 import * as Router from '@koa/router';
-import * as request from 'request';
+import { getJson } from '../../../misc/fetch';
 import { OAuth2 } from 'oauth';
 import config from '../../../config';
 import { publishMainStream } from '../../../services/stream';
@@ -174,20 +174,9 @@ router.get('/dc/cb', async ctx => {
 				}
 			}));
 
-		const { id, username, discriminator } = await new Promise<any>((res, rej) =>
-			request({
-				url: 'https://discordapp.com/api/users/@me',
-				headers: {
-					'Authorization': `Bearer ${accessToken}`,
-					'User-Agent': config.userAgent
-				}
-			}, (err, response, body) => {
-				if (err) {
-					rej(err);
-				} else {
-					res(JSON.parse(body));
-				}
-			}));
+		const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
+			'Authorization': `Bearer ${accessToken}`,
+		});
 
 		if (!id || !username || !discriminator) {
 			ctx.throw(400, 'invalid session');
@@ -256,21 +245,9 @@ router.get('/dc/cb', async ctx => {
 				}
 			}));
 
-		const { id, username, discriminator } = await new Promise<any>((res, rej) =>
-			request({
-				url: 'https://discordapp.com/api/users/@me',
-				headers: {
-					'Authorization': `Bearer ${accessToken}`,
-					'User-Agent': config.userAgent
-				}
-			}, (err, response, body) => {
-				if (err) {
-					rej(err);
-				} else {
-					res(JSON.parse(body));
-				}
-			}));
-
+		const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
+			'Authorization': `Bearer ${accessToken}`,
+		});
 		if (!id || !username || !discriminator) {
 			ctx.throw(400, 'invalid session');
 			return;

src/server/api/service/github.ts

@@ -1,6 +1,6 @@
 import * as Koa from 'koa';
 import * as Router from '@koa/router';
-import * as request from 'request';
+import { getJson } from '../../../misc/fetch';
 import { OAuth2 } from 'oauth';
 import config from '../../../config';
 import { publishMainStream } from '../../../services/stream';
@@ -167,21 +167,9 @@ router.get('/gh/cb', async ctx => {
 				}
 			}));
 
-		const { login, id } = await new Promise<any>((res, rej) =>
-			request({
-				url: 'https://api.github.com/user',
-				headers: {
-					'Accept': 'application/vnd.github.v3+json',
-					'Authorization': `bearer ${accessToken}`,
-					'User-Agent': config.userAgent
-				}
-			}, (err, response, body) => {
-				if (err)
-					rej(err);
-				else
-					res(JSON.parse(body));
-			}));
-
+		const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+			'Authorization': `bearer ${accessToken}`
+		});
 		if (!login || !id) {
 			ctx.throw(400, 'invalid session');
 			return;
@@ -230,20 +218,9 @@ router.get('/gh/cb', async ctx => {
 						res({ accessToken });
 				}));
 
-		const { login, id } = await new Promise<any>((res, rej) =>
-			request({
-				url: 'https://api.github.com/user',
-				headers: {
-					'Accept': 'application/vnd.github.v3+json',
-					'Authorization': `bearer ${accessToken}`,
-					'User-Agent': config.userAgent
-				}
-			}, (err, response, body) => {
-				if (err)
-					rej(err);
-				else
-					res(JSON.parse(body));
-			}));
+		const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+			'Authorization': `bearer ${accessToken}`
+		});
 
 		if (!login || !id) {
 			ctx.throw(400, 'invalid session');

src/server/web/url-preview.ts

@@ -1,10 +1,10 @@
 import * as Koa from 'koa';
-import * as request from 'request-promise-native';
 import summaly from 'summaly';
 import { fetchMeta } from '../../misc/fetch-meta';
 import Logger from '../../services/logger';
 import config from '../../config';
 import { query } from '../../prelude/url';
+import { getJson } from '../../misc/fetch';
 
 const logger = new Logger('url-preview');
 
@@ -16,15 +16,10 @@ module.exports = async (ctx: Koa.Context) => {
 		: `Getting preview of ${ctx.query.url}@${ctx.query.lang} ...`);
 
 	try {
-		const summary = meta.summalyProxy ? await request.get({
-			url: meta.summalyProxy,
-			qs: {
-				url: ctx.query.url,
-				lang: ctx.query.lang || 'ja-JP'
-			},
-			forever: true,
-			json: true
-		}) : await summaly(ctx.query.url, {
+		const summary = meta.summalyProxy ? await getJson(`${meta.summalyProxy}?${query({
+			url: ctx.query.url,
+			lang: ctx.query.lang || 'ja-JP'
+		})}`) : await summaly(ctx.query.url, {
 			followRedirects: false,
 			lang: ctx.query.lang || 'ja-JP'
 		});

src/services/drive/s3.ts

@@ -1,32 +1,17 @@
 import * as S3 from 'aws-sdk/clients/s3';
-import config from '../../config';
 import { Meta } from '../../models/entities/meta';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import * as agentkeepalive from 'agentkeepalive';
-
-const httpsAgent = config.proxy
-	? new HttpsProxyAgent(config.proxy)
-	: new agentkeepalive.HttpsAgent({
-			freeSocketTimeout: 30 * 1000
-		});
+import { httpsAgent, httpAgent } from '../../misc/fetch';
 
 export function getS3(meta: Meta) {
-	const conf = {
+	return new S3({
 		endpoint: meta.objectStorageEndpoint || undefined,
-		accessKeyId: meta.objectStorageAccessKey,
-		secretAccessKey: meta.objectStorageSecretKey,
+		accessKeyId: meta.objectStorageAccessKey!,
+		secretAccessKey: meta.objectStorageSecretKey!,
 		region: meta.objectStorageRegion || undefined,
 		sslEnabled: meta.objectStorageUseSSL,
 		s3ForcePathStyle: !!meta.objectStorageEndpoint,
 		httpOptions: {
+			agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
 		}
-	} as S3.ClientConfiguration;
-
-	if (meta.objectStorageUseSSL) {
-		conf.httpOptions!.agent = httpsAgent;
-	}
-
-	const s3 = new S3(conf);
-
-	return s3;
+	});
 }

src/services/fetch-nodeinfo.ts

@@ -1,7 +1,6 @@
-import * as request from 'request-promise-native';
+import { getJson } from '../misc/fetch';
 import { Instance } from '../models/entities/instance';
 import { Instances } from '../models';
-import config from '../config';
 import { getNodeinfoLock } from '../misc/app-lock';
 import Logger from '../services/logger';
 
@@ -20,23 +19,14 @@ export async function fetchNodeinfo(instance: Instance) {
 	logger.info(`Fetching nodeinfo of ${instance.host} ...`);
 
 	try {
-		const wellknown = await request({
-			url: 'https://' + instance.host + '/.well-known/nodeinfo',
-			proxy: config.proxy,
-			timeout: 1000 * 10,
-			forever: true,
-			headers: {
-				'User-Agent': config.userAgent,
-				Accept: 'application/json, */*'
-			},
-			json: true
-		}).catch(e => {
-			if (e.statusCode === 404) {
-				throw 'No nodeinfo provided';
-			} else {
-				throw e.statusCode || e.message;
-			}
-		});
+		const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
+			.catch(e => {
+				if (e.statusCode === 404) {
+					throw 'No nodeinfo provided';
+				} else {
+					throw e.statusCode || e.message;
+				}
+			});
 
 		if (wellknown.links == null || !Array.isArray(wellknown.links)) {
 			throw 'No wellknown links';
@@ -53,19 +43,10 @@ export async function fetchNodeinfo(instance: Instance) {
 			throw 'No nodeinfo link provided';
 		}
 
-		const info = await request({
-			url: link.href,
-			proxy: config.proxy,
-			timeout: 1000 * 10,
-			forever: true,
-			headers: {
-				'User-Agent': config.userAgent,
-				Accept: 'application/json, */*'
-			},
-			json: true
-		}).catch(e => {
-			throw e.statusCode || e.message;
-		});
+		const info = await getJson(link.href)
+			.catch(e => {
+				throw e.statusCode || e.message;
+			});
 
 		await Instances.update(instance.id, {
 			infoUpdatedAt: new Date(),

test/utils.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as WebSocket from 'ws';
-const fetch = require('node-fetch');
-import * as req from 'request';
+import fetch from 'node-fetch';
+const FormData = require('form-data');
 import * as childProcess from 'child_process';
 
 export const async = (fn: Function) => (done: Function) => {
@@ -20,6 +20,9 @@ export const request = async (endpoint: string, params: any, me?: any): Promise<
 	try {
 		const res = await fetch('http://localhost:8080/api' + endpoint, {
 			method: 'POST',
+			headers: {
+				'Content-Type': 'application/json'
+			},
 			body: JSON.stringify(Object.assign(auth, params))
 		});
 
@@ -64,18 +67,23 @@ export const react = async (user: any, note: any, reaction: string): Promise<any
 	}, user);
 };
 
-export const uploadFile = (user: any, path?: string): Promise<any> => new Promise((ok, rej) => {
-	req.post({
-		url: 'http://localhost:8080/api/drive/files/create',
-		formData: {
-			i: user.token,
-			file: fs.createReadStream(path || __dirname + '/resources/Lenna.png')
-		},
-		json: true
-	}, (err, httpResponse, body) => {
-		ok(body);
-	});
-});
+export const uploadFile = (user: any, path?: string): Promise<any> => {
+		const formData = new FormData();
+		formData.append('i', user.token);
+		formData.append('file', fs.createReadStream(path || __dirname + '/resources/Lenna.png'));
+
+		return fetch('http://localhost:8080/api/drive/files/create', {
+			method: 'post',
+			body: formData,
+			timeout: 30 * 1000,
+		}).then(res => {
+			if (!res.ok) {
+				throw `${res.status} ${res.statusText}`;
+			} else {
+				return res.json();
+			}
+		});
+};
 
 export function connectStream(user: any, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
 	return new Promise((res, rej) => {