2022-04-16 08:19:30 +00:00
|
|
|
import * as http from 'node:http';
|
|
|
|
import * as https from 'node:https';
|
|
|
|
import { URL } from 'node:url';
|
2021-08-16 08:44:43 +00:00
|
|
|
import CacheableLookup from 'cacheable-lookup';
|
2021-10-16 08:16:24 +00:00
|
|
|
import fetch from 'node-fetch';
|
2021-08-16 08:44:43 +00:00
|
|
|
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
2022-11-20 22:15:22 +00:00
|
|
|
import { SECOND } from '@/const.js';
|
2022-02-27 02:07:39 +00:00
|
|
|
import config from '@/config/index.js';
|
2020-04-09 14:42:23 +00:00
|
|
|
|
2023-07-16 15:02:22 +00:00
|
|
|
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
|
2021-10-16 08:16:24 +00:00
|
|
|
const res = await getResponse({
|
|
|
|
url,
|
|
|
|
method: 'GET',
|
2020-04-09 14:42:23 +00:00
|
|
|
headers: Object.assign({
|
2021-12-09 14:58:30 +00:00
|
|
|
Accept: accept,
|
2023-07-16 15:02:22 +00:00
|
|
|
}, headers),
|
2021-12-09 14:58:30 +00:00
|
|
|
timeout,
|
2020-04-09 14:42:23 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
return await res.json();
|
|
|
|
}
|
|
|
|
|
2023-07-16 15:02:22 +00:00
|
|
|
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
|
2021-10-16 08:16:24 +00:00
|
|
|
const res = await getResponse({
|
|
|
|
url,
|
|
|
|
method: 'GET',
|
2020-07-26 02:04:07 +00:00
|
|
|
headers: Object.assign({
|
2021-12-09 14:58:30 +00:00
|
|
|
Accept: accept,
|
2023-07-16 15:02:22 +00:00
|
|
|
}, headers),
|
2021-12-09 14:58:30 +00:00
|
|
|
timeout,
|
2021-10-16 08:16:24 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
return await res.text();
|
|
|
|
}
|
|
|
|
|
2023-07-16 15:02:22 +00:00
|
|
|
export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout: number = 10 * SECOND, size?: number, redirect: 'follow' | 'manual' | 'error' = 'follow' }) {
|
2021-10-16 08:16:24 +00:00
|
|
|
const controller = new AbortController();
|
|
|
|
setTimeout(() => {
|
|
|
|
controller.abort();
|
2023-07-16 15:02:22 +00:00
|
|
|
}, timeout);
|
2021-10-16 08:16:24 +00:00
|
|
|
|
|
|
|
const res = await fetch(args.url, {
|
|
|
|
method: args.method,
|
2023-07-16 15:02:22 +00:00
|
|
|
headers: Object.assign({
|
|
|
|
'User-Agent': config.userAgent,
|
2023-07-16 15:43:15 +00:00
|
|
|
}, args.headers),
|
2021-10-16 08:16:24 +00:00
|
|
|
body: args.body,
|
2022-12-20 21:06:54 +00:00
|
|
|
redirect: args.redirect,
|
|
|
|
size: args.size || 10 * 1024 * 1024, // 10 MiB
|
2020-07-26 02:04:07 +00:00
|
|
|
agent: getAgentByUrl,
|
2021-10-16 08:16:24 +00:00
|
|
|
signal: controller.signal,
|
2020-07-26 02:04:07 +00:00
|
|
|
});
|
|
|
|
|
2023-01-05 19:03:38 +00:00
|
|
|
if (
|
|
|
|
!res.ok
|
|
|
|
&&
|
|
|
|
// intended redirect is not an error
|
|
|
|
!(args.redirect != 'follow' && res.status >= 300 && res.status < 400)) {
|
2021-10-16 08:16:24 +00:00
|
|
|
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
|
2020-07-26 02:04:07 +00:00
|
|
|
}
|
|
|
|
|
2021-10-16 08:16:24 +00:00
|
|
|
return res;
|
2020-07-26 02:04:07 +00:00
|
|
|
}
|
|
|
|
|
2021-08-16 08:44:43 +00:00
|
|
|
const cache = new CacheableLookup({
|
|
|
|
maxTtl: 3600, // 1hours
|
|
|
|
errorTtl: 30, // 30secs
|
|
|
|
lookup: false, // nativeのdns.lookupにfallbackしない
|
|
|
|
});
|
|
|
|
|
2020-04-12 11:32:34 +00:00
|
|
|
/**
|
|
|
|
* Get http non-proxy agent
|
|
|
|
*/
|
|
|
|
const _http = new http.Agent({
|
|
|
|
keepAlive: true,
|
2022-11-20 22:15:22 +00:00
|
|
|
keepAliveMsecs: 30 * SECOND,
|
2021-05-19 07:11:47 +00:00
|
|
|
lookup: cache.lookup,
|
|
|
|
} as http.AgentOptions);
|
2020-04-12 11:32:34 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Get https non-proxy agent
|
|
|
|
*/
|
|
|
|
const _https = new https.Agent({
|
|
|
|
keepAlive: true,
|
2022-11-20 22:15:22 +00:00
|
|
|
keepAliveMsecs: 30 * SECOND,
|
2020-04-12 11:32:34 +00:00
|
|
|
lookup: cache.lookup,
|
2021-08-16 08:44:43 +00:00
|
|
|
} as https.AgentOptions);
|
|
|
|
|
2023-02-04 16:56:15 +00:00
|
|
|
const maxSockets = Math.max(256, config.deliverJobConcurrency);
|
2020-04-12 11:32:34 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Get http proxy or non-proxy agent
|
|
|
|
*/
|
2020-04-09 14:42:23 +00:00
|
|
|
export const httpAgent = config.proxy
|
2021-08-16 08:44:43 +00:00
|
|
|
? new HttpProxyAgent({
|
|
|
|
keepAlive: true,
|
2022-11-20 22:15:22 +00:00
|
|
|
keepAliveMsecs: 30 * SECOND,
|
2021-08-16 08:44:43 +00:00
|
|
|
maxSockets,
|
|
|
|
maxFreeSockets: 256,
|
|
|
|
scheduling: 'lifo',
|
2021-12-09 14:58:30 +00:00
|
|
|
proxy: config.proxy,
|
2021-08-16 08:44:43 +00:00
|
|
|
})
|
2020-04-12 11:32:34 +00:00
|
|
|
: _http;
|
2020-04-09 14:42:23 +00:00
|
|
|
|
2020-04-12 11:32:34 +00:00
|
|
|
/**
|
|
|
|
* Get https proxy or non-proxy agent
|
|
|
|
*/
|
2020-04-09 14:42:23 +00:00
|
|
|
export const httpsAgent = config.proxy
|
2021-08-16 08:44:43 +00:00
|
|
|
? new HttpsProxyAgent({
|
|
|
|
keepAlive: true,
|
2022-11-20 22:15:22 +00:00
|
|
|
keepAliveMsecs: 30 * SECOND,
|
2021-08-16 08:44:43 +00:00
|
|
|
maxSockets,
|
|
|
|
maxFreeSockets: 256,
|
|
|
|
scheduling: 'lifo',
|
2021-12-09 14:58:30 +00:00
|
|
|
proxy: config.proxy,
|
2021-08-16 08:44:43 +00:00
|
|
|
})
|
2020-04-12 11:32:34 +00:00
|
|
|
: _https;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get agent by URL
|
|
|
|
* @param url URL
|
|
|
|
* @param bypassProxy Allways bypass proxy
|
|
|
|
*/
|
|
|
|
export function getAgentByUrl(url: URL, bypassProxy = false) {
|
|
|
|
if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
|
2022-04-03 06:33:22 +00:00
|
|
|
return url.protocol === 'http:' ? _http : _https;
|
2020-04-12 11:32:34 +00:00
|
|
|
} else {
|
2022-04-03 06:33:22 +00:00
|
|
|
return url.protocol === 'http:' ? httpAgent : httpsAgent;
|
2020-04-12 11:32:34 +00:00
|
|
|
}
|
|
|
|
}
|
2021-10-16 08:16:24 +00:00
|
|
|
|
|
|
|
export class StatusError extends Error {
|
|
|
|
public statusCode: number;
|
|
|
|
public statusMessage?: string;
|
|
|
|
public isClientError: boolean;
|
|
|
|
|
|
|
|
constructor(message: string, statusCode: number, statusMessage?: string) {
|
|
|
|
super(message);
|
|
|
|
this.name = 'StatusError';
|
|
|
|
this.statusCode = statusCode;
|
|
|
|
this.statusMessage = statusMessage;
|
|
|
|
this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
|
|
|
|
}
|
|
|
|
}
|