FoundKey/packages/backend/src/misc/fetch.ts

import * as http from 'node:http';
import * as https from 'node:https';
import { URL } from 'node:url';
import * as dns from 'node:dns';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import ipaddr from 'ipaddr.js';
import IPCIDR from 'ip-cidr';
import { SECOND } from '@/const.js';
import config from '@/config/index.js';

export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
	const res = await getResponse({
		url,
		method: 'GET',
		headers: Object.assign({
			Accept: accept,
		}, headers),
		timeout,
	});

	return await res.json();
}

export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers: Record<string, string> = {}) {
	const res = await getResponse({
		url,
		method: 'GET',
		headers: Object.assign({
			Accept: accept,
		}, headers),
		timeout,
	});

	return await res.text();
}

export async function getResponse(_args: {
	url: string,
	method: string,
	body?: string,
	headers: Record<string, string>,
	timeout?: number,
	size?: number,
	redirect?: 'follow' | 'manual' | 'error',
}) {
	const args = {
		timeout: 10 * SECOND,
		size: 10 * 1024 * 1024, // 10 MiB
		redirect: 'follow',
		..._args,
	};

	const res = await fetch(args.url, {
		method: args.method,
		headers: Object.assign({
			'User-Agent': config.userAgent,
		}, args.headers),
		body: args.body,
		redirect: args.redirect,
		size: args.size,
		agent: getAgentByUrl,
		signal: AbortSignal.timeout(args.timeout),
	});

	if (
		!res.ok
		&&
		// intended redirect is not an error
		!(args.redirect != 'follow' && res.status >= 300 && res.status < 400)) {
		throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
	}

	return res;
}

const cache = new CacheableLookup({
	maxTtl: 3600,	// 1hours
	errorTtl: 30,	// 30secs
	lookup: false,	// nativeのdns.lookupにfallbackしない
});

// because `cacheable-lookup` is annoying and prefers IPv4 by default,
// we need a little wrapper setting some extra options
const cacheLookup = (host, _2, _3) => {
	// do all the weird parameter shenanigans that nodejs's dns.lookup does.
	let options = {}, callback;
	if (_3) {
		options = _2;
		if (typeof options === 'number') {
			options = { family: options };
		}
		callback = _3;
	} else {
		callback = _2;
	}

	// here come the shenanigans, trying to be careful not to mess up
	// intentionally different behaviour
	if (options.family == null && (options.hints ?? 0) === 0) {
		options.family = 6;
		options.hints = dns.V4MAPPED;
	}

	// callback wrapper that checks whether an IP is private.
	// Private IPs will not be returned in the first place.
	const wrapper = (err, address, family) => {
		// mimics dns.lookup behaviour as if no result was found
		const fakeErr = new Error("private IP");
		fakeErr.code = 'ENOTFOUND';

		if (err != null) {
			return callback(err, address, family);
		} else if (options.all) {
			const results = address.filter(({ address, family }) => isPublicIp(address));
			if (results.length === 0) {
				return callback(fakeErr);
			} else {
				return callback(err, results);
			}
		} else {
			if (isPublicIp(address)) {
				return callback(err, address, family);
			} else {
				return callback(fakeErr);
			}
		}
	};

	return cache.lookup(host, options, wrapper);
};

/**
 * Get http non-proxy agent
 */
const _http = new http.Agent({
	keepAlive: true,
	keepAliveMsecs: 30 * SECOND,
	lookup: cacheLookup,
} as http.AgentOptions);

/**
 * Get https non-proxy agent
 */
const _https = new https.Agent({
	keepAlive: true,
	keepAliveMsecs: 30 * SECOND,
	lookup: cacheLookup,
} as https.AgentOptions);

const maxSockets = Math.max(256, config.deliverJobConcurrency);

/**
 * Get http proxy or non-proxy agent
 */
export const httpAgent = config.proxy
	? new HttpProxyAgent({
		keepAlive: true,
		keepAliveMsecs: 30 * SECOND,
		maxSockets,
		maxFreeSockets: 256,
		scheduling: 'lifo',
		proxy: config.proxy,
	})
	: _http;

/**
 * Get https proxy or non-proxy agent
 */
export const httpsAgent = config.proxy
	? new HttpsProxyAgent({
		keepAlive: true,
		keepAliveMsecs: 30 * SECOND,
		maxSockets,
		maxFreeSockets: 256,
		scheduling: 'lifo',
		proxy: config.proxy,
	})
	: _https;

/**
 * Get agent by URL
 * @param url URL
 * @param bypassProxy Allways bypass proxy
 */
export function getAgentByUrl(url: URL, bypassProxy = false) {
	if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
		return url.protocol === 'http:' ? _http : _https;
	} else {
		return url.protocol === 'http:' ? httpAgent : httpsAgent;
	}
}

export class StatusError extends Error {
	public statusCode: number;
	public statusMessage?: string;
	public isClientError: boolean;

	constructor(message: string, statusCode: number, statusMessage?: string) {
		super(message);
		this.name = 'StatusError';
		this.statusCode = statusCode;
		this.statusMessage = statusMessage;
		this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
	}
}

function isPublicIp(ip: string): boolean {
	for (const net of config.allowedPrivateNetworks || []) {
		const cidr = new IPCIDR(net);
		if (cidr.contains(ip)) {
			return true;
		}
	}

	// Workaround because ipaddr.js in the installed version cannot cope
	// with v4mapped addresses. a fix has been submitted and merged, but a new
	// version was not yet released.
	// FIXME: update ipaddr.js and remove workaround
	ip = ip.replace(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/i, '$1');

	try {
		let range = ipaddr.process(ip).range();
		// only unicast or multicast addresses are allowed by default
		// this does not include e.g. loopback addresses
		return ['unicast', 'multicast'].includes(range);
	} catch {
		return false;
	}
}