server: use time constants

This commit is contained in:
Johann150 2022-11-20 23:15:22 +01:00
parent 66a7c62342
commit 9e2553909e
Signed by untrusted user: Johann150
GPG Key ID: 9EE6577A2A06F8F1
24 changed files with 78 additions and 58 deletions

View File

@ -1,9 +1,10 @@
import Xev from 'xev';
import { deliverQueue, inboxQueue } from '@/queue/queues.js';
import { SECOND } from '@/const.js';
const ev = new Xev();
const interval = 10000;
const interval = 10 * SECOND;
/**
* Report queue stats regularly

View File

@ -1,5 +1,7 @@
// https://github.com/typeorm/typeorm/issues/2400
import pg from 'pg';
import { SECOND } from '@/const.js';
pg.types.setTypeParser(20, Number);
import { Logger, DataSource } from 'typeorm';
@ -182,7 +184,7 @@ export const db = new DataSource({
password: config.db.pass,
database: config.db.db,
extra: {
statement_timeout: 1000 * 10,
statement_timeout: 10 * SECOND,
...config.db.extra,
},
synchronize: process.env.NODE_ENV === 'test',
@ -234,7 +236,7 @@ export async function resetDb() {
if (i === 3) {
throw e;
} else {
await new Promise(resolve => setTimeout(resolve, 1000));
await new Promise(resolve => setTimeout(resolve, SECOND));
continue;
}
}

View File

@ -1,6 +1,7 @@
import { promisify } from 'node:util';
import redisLock from 'redis-lock';
import { redisClient } from '@/db/redis.js';
import { SECOND } from '@/const.js';
/**
* Retry delay (ms) for lock acquisition
@ -18,14 +19,14 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
* @param timeout Lock timeout (ms), The timeout releases previous lock.
* @returns Unlock function
*/
export function getApLock(uri: string, timeout = 30 * 1000) {
export function getApLock(uri: string, timeout = 30 * SECOND) {
return lock(`ap-object:${uri}`, timeout);
}
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * SECOND) {
return lock(`instance:${host}`, timeout);
}
export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {
export function getChartInsertLock(lockKey: string, timeout = 30 * SECOND) {
return lock(`chart-insert:${lockKey}`, timeout);
}

View File

@ -5,6 +5,7 @@ import chalk from 'chalk';
import got, * as Got from 'got';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
import { SECOND, MINUTE } from '@/const.js';
import config from '@/config/index.js';
import Logger from '@/services/logger.js';
import { httpAgent, httpsAgent, StatusError } from './fetch.js';
@ -16,8 +17,8 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const timeout = 30 * SECOND;
const operationTimeout = MINUTE;
const maxSize = config.maxFileSize || 262144000;
const req = got.stream(url, {

View File

@ -4,9 +4,10 @@ import { URL } from 'node:url';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { SECOND } from '@/const.js';
import config from '@/config/index.js';
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) {
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
const res = await getResponse({
url,
method: 'GET',
@ -20,7 +21,7 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
return await res.json();
}
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) {
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10 * SECOND, headers?: Record<string, string>) {
const res = await getResponse({
url,
method: 'GET',
@ -35,7 +36,7 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout =
}
export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
const timeout = args.timeout || 10 * 1000;
const timeout = args.timeout || 10 * SECOND;
const controller = new AbortController();
setTimeout(() => {
@ -70,7 +71,7 @@ const cache = new CacheableLookup({
*/
const _http = new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
keepAliveMsecs: 30 * SECOND,
lookup: cache.lookup,
} as http.AgentOptions);
@ -79,7 +80,7 @@ const _http = new http.Agent({
*/
const _https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
keepAliveMsecs: 30 * SECOND,
lookup: cache.lookup,
} as https.AgentOptions);
@ -91,7 +92,7 @@ const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
export const httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
keepAliveMsecs: 30 * SECOND,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
@ -105,7 +106,7 @@ export const httpAgent = config.proxy
export const httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
keepAliveMsecs: 30 * SECOND,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',

View File

@ -6,6 +6,7 @@ import { DriveFile } from '@/models/entities/drive-file.js';
import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js';
import { IActivity } from '@/remote/activitypub/type.js';
import { envOption } from '@/env.js';
import { MINUTE } from '@/const.js';
import processDeliver from './processors/deliver.js';
import processInbox from './processors/inbox.js';
@ -96,7 +97,7 @@ export function deliver(user: ThinUser, content: unknown, to: string | null) {
return deliverQueue.add(data, {
attempts: config.deliverJobMaxAttempts || 12,
timeout: 1 * 60 * 1000, // 1min
timeout: MINUTE,
backoff: {
type: 'apBackoff',
},
@ -113,7 +114,7 @@ export function inbox(activity: IActivity, signature: httpSignature.IParsedSigna
return inboxQueue.add(data, {
attempts: config.inboxJobMaxAttempts || 8,
timeout: 5 * 60 * 1000, // 5min
timeout: 5 * MINUTE,
backoff: {
type: 'apBackoff',
},
@ -278,7 +279,7 @@ export function webhookDeliver(webhook: Webhook, type: typeof webhookEventTypes[
return webhookDeliverQueue.add(data, {
attempts: 4,
timeout: 1 * 60 * 1000, // 1min
timeout: MINUTE,
backoff: {
type: 'apBackoff',
},

View File

@ -1,4 +1,5 @@
import Bull from 'bull';
import { SECOND, MINUTE, HOUR } from '@/const.js';
import config from '@/config/index.js';
import { getRedisOptions } from '@/config/redis.js';
@ -8,7 +9,7 @@ export function initialize<T>(name: string, limitPerSec = -1): Bull.Queue<T> {
prefix: config.redis.prefix ? `${config.redis.prefix}:queue` : 'queue',
limiter: limitPerSec > 0 ? {
max: limitPerSec,
duration: 1000,
duration: SECOND,
} : undefined,
settings: {
backoffStrategies: {
@ -20,8 +21,8 @@ export function initialize<T>(name: string, limitPerSec = -1): Bull.Queue<T> {
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
function apBackoff(attemptsMade: number, err: Error) {
const baseDelay = 60 * 1000; // 1min
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
const baseDelay = MINUTE;
const maxBackoff = 8 * HOUR;
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
backoff = Math.min(backoff, maxBackoff);
backoff += Math.round(backoff * Math.random() * 0.2);

View File

@ -3,6 +3,7 @@ import Bull from 'bull';
import { format as dateFormat } from 'date-fns';
import { In, MoreThan, Not } from 'typeorm';
import { MONTH } from '@/const.js';
import { getFullApAccount } from '@/misc/convert-host.js';
import { createTemp } from '@/misc/create-temp.js';
import { Following } from '@/models/entities/following.js';
@ -61,7 +62,7 @@ export async function exportFollowing(job: Bull.Job<DbUserJobData>, done: () =>
continue;
}
if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) {
if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 3 * MONTH)) {
continue;
}

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { CacheableRemoteUser } from '@/models/entities/user.js';
import { IObject } from './type.js';
import { performActivity } from './kernel/index.js';
@ -6,9 +7,9 @@ import { updatePerson } from './models/person.js';
export default async (actor: CacheableRemoteUser, activity: IObject): Promise<void> => {
await performActivity(actor, activity);
// ついでにリモートユーザーの情報が古かったら更新しておく
// And while I'm at it, I'll update the remote user information if it's out of date.
if (actor.uri) {
if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > DAY) {
setImmediate(() => {
updatePerson(actor.uri!);
});

View File

@ -1,6 +1,7 @@
import { URL } from 'node:url';
import chalk from 'chalk';
import { IsNull } from 'typeorm';
import { DAY } from '@/const.js';
import config from '@/config/index.js';
import { isSelfHost, toPuny } from '@/misc/convert-host.js';
import { User, IRemoteUser } from '@/models/entities/user.js';
@ -49,9 +50,9 @@ export async function resolveUser(username: string, idnHost: string | null): Pro
return await createPerson(self.href);
}
// ユーザー情報が古い場合は、WebFilgerからやりなおして返す
if (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
// 繋がらないインスタンスに何回も試行するのを防ぐ, 後続の同様処理の連続試行を防ぐ ため 試行前にも更新する
// If user information is out of date, start over with webfinger
if (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > DAY) {
// Prevent race conditions
await Users.update(user.id, {
lastFetchedAt: new Date(),
});

View File

@ -1,4 +1,5 @@
import rndstr from 'rndstr';
import { DAY } from '@/const.js';
import { Note } from '@/models/entities/note.js';
import { User } from '@/models/entities/user.js';
import { Notes, UserProfiles, NoteReactions } from '@/models/index.js';
@ -16,13 +17,13 @@ export async function injectFeatured(timeline: Note[], user?: User | null) {
}
const max = 30;
const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで
const offset = 3 * DAY;
const query = Notes.createQueryBuilder('note')
.addSelect('note.score')
.where('note.userHost IS NULL')
.andWhere('note.score > 0')
.andWhere('note.createdAt > :date', { date: new Date(Date.now() - day) })
.andWhere('note.createdAt > :date', { date: new Date(Date.now() - offset) })
.andWhere("note.visibility = 'public'")
.innerJoinAndSelect('note.user', 'user');

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { Users } from '@/models/index.js';
import define from '../../define.js';
@ -46,7 +47,7 @@ export default define(meta, paramDef, async (ps, me) => {
case 'admin': query.where('user.isAdmin = TRUE'); break;
case 'moderator': query.where('user.isModerator = TRUE'); break;
case 'adminOrModerator': query.where('user.isAdmin = TRUE OR user.isModerator = TRUE'); break;
case 'alive': query.where('user.updatedAt > :date', { date: new Date(Date.now() - 1000 * 60 * 60 * 24 * 5) }); break;
case 'alive': query.where('user.updatedAt > :date', { date: new Date(Date.now() - 5 * DAY) }); break;
case 'silenced': query.where('user.isSilenced = TRUE'); break;
case 'suspended': query.where('user.isSuspended = TRUE'); break;
}

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { GalleryPosts } from '@/models/index.js';
import define from '../../define.js';
@ -26,7 +27,7 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const query = GalleryPosts.createQueryBuilder('post')
.andWhere('post.createdAt > :date', { date: new Date(Date.now() - (1000 * 60 * 60 * 24 * 3)) })
.andWhere('post.createdAt > :date', { date: new Date(Date.now() - 3 * DAY) })
.andWhere('post.likedCount > 0')
.orderBy('post.likedCount', 'DESC');

View File

@ -1,4 +1,5 @@
import { Brackets } from 'typeorm';
import { MINUTE, HOUR } from '@/const.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { Notes } from '@/models/index.js';
import { Note } from '@/models/entities/note.js';
@ -13,8 +14,8 @@ import define from '../../define.js';
..PostgreSQLでどうするのか分からないので単にAの内に投稿されたユニーク投稿数が多いハッシュタグ
*/
const rangeA = 1000 * 60 * 60; // 60分
//const rangeB = 1000 * 60 * 120; // 2時間
const rangeA = HOUR;
//const rangeB = 2 * HOUR;
//const coefficient = 1.25; // 「n倍」の部分
//const requiredUsers = 3; // 最低何人がそのタグを投稿している必要があるか
@ -116,8 +117,7 @@ export default define(meta, paramDef, async () => {
const range = 20;
// 10分
const interval = 1000 * 60 * 10;
const interval = 10 * MINUTE;
for (let i = 0; i < range; i++) {
countPromises.push(Promise.all(hots.map(tag => Notes.createQueryBuilder('note')

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { Users } from '@/models/index.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import define from '../../define.js';
@ -35,7 +36,7 @@ export default define(meta, paramDef, async (ps, me) => {
const query = Users.createQueryBuilder('user')
.where(':tag = ANY(user.tags)', { tag: normalizeForSearch(ps.tag) });
const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5));
const recent = new Date(Date.now() - 5 * DAY);
if (ps.state === 'alive') {
query.andWhere('user.updatedAt > :date', { date: recent });

View File

@ -1,6 +1,7 @@
import { promisify } from 'node:util';
import bcrypt from 'bcryptjs';
import * as cbor from 'cbor';
import { MINUTE } from '@/const.js';
import {
UserProfiles,
UserSecurityKeys,
@ -112,10 +113,10 @@ export default define(meta, paramDef, async (ps, user) => {
id: ps.challengeId,
});
// Expired challenge (> 5min old)
// Expired challenge
if (
new Date().getTime() - attestationChallenge.createdAt.getTime() >=
5 * 60 * 1000
5 * MINUTE
) {
throw new Error('expired challenge');
}

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { Notes } from '@/models/index.js';
import define from '../../define.js';
import { generateMutedUserQuery } from '../../common/generate-muted-user-query.js';
@ -31,7 +32,7 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, user) => {
const max = 30;
const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで
const day = 3 * DAY;
const query = Notes.createQueryBuilder('note')
.addSelect('note.score')

View File

@ -1,3 +1,4 @@
import { DAY } from '@/const.js';
import { Users } from '@/models/index.js';
import define from '../define.js';
import { generateMutedUserQueryForUsers } from '../common/generate-muted-user-query.js';
@ -46,7 +47,7 @@ export default define(meta, paramDef, async (ps, me) => {
case 'admin': query.andWhere('user.isAdmin = TRUE'); break;
case 'moderator': query.andWhere('user.isModerator = TRUE'); break;
case 'adminOrModerator': query.andWhere('user.isAdmin = TRUE OR user.isModerator = TRUE'); break;
case 'alive': query.andWhere('user.updatedAt > :date', { date: new Date(Date.now() - 1000 * 60 * 60 * 24 * 5) }); break;
case 'alive': query.andWhere('user.updatedAt > :date', { date: new Date(Date.now() - 5 * DAY) }); break;
}
switch (ps.origin) {

View File

@ -1,3 +1,4 @@
import { MONTH } from '@/const.js';
import { Brackets } from 'typeorm';
import { Followings, Users } from '@/models/index.js';
import { User } from '@/models/entities/user.js';
@ -39,7 +40,7 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const activeThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)); // 30日
const activeThreshold = new Date(Date.now() - MONTH);
if (ps.host) {
const q = Users.createQueryBuilder('user')

View File

@ -1,4 +1,5 @@
import { Brackets } from 'typeorm';
import { MONTH } from '@/const.js';
import { UserProfiles, Users } from '@/models/index.js';
import { User } from '@/models/entities/user.js';
import define from '../../define.js';
@ -35,7 +36,7 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const activeThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)); // 30日
const activeThreshold = new Date(Date.now() - MONTH);
const isUsername = ps.query.startsWith('@');

View File

@ -3,6 +3,7 @@ import { IsNull } from 'typeorm';
import Koa from 'koa';
import bcrypt from 'bcryptjs';
import * as speakeasy from 'speakeasy';
import { SECOND, MINUTE, HOUR } from '@/const.js';
import config from '@/config/index.js';
import { Users, Signins, UserProfiles, UserSecurityKeys, AttestationChallenges } from '@/models/index.js';
import { ILocalUser } from '@/models/entities/user.js';
@ -38,7 +39,7 @@ export default async (ctx: Koa.Context) => {
try {
// not more than 1 attempt per second and not more than 10 attempts per hour
await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(ctx.ip));
await limiter({ key: 'signin', duration: HOUR, max: 10, minInterval: SECOND }, getIpHash(ctx.ip));
} catch (err) {
error(new ApiError('RATE_LIMIT_EXCEEDED'));
return;
@ -149,7 +150,7 @@ export default async (ctx: Koa.Context) => {
id: body.challengeId,
});
if (new Date().getTime() - challenge.createdAt.getTime() >= 5 * 60 * 1000) {
if (new Date().getTime() - challenge.createdAt.getTime() >= 5 * MINUTE) {
await fail();
return;
}

View File

@ -1,13 +1,10 @@
import { WEEK, MONTH, YEAR } from '@/const.js';
import { User } from '@/models/entities/user.js';
import Chart, { KVs } from '../core.js';
import { name, schema } from './entities/active-users.js';
const week = 1000 * 60 * 60 * 24 * 7;
const month = 1000 * 60 * 60 * 24 * 30;
const year = 1000 * 60 * 60 * 24 * 365;
/**
*
* Chart on Active Users
*/
// eslint-disable-next-line import/no-default-export
export default class ActiveUsersChart extends Chart<typeof schema> {
@ -26,12 +23,12 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
public async read(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
await this.commit({
'read': [user.id],
'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < week) ? [user.id] : [],
'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < month) ? [user.id] : [],
'registeredWithinYear': (Date.now() - user.createdAt.getTime() < year) ? [user.id] : [],
'registeredOutsideWeek': (Date.now() - user.createdAt.getTime() > week) ? [user.id] : [],
'registeredOutsideMonth': (Date.now() - user.createdAt.getTime() > month) ? [user.id] : [],
'registeredOutsideYear': (Date.now() - user.createdAt.getTime() > year) ? [user.id] : [],
'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < WEEK) ? [user.id] : [],
'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < MONTH) ? [user.id] : [],
'registeredWithinYear': (Date.now() - user.createdAt.getTime() < YEAR) ? [user.id] : [],
'registeredOutsideWeek': (Date.now() - user.createdAt.getTime() > WEEK) ? [user.id] : [],
'registeredOutsideMonth': (Date.now() - user.createdAt.getTime() > MONTH) ? [user.id] : [],
'registeredOutsideYear': (Date.now() - user.createdAt.getTime() > YEAR) ? [user.id] : [],
});
}

View File

@ -1,3 +1,4 @@
import { MONTH } from '@/const.js';
import { Followings, Instances } from '@/models/index.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import Chart, { KVs } from '../core.js';
@ -65,7 +66,7 @@ export default class FederationChart extends Chart<typeof schema> {
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere('instance.isSuspended = false')
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - MONTH) })
.getRawOne()
.then(x => parseInt(x.count, 10)),
Instances.createQueryBuilder('instance')
@ -73,7 +74,7 @@ export default class FederationChart extends Chart<typeof schema> {
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts })
.andWhere('instance.isSuspended = false')
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) })
.andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - MONTH) })
.getRawOne()
.then(x => parseInt(x.count, 10)),
]);

View File

@ -2,6 +2,7 @@ import { URL } from 'node:url';
import { DOMWindow, JSDOM } from 'jsdom';
import fetch from 'node-fetch';
import tinycolor from 'tinycolor2';
import { DAY } from '@/const.js';
import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch.js';
import { Instance } from '@/models/entities/instance.js';
import { Instances } from '@/models/index.js';
@ -16,7 +17,7 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
if (!force) {
const _instance = await Instances.findOneBy({ host: instance.host });
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < DAY)) {
unlock();
return;
}