From 91a4f38871e4f39fc30e943fa913dd9ab43cc600 Mon Sep 17 00:00:00 2001 From: Chloe Kudryavtsev Date: Sun, 16 Oct 2022 08:08:40 -0400 Subject: [PATCH 1/3] backend: add automatic dead instance detection It works by having a day-long cache of "when did we last successfully communicate with this instance?" Anything over a specified threshold (1 month) will act as though the instance is suspended - all outgoing jobs are dropped on processing. The day-long cache is in place because the ordering is necessarily a linear scan. Once an instance comes back online, we will detect that is the case as soon as we receive an activity from them (which will update the "last communicated at") field. Potential future TODOs: * Improve the caching system, it's actually pretty inefficient as it is. CacheBox with a call override? * Think of ways to make it not-a-linear-scan, since the instances table can get pretty big. It's around 4500 on toast cafe. ChangeLog: Added --- .../backend/src/queue/processors/deliver.ts | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index 108379aa8..a518c4116 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -12,36 +12,52 @@ import { Cache } from '@/misc/cache.js'; import { Instance } from '@/models/entities/instance.js'; import { StatusError } from '@/misc/fetch.js'; import { DeliverJobData } from '@/queue/types.js'; +import { LessThan } from 'typeorm'; const logger = new Logger('deliver'); let latest: string | null = null; const suspendedHostsCache = new Cache(1000 * 60 * 60); +// dead host list is a linear scan, so cache it longer +const deadHostsCache = new Cache(1000 * 60 * 60 * 24); +const deadThreshold = 1000 * 60 * 60 * 24 * 30; // 1 month export default async (job: Bull.Job) => { const { host } = new URL(job.data.to); + const puny = toPuny(host); // ブロックしてたら中断 const meta = await fetchMeta(); - if (meta.blockedHosts.includes(toPuny(host))) { + if (meta.blockedHosts.includes(puny)) { return 'skip (blocked)'; } - // isSuspendedなら中断 + // isSuspended let suspendedHosts = suspendedHostsCache.get(null); if (suspendedHosts == null) { - suspendedHosts = await Instances.find({ - where: { - isSuspended: true, - }, + suspendedHosts = await Instances.findBy({ + isSuspended: true, }); suspendedHostsCache.set(null, suspendedHosts); } - if (suspendedHosts.map(x => x.host).includes(toPuny(host))) { + if (suspendedHosts.some(x => x.host == puny)) { return 'skip (suspended)'; } + // dead + let deadHosts = deadHostsCache.get(null); + if (deadHosts == null) { + const deadTime = new Date(Date.now() - deadThreshold); + deadHosts = await Instances.findBy({ + lastCommunicatedAt: LessThan(deadTime), + }); + deadHostsCache.set(null, deadHosts); + } + if (deadHosts.some(x => x.host == puny)) { + return 'skip (dead instance)'; + } + try { if (latest !== (latest = JSON.stringify(job.data.content, null, 2))) { logger.debug(`delivering ${latest}`); -- 2.43.0 From 21c1e5c06c9216d9b3acc28cd9f06a21c24755bf Mon Sep 17 00:00:00 2001 From: Chloe Kudryavtsev Date: Sun, 16 Oct 2022 09:22:05 -0400 Subject: [PATCH 2/3] backend: simplify suspended and dead queries This should also have better latency due to being a single query. Furthermore, it's no longer a linear scan, since host is indexed. Would be cool to simplify it further to a single query for blocks also... Why exactly are blocks not in the db? --- .../backend/src/queue/processors/deliver.ts | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index a518c4116..67cb2fe60 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -13,15 +13,13 @@ import { Instance } from '@/models/entities/instance.js'; import { StatusError } from '@/misc/fetch.js'; import { DeliverJobData } from '@/queue/types.js'; import { LessThan } from 'typeorm'; +import { DAY } from '@/const.ts'; const logger = new Logger('deliver'); let latest: string | null = null; -const suspendedHostsCache = new Cache(1000 * 60 * 60); -// dead host list is a linear scan, so cache it longer -const deadHostsCache = new Cache(1000 * 60 * 60 * 24); -const deadThreshold = 1000 * 60 * 60 * 24 * 30; // 1 month +const deadThreshold = 30 * DAY; export default async (job: Bull.Job) => { const { host } = new URL(job.data.to); @@ -33,29 +31,12 @@ export default async (job: Bull.Job) => { return 'skip (blocked)'; } - // isSuspended - let suspendedHosts = suspendedHostsCache.get(null); - if (suspendedHosts == null) { - suspendedHosts = await Instances.findBy({ - isSuspended: true, - }); - suspendedHostsCache.set(null, suspendedHosts); - } - if (suspendedHosts.some(x => x.host == puny)) { - return 'skip (suspended)'; - } - - // dead - let deadHosts = deadHostsCache.get(null); - if (deadHosts == null) { - const deadTime = new Date(Date.now() - deadThreshold); - deadHosts = await Instances.findBy({ - lastCommunicatedAt: LessThan(deadTime), - }); - deadHostsCache.set(null, deadHosts); - } - if (deadHosts.some(x => x.host == puny)) { - return 'skip (dead instance)'; + const isSuspendedOrDead = await Instances.countBy([ + { host: puny, isSuspended: true }, + { host: puny, lastCommunicatedAt: LessThan(deadTime) }, + ]); + if (isSuspendedOrDead) { + return 'skip (suspended or dead)'; } try { -- 2.43.0 From d762143b892399340d23a3697ee20aa317985084 Mon Sep 17 00:00:00 2001 From: Chloe Kudryavtsev Date: Sun, 16 Oct 2022 09:32:01 -0400 Subject: [PATCH 3/3] backend: fixup missing deadTime and incorrect import --- packages/backend/src/queue/processors/deliver.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index 67cb2fe60..1f99b305d 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -13,7 +13,7 @@ import { Instance } from '@/models/entities/instance.js'; import { StatusError } from '@/misc/fetch.js'; import { DeliverJobData } from '@/queue/types.js'; import { LessThan } from 'typeorm'; -import { DAY } from '@/const.ts'; +import { DAY } from '@/const.js'; const logger = new Logger('deliver'); @@ -31,6 +31,7 @@ export default async (job: Bull.Job) => { return 'skip (blocked)'; } + const deadTime = new Date(Date.now() - deadThreshold); const isSuspendedOrDead = await Instances.countBy([ { host: puny, isSuspended: true }, { host: puny, lastCommunicatedAt: LessThan(deadTime) }, -- 2.43.0