diff --git a/packages/backend/src/queue/initialize.ts b/packages/backend/src/queue/initialize.ts index d866ef61b..569bcc8b6 100644 --- a/packages/backend/src/queue/initialize.ts +++ b/packages/backend/src/queue/initialize.ts @@ -23,6 +23,19 @@ export function initialize(name: string, limitPerSec = -1): Bull.Queue { function apBackoff(attemptsMade: number /*, err: Error */): number { const baseDelay = MINUTE; const maxBackoff = 8 * HOUR; + /* + attempt | average seconds + up to 2% random offset + 0 | 0 + 1 | 60 = 1min + 2 | 180 = 3min + 3 | 420 = 7min + 4 | 900 = 15min + 5 | 1860 = 31min + 6 | 3780 = 63min + 7 | 7620 = 127min ~= 2.1h + 8 | 15300 = 4.25h + >8 | 28800 = 8h + */ let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay; backoff = Math.min(backoff, maxBackoff); backoff += Math.round(backoff * Math.random() * 0.2); diff --git a/packages/backend/src/queue/processors/deliver.ts b/packages/backend/src/queue/processors/deliver.ts index 22a770c07..910ee8613 100644 --- a/packages/backend/src/queue/processors/deliver.ts +++ b/packages/backend/src/queue/processors/deliver.ts @@ -17,7 +17,18 @@ export default async (job: Bull.Job) => { const { host } = new URL(job.data.to); const puny = toPuny(host); - if (await shouldSkipInstance(puny)) return 'skip'; + // for the first few tries (where most attempts will be made) + // we assume that inserting deliver jobs took care of this check + // only on later attempts do we actually do it, to ease database + // performance. this might cause a slight delay of a few minutes + // for instance blocks being applied + // + // with apBackoff, attempt 2 happens ~4min after the initial try, while + // attempt 3 happens ~11 min after the initial try, which seems like a + // good tradeoff between database and blocks being applied reasonably quick + if (job.attemptsMade >= 3 && await shouldSkipInstance(puny)) { + return 'skip'; + } const keypair = await getUserKeypair(job.data.user.id);