server: better performace for mass delivery

This should hopefully relieve some of the massive hammering on the
database when mass delivery jobs are running.

However, this also means that instance blocks are applied with a
slight delay on delivery queue. Since the settings page in the
native frontend already warns about this, I think it should be fine.
And the maximum time an instance block would be delayed would be
somewhere around 5min which IMO is also tolerable.

Changelog: Changed

packages/backend/src/queue/initialize.ts

@@ -23,6 +23,19 @@ export function initialize<T>(name: string, limitPerSec = -1): Bull.Queue<T> {
 function apBackoff(attemptsMade: number /*, err: Error */): number {
 	const baseDelay = MINUTE;
 	const maxBackoff = 8 * HOUR;
+	/*
+	attempt | average seconds + up to 2% random offset
+	0  |     0
+	1  |    60 = 1min
+	2  |   180 = 3min
+	3  |   420 = 7min
+	4  |   900 = 15min
+	5  |  1860 = 31min
+	6  |  3780 = 63min
+	7  |  7620 = 127min ~= 2.1h
+	8  | 15300 = 4.25h
+	>8 | 28800 = 8h
+	*/
 	let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
 	backoff = Math.min(backoff, maxBackoff);
 	backoff += Math.round(backoff * Math.random() * 0.2);

packages/backend/src/queue/processors/deliver.ts

@@ -17,7 +17,18 @@ export default async (job: Bull.Job<DeliverJobData>) => {
 	const { host } = new URL(job.data.to);
 	const puny = toPuny(host);
 
-	if (await shouldSkipInstance(puny)) return 'skip';
+	// for the first few tries (where most attempts will be made)
+	// we assume that inserting deliver jobs took care of this check
+	// only on later attempts do we actually do it, to ease database
+	// performance. this might cause a slight delay of a few minutes
+	// for instance blocks being applied
+	//
+	// with apBackoff, attempt 2 happens ~4min after the initial try, while
+	// attempt 3 happens ~11 min after the initial try, which seems like a
+	// good tradeoff between database and blocks being applied reasonably quick
+	if (job.attemptsMade >= 3 && await shouldSkipInstance(puny)) {
+		return 'skip';
+	}
 
 	const keypair = await getUserKeypair(job.data.user.id);