From 9a6bb8be7de3acee4815339cdeb12d1a925505a5 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Sat, 4 Feb 2023 17:56:15 +0100 Subject: [PATCH] server: default config items on load --- .config/example.yml | 50 +++++++++++++++-------- packages/backend/src/config/load.ts | 36 ++++++++-------- packages/backend/src/misc/download-url.ts | 9 ++-- packages/backend/src/misc/fetch.ts | 2 +- packages/backend/src/queue/index.ts | 8 ++-- packages/backend/src/queue/queues.ts | 4 +- 6 files changed, 65 insertions(+), 44 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 2924114fb..0cf399a64 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -6,10 +6,11 @@ #───┘ URL └───────────────────────────────────────────────────── # Final accessible URL seen by a user. -url: https://example.tld/ - +# Only the host part will be used. # ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE # URL SETTINGS AFTER THAT! +url: https://example.tld/ + # ┌───────────────────────┐ #───┘ Port and TLS settings └─────────────────────────────────── @@ -45,6 +46,7 @@ db: pass: example-foundkey-pass # Whether to disable query caching + # Default is to cache, i.e. false. #disableCache: true # Extra connection options @@ -57,7 +59,11 @@ db: redis: host: localhost port: 6379 - #family: dual # can be either a number or string (0/dual, 4/ipv4, 6/ipv6) + # Address family to connect over. + # Can be either a number or string (0/dual, 4/ipv4, 6/ipv6) + # Default is "dual". + #family: dual + # The following properties are optional. #pass: example-pass #prefix: example-prefix #db: 1 @@ -65,6 +71,7 @@ redis: # ┌─────────────────────────────┐ #───┘ Elasticsearch configuration └───────────────────────────── +# Elasticsearch is optional. #elasticsearch: # host: localhost # port: 9200 @@ -75,35 +82,41 @@ redis: # ┌─────────────────────┐ #───┘ Other configuration └───────────────────────────────────── -# Whether disable HSTS +# Whether to disable HSTS (not recommended) +# Default is to enable HSTS, i.e. false. #disableHsts: true # Number of worker processes by type. -# The sum must not exceed the number of available cores. +# The sum should not exceed the number of available cores. #clusterLimits: # web: 1 # queue: 1 -# Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +# Jobs each worker will try to work on at a time. +#deliverJobConcurrency: 128 +#inboxJobConcurrency: 16 -# Job rate limiter -# deliverJobPerSec: 128 -# inboxJobPerSec: 16 +# Rate limit for each Worker. +# Use -1 to disable. +# A rate limit for deliver jobs is not recommended as it comes with +# a big performance penalty due to overhead of rate limiting. +#deliverJobPerSec: 128 +#inboxJobPerSec: 16 -# Job attempts -# deliverJobMaxAttempts: 12 -# inboxJobMaxAttempts: 8 +# Number of times each job will be tried. +# 1 means only try once and don't retry. +#deliverJobMaxAttempts: 12 +#inboxJobMaxAttempts: 8 # Syslog option #syslog: # host: localhost # port: 514 -# Proxy for HTTP/HTTPS +# Proxy for HTTP/HTTPS outgoing connections #proxy: http://127.0.0.1:3128 +# Hosts that should not be connected to through the proxy specified above #proxyBypassHosts: [ # 'example.com', # '192.0.2.8' @@ -117,7 +130,8 @@ redis: # Media Proxy #mediaProxy: https://example.com/proxy -# Proxy remote files (default: false) +# Proxy remote files +# Default is to not proxy remote files, i.e. false. #proxyRemoteFiles: true # Storage path for files if stored locally (absolute path) @@ -125,11 +139,15 @@ redis: #internalStoragePath: '/etc/foundkey/files' # Upload or download file size limits (bytes) +# default is 262144000 = 250MiB #maxFileSize: 262144000 # Max note text length (in characters) #maxNoteTextLength: 3000 +# By default, Foundkey will fail when something tries to make it fetch something from private IPs. +# With the following setting you can explicitly allow some private CIDR subnets. +# Default is an empty list, i.e. none allowed. #allowedPrivateNetworks: [ # '127.0.0.1/32' #] diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index 95286585d..9adf5677b 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -38,13 +38,30 @@ export default function load(): Config { config.port = config.port || parseInt(process.env.PORT || '', 10); + // set default values config.images = Object.assign({ info: '/twemoji/1f440.svg', notFound: '/twemoji/2049.svg', error: '/twemoji/1f480.svg', }, config.images ?? {}); - if (!config.maxNoteTextLength) config.maxNoteTextLength = 3000; + config.clusterLimits = Object.assign({ + web: 1, + queue: 1, + }, config.clusterLimits ?? {}); + + config = Object.assign({ + disableHsts: false, + deliverJobConcurrency: 128, + inboxJobConcurrency: 16, + deliverJobPerSec: 128, + inboxJobPerSec: 16, + deliverJobMaxAttempts: 12, + inboxJobMaxAttempts: 8, + proxyRemoteFiles: false, + maxFileSize: 262144000, // 250 MiB + maxNoteTextLength: 3000, + }, config); mixin.version = meta.version; mixin.host = url.host; @@ -60,21 +77,8 @@ export default function load(): Config { if (!config.redis.prefix) config.redis.prefix = mixin.host; - if (!config.clusterLimits) { - config.clusterLimits = { - web: 1, - queue: 1, - }; - } else { - config.clusterLimits = { - web: 1, - queue: 1, - ...config.clusterLimits, - }; - - if (config.clusterLimits.web < 1 || config.clusterLimits.queue < 1) { - throw new Error('invalid cluster limits'); - } + if (config.clusterLimits.web < 1 || config.clusterLimits.queue < 1) { + throw new Error('invalid cluster limits'); } return Object.assign(config, mixin); diff --git a/packages/backend/src/misc/download-url.ts b/packages/backend/src/misc/download-url.ts index 079e84e8a..694a80457 100644 --- a/packages/backend/src/misc/download-url.ts +++ b/packages/backend/src/misc/download-url.ts @@ -19,7 +19,6 @@ export async function downloadUrl(url: string, path: string): Promise { const timeout = 30 * SECOND; const operationTimeout = MINUTE; - const maxSize = config.maxFileSize || 262144000; const req = got.stream(url, { headers: { @@ -53,14 +52,14 @@ export async function downloadUrl(url: string, path: string): Promise { const contentLength = res.headers['content-length']; if (contentLength != null) { const size = Number(contentLength); - if (size > maxSize) { - logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); + if (size > config.maxFileSize) { + logger.warn(`maxSize exceeded (${size} > ${config.maxFileSize}) on response`); req.destroy(); } } }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > maxSize) { - logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); + if (progress.transferred > config.maxFileSize) { + logger.warn(`maxSize exceeded (${progress.transferred} > ${config.maxFileSize}) on downloadProgress`); req.destroy(); } }); diff --git a/packages/backend/src/misc/fetch.ts b/packages/backend/src/misc/fetch.ts index 42eb445d9..6bd794e68 100644 --- a/packages/backend/src/misc/fetch.ts +++ b/packages/backend/src/misc/fetch.ts @@ -89,7 +89,7 @@ const _https = new https.Agent({ lookup: cache.lookup, } as https.AgentOptions); -const maxSockets = Math.max(256, config.deliverJobConcurrency || 128); +const maxSockets = Math.max(256, config.deliverJobConcurrency); /** * Get http proxy or non-proxy agent diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index 57de62a79..cc125a3de 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -96,7 +96,7 @@ export function deliver(user: ThinUser, content: unknown, to: string | null) { }; return deliverQueue.add(data, { - attempts: config.deliverJobMaxAttempts || 12, + attempts: config.deliverJobMaxAttempts, timeout: MINUTE, backoff: { type: 'apBackoff', @@ -113,7 +113,7 @@ export function inbox(activity: IActivity, signature: httpSignature.IParsedSigna }; return inboxQueue.add(data, { - attempts: config.inboxJobMaxAttempts || 8, + attempts: config.inboxJobMaxAttempts, timeout: 5 * MINUTE, backoff: { type: 'apBackoff', @@ -291,8 +291,8 @@ export function webhookDeliver(webhook: Webhook, type: typeof webhookEventTypes[ export default function() { if (envOption.onlyServer) return; - deliverQueue.process(config.deliverJobConcurrency || 128, processDeliver); - inboxQueue.process(config.inboxJobConcurrency || 16, processInbox); + deliverQueue.process(config.deliverJobConcurrency, processDeliver); + inboxQueue.process(config.inboxJobConcurrency, processInbox); endedPollNotificationQueue.process(endedPollNotification); webhookDeliverQueue.process(64, processWebhookDeliver); processDb(dbQueue); diff --git a/packages/backend/src/queue/queues.ts b/packages/backend/src/queue/queues.ts index f3a267790..0f0f2f69d 100644 --- a/packages/backend/src/queue/queues.ts +++ b/packages/backend/src/queue/queues.ts @@ -4,8 +4,8 @@ import { DeliverJobData, InboxJobData, DbJobData, ObjectStorageJobData, EndedPol export const systemQueue = initializeQueue>('system'); export const endedPollNotificationQueue = initializeQueue('endedPollNotification'); -export const deliverQueue = initializeQueue('deliver', config.deliverJobPerSec || 128); -export const inboxQueue = initializeQueue('inbox', config.inboxJobPerSec || 16); +export const deliverQueue = initializeQueue('deliver', config.deliverJobPerSec); +export const inboxQueue = initializeQueue('inbox', config.inboxJobPerSec); export const dbQueue = initializeQueue('db'); export const objectStorageQueue = initializeQueue('objectStorage'); export const webhookDeliverQueue = initializeQueue('webhookDeliver', 64);