From 075e251822aa0812b15a111d89b5cc0354ce75d7 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 1 Dec 2022 01:42:09 -0500 Subject: [PATCH 1/4] server: add wildcard matching to blocked hosts This adds in wildcard matching. For instance: - `*.bad.tld` will match: `very.bad.tld` - `bad.*` will match: `bad.something` - `*.bad.*` will match: `very.bad.evil` Changelog: Changed --- locales/en-US.yml | 2 +- .../backend/src/misc/skipped-instances.ts | 45 ++++++++++++++----- .../backend/src/queue/processors/inbox.ts | 11 +++-- .../src/remote/activitypub/resolver.ts | 4 +- .../src/server/api/endpoints/ap/show.ts | 11 ++--- 5 files changed, 47 insertions(+), 26 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index 1392fc897..3ccdc7586 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -187,7 +187,7 @@ clearCachedFiles: "Clear cache" clearCachedFilesConfirm: "Are you sure that you want to delete all cached remote files?" blockedInstances: "Blocked Instances" blockedInstancesDescription: "List the hostnames of the instances that you want to\ - \ block. Listed instances will no longer be able to communicate with this instance." + \ block. Listed instances will no longer be able to communicate with this instance. You can use an asterisk (*) as a placeholder for zero or more character(s)." muteAndBlock: "Mutes and Blocks" mutedUsers: "Muted users" blockedUsers: "Blocked users" diff --git a/packages/backend/src/misc/skipped-instances.ts b/packages/backend/src/misc/skipped-instances.ts index 3f13a8f7a..081f883cb 100644 --- a/packages/backend/src/misc/skipped-instances.ts +++ b/packages/backend/src/misc/skipped-instances.ts @@ -1,7 +1,5 @@ -import { Brackets } from 'typeorm'; import { db } from '@/db/postgre.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; -import { Instances } from '@/models/index.js'; import { Instance } from '@/models/entities/instance.js'; import { DAY } from '@/const.js'; @@ -9,19 +7,42 @@ import { DAY } from '@/const.js'; // "dead" and should no longer get activities delivered to it. const deadThreshold = 7 * DAY; +/** + * Returns whether a given host matches a wildcard pattern. + * @param host punycoded instance host + * @param pattern wildcard pattern containing a punycoded instance host + * @returns whether the post matches the pattern + */ +function matchHost(host: Instance['host'], pattern: string): boolean { + // Escape all of the regex special characters. Pattern from: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + const escape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + + const re = new RegExp('^' + pattern.split('*').map(escape).join('.*') + '$'); + return re.test(host); +} + +/** + * Returns whether a specific host (punycoded) should be blocked. + * + * @param host punycoded instance host + * @returns whether the given host should be blocked + */ +export async function shouldBlockInstance(host: string): Promise { + const { blockedHosts } = await fetchMeta(); + return blockedHosts.some(blockedHost => matchHost(host, blockedHost)); +} + /** * Returns the subset of hosts which should be skipped. * * @param hosts array of punycoded instance hosts - * @returns array of punycoed instance hosts that should be skipped (subset of hosts parameter) + * @returns array of punycoded instance hosts that should be skipped (subset of hosts parameter) */ -export async function skippedInstances(hosts: Array): Array { - // first check for blocked instances since that info may already be in memory - const { blockedHosts } = await fetchMeta(); - - const skipped = hosts.filter(host => blockedHosts.includes(host)); +export async function skippedInstances(hosts: Array): Promise> { + const skipped = hosts.filter(host => shouldBlockInstance(host)); // if possible return early and skip accessing the database - if (skipped.length === hosts.length) return hosts; + if (skipped.length === hosts.length) return hosts; const deadTime = new Date(Date.now() - deadThreshold); @@ -32,10 +53,10 @@ export async function skippedInstances(hosts: Array): Array !skipped.includes(host) && !host.includes(',')).join(','), + hosts.filter(host => !skipped.some(blockedHost => matchHost(host, blockedHost)) && !host.includes(',')).join(','), ], ) - .then(res => res.map(row => row.host)) + .then(res => res.map(row => row.host)), ); } @@ -47,7 +68,7 @@ export async function skippedInstances(hosts: Array): Array { const skipped = await skippedInstances([host]); return skipped.length > 0; } diff --git a/packages/backend/src/queue/processors/inbox.ts b/packages/backend/src/queue/processors/inbox.ts index 426713358..2334cc9f8 100644 --- a/packages/backend/src/queue/processors/inbox.ts +++ b/packages/backend/src/queue/processors/inbox.ts @@ -6,7 +6,6 @@ import Logger from '@/services/logger.js'; import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js'; import { Instances } from '@/models/index.js'; import { apRequestChart, federationChart, instanceChart } from '@/services/chart/index.js'; -import { fetchMeta } from '@/misc/fetch-meta.js'; import { toPuny, extractDbHost } from '@/misc/convert-host.js'; import { getApId } from '@/remote/activitypub/type.js'; import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js'; @@ -17,6 +16,7 @@ import { StatusError } from '@/misc/fetch.js'; import { CacheableRemoteUser } from '@/models/entities/user.js'; import { UserPublickey } from '@/models/entities/user-publickey.js'; import { InboxJobData } from '@/queue/types.js'; +import { shouldBlockInstance } from '@/misc/skipped-instances.js'; const logger = new Logger('inbox'); @@ -33,9 +33,8 @@ export default async (job: Bull.Job): Promise => { const host = toPuny(new URL(signature.keyId).hostname); - // ブロックしてたら中断 - const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host)) { + // Stop if the host is blocked. + if (await shouldBlockInstance(host)) { return `Blocked request: ${host}`; } @@ -117,9 +116,9 @@ export default async (job: Bull.Job): Promise => { return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`; } - // ブロックしてたら中断 + // Stop if the host is blocked. const ldHost = extractDbHost(authUser.user.uri); - if (meta.blockedHosts.includes(ldHost)) { + if (await shouldBlockInstance(ldHost)) { return `Blocked request: ${ldHost}`; } } else { diff --git a/packages/backend/src/remote/activitypub/resolver.ts b/packages/backend/src/remote/activitypub/resolver.ts index 3cea4c44e..44b01b3eb 100644 --- a/packages/backend/src/remote/activitypub/resolver.ts +++ b/packages/backend/src/remote/activitypub/resolver.ts @@ -12,6 +12,7 @@ import renderQuestion from '@/remote/activitypub/renderer/question.js'; import renderCreate from '@/remote/activitypub/renderer/create.js'; import { renderActivity } from '@/remote/activitypub/renderer/index.js'; import renderFollow from '@/remote/activitypub/renderer/follow.js'; +import { shouldBlockInstance } from '@/misc/skipped-instances.js'; import { signedGet } from './request.js'; import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type.js'; import { parseUri } from './db-resolver.js'; @@ -67,8 +68,7 @@ export default class Resolver { return await this.resolveLocal(value); } - const meta = await fetchMeta(); - if (meta.blockedHosts.includes(host)) { + if (await shouldBlockInstance(host)) { throw new Error('Instance is blocked'); } diff --git a/packages/backend/src/server/api/endpoints/ap/show.ts b/packages/backend/src/server/api/endpoints/ap/show.ts index a1b18cab1..ef66bac3f 100644 --- a/packages/backend/src/server/api/endpoints/ap/show.ts +++ b/packages/backend/src/server/api/endpoints/ap/show.ts @@ -1,4 +1,3 @@ -import config from '@/config/index.js'; import { createPerson } from '@/remote/activitypub/models/person.js'; import { createNote } from '@/remote/activitypub/models/note.js'; import DbResolver from '@/remote/activitypub/db-resolver.js'; @@ -7,10 +6,10 @@ import { extractDbHost } from '@/misc/convert-host.js'; import { Users, Notes } from '@/models/index.js'; import { Note } from '@/models/entities/note.js'; import { CacheableLocalUser, User } from '@/models/entities/user.js'; -import { fetchMeta } from '@/misc/fetch-meta.js'; import { isActor, isPost, getApId } from '@/remote/activitypub/type.js'; import { SchemaType } from '@/misc/schema.js'; import { HOUR } from '@/const.js'; +import { shouldBlockInstance } from '@/misc/skipped-instances.js'; import define from '../../define.js'; import { ApiError } from '../../error.js'; @@ -85,9 +84,11 @@ export default define(meta, paramDef, async (ps, me) => { * URIからUserかNoteを解決する */ async function fetchAny(uri: string, me: CacheableLocalUser | null | undefined): Promise | null> { - // ブロックしてたら中断 - const fetchedMeta = await fetchMeta(); - if (fetchedMeta.blockedHosts.includes(extractDbHost(uri))) return null; + // Stop if the host is blocked. + const host = extractDbHost(uri); + if (await shouldBlockInstance(host)) { + return null; + } const dbResolver = new DbResolver(); From a35c98bbd5fba3a997047468f7bf708169e4b638 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 1 Dec 2022 11:34:11 -0500 Subject: [PATCH 2/4] server: encode non-ascii domains in punycode in matchHost --- packages/backend/src/misc/skipped-instances.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/misc/skipped-instances.ts b/packages/backend/src/misc/skipped-instances.ts index 081f883cb..ebedb50d3 100644 --- a/packages/backend/src/misc/skipped-instances.ts +++ b/packages/backend/src/misc/skipped-instances.ts @@ -1,3 +1,4 @@ +import { toASCII } from 'punycode/'; import { db } from '@/db/postgre.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { Instance } from '@/models/entities/instance.js'; @@ -9,17 +10,19 @@ const deadThreshold = 7 * DAY; /** * Returns whether a given host matches a wildcard pattern. - * @param host punycoded instance host - * @param pattern wildcard pattern containing a punycoded instance host + * @param host instance host + * @param pattern wildcard pattern containing an instance host * @returns whether the post matches the pattern */ function matchHost(host: Instance['host'], pattern: string): boolean { // Escape all of the regex special characters. Pattern from: // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping const escape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - - const re = new RegExp('^' + pattern.split('*').map(escape).join('.*') + '$'); - return re.test(host); + const re = new RegExp('^' + pattern.split('*').map(toASCII).map(escape).join('.*') + '$'); + + // Encode the domain in punycode in case it uses non-ascii + const punycoded = toASCII(host); + return re.test(punycoded); } /** @@ -53,7 +56,7 @@ export async function skippedInstances(hosts: Array): Promise< deadTime.toISOString(), // don't check hosts again that we already know are suspended // also avoids adding duplicates to the list - hosts.filter(host => !skipped.some(blockedHost => matchHost(host, blockedHost)) && !host.includes(',')).join(','), + hosts.filter(host => !skipped.includes(host) && !host.includes(',')).join(','), ], ) .then(res => res.map(row => row.host)), From b3e34795c0964faca8296d297251b3af5b681950 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 1 Dec 2022 12:07:43 -0500 Subject: [PATCH 3/4] require punycode conversion beforehand for admins --- locales/en-US.yml | 2 +- packages/backend/src/misc/skipped-instances.ts | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index 3ccdc7586..a6a8a2110 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -187,7 +187,7 @@ clearCachedFiles: "Clear cache" clearCachedFilesConfirm: "Are you sure that you want to delete all cached remote files?" blockedInstances: "Blocked Instances" blockedInstancesDescription: "List the hostnames of the instances that you want to\ - \ block. Listed instances will no longer be able to communicate with this instance. You can use an asterisk (*) as a placeholder for zero or more character(s)." + \ block. Listed instances will no longer be able to communicate with this instance. Non-ASCII domain names must be encoded in punycode. You can use an asterisk (*) as a placeholder for zero or more character(s)." muteAndBlock: "Mutes and Blocks" mutedUsers: "Muted users" blockedUsers: "Blocked users" diff --git a/packages/backend/src/misc/skipped-instances.ts b/packages/backend/src/misc/skipped-instances.ts index ebedb50d3..de42e14d2 100644 --- a/packages/backend/src/misc/skipped-instances.ts +++ b/packages/backend/src/misc/skipped-instances.ts @@ -1,4 +1,3 @@ -import { toASCII } from 'punycode/'; import { db } from '@/db/postgre.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { Instance } from '@/models/entities/instance.js'; @@ -10,19 +9,17 @@ const deadThreshold = 7 * DAY; /** * Returns whether a given host matches a wildcard pattern. - * @param host instance host - * @param pattern wildcard pattern containing an instance host + * @param host punycoded instance host + * @param pattern wildcard pattern containing a punycoded instance host * @returns whether the post matches the pattern */ function matchHost(host: Instance['host'], pattern: string): boolean { // Escape all of the regex special characters. Pattern from: // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping const escape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const re = new RegExp('^' + pattern.split('*').map(toASCII).map(escape).join('.*') + '$'); + const re = new RegExp('^' + pattern.split('*').map(escape).join('.*') + '$'); - // Encode the domain in punycode in case it uses non-ascii - const punycoded = toASCII(host); - return re.test(punycoded); + return re.test(host); } /** From 721a327192864248cda1516489ec4613da6347a8 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Thu, 1 Dec 2022 20:46:46 +0100 Subject: [PATCH 4/4] fixup: remove unused import --- packages/backend/src/remote/activitypub/resolver.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/remote/activitypub/resolver.ts b/packages/backend/src/remote/activitypub/resolver.ts index 44b01b3eb..800d6d1ff 100644 --- a/packages/backend/src/remote/activitypub/resolver.ts +++ b/packages/backend/src/remote/activitypub/resolver.ts @@ -2,7 +2,6 @@ import config from '@/config/index.js'; import { getJson } from '@/misc/fetch.js'; import { ILocalUser } from '@/models/entities/user.js'; import { getInstanceActor } from '@/services/instance-actor.js'; -import { fetchMeta } from '@/misc/fetch-meta.js'; import { extractDbHost, isSelfHost } from '@/misc/convert-host.js'; import { Notes, NoteReactions, Polls, Users } from '@/models/index.js'; import renderNote from '@/remote/activitypub/renderer/note.js';