FoundKeyGang/FoundKey
FoundKeyGang/FoundKey
7
19
Fork 22
Code Issues 37 Pull requests 4 Releases Activity

server: refactor fetchInstanceMetadata

Browse source 
The function only uses the host aspect of the instance and re-fetches
the rest from the database anyway, so it does not make sense to require
to pass the whole instance object as an argument everywhere.
This commit is contained in:
Johann150 2024-09-07 09:21:20 +02:00
parent 886e21e2dd
commit d5cf305a3a
Signed by: Johann150
GPG key ID: 9EE6577A2A06F8F1
6 changed files with 36 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/src/queue/processors/deliver.ts
View file

@ -47,7 +47,7 @@ export default async (job: Bull.Job<DeliverJobData>) => {
isNotResponding: false, isNotResponding: false,
}); });
fetchInstanceMetadata(i); fetchInstanceMetadata(host);
}); });
} catch (res) { } catch (res) {
// Update stats // Update stats

2
packages/backend/src/queue/processors/inbox.ts
View file

@ -69,7 +69,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
isNotResponding: false, isNotResponding: false,
}); });
fetchInstanceMetadata(i); fetchInstanceMetadata(i.host);
instanceChart.requestReceived(i.host); instanceChart.requestReceived(i.host);
apRequestChart.inbox(); apRequestChart.inbox();

2
packages/backend/src/queue/processors/system/update-instance.ts
View file

@ -2,7 +2,7 @@ import Bull from 'bull';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js'; import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
export async function updateInstance(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> { export async function updateInstance(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
await fetchInstanceMetadata({ host: job.data.host }, true); await fetchInstanceMetadata(job.data.host, true);
done(); done();
} }

2
packages/backend/src/remote/activitypub/models/person.ts
View file

@ -253,7 +253,7 @@ export async function createPerson(value: string | IObject, resolver: Resolver):
registerOrFetchInstanceDoc(host).then(i => { registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1); Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host); instanceChart.newUser(i.host);
fetchInstanceMetadata(i); fetchInstanceMetadata(i.host);
}); });
usersChart.update(user!, true); usersChart.update(user!, true);

12
packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts
View file

@ -1,6 +1,4 @@
import { Instances } from '@/models/index.js';
import { toPuny } from '@/misc/convert-host.js'; import { toPuny } from '@/misc/convert-host.js';
import { ApiError } from '@/server/api/error.js';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js'; import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
import define from '@/server/api/define.js'; import define from '@/server/api/define.js';
@ -9,8 +7,6 @@ export const meta = {
requireCredential: true, requireCredential: true,
requireModerator: true, requireModerator: true,
errors: ['NO_SUCH_OBJECT'],
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,11 +19,5 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export // eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
const instance = await Instances.findOneBy({ host: toPuny(ps.host) }); fetchInstanceMetadata(toPuny(ps.host), true);
if (instance == null) {
throw new ApiError('NO_SUCH_OBJECT');
}
fetchInstanceMetadata(instance, true);
}); });

61
packages/backend/src/services/fetch-instance-metadata.ts
View file

@ -4,7 +4,6 @@ import fetch from 'node-fetch';
import tinycolor from 'tinycolor2'; import tinycolor from 'tinycolor2';
import { DAY } from '@/const.js'; import { DAY } from '@/const.js';
import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch.js'; import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch.js';
import { Instance } from '@/models/entities/instance.js';
import { Instances } from '@/models/index.js'; import { Instances } from '@/models/index.js';
import { getFetchInstanceMetadataLock } from '@/misc/app-lock.js'; import { getFetchInstanceMetadataLock } from '@/misc/app-lock.js';
import { createUpdateInstanceJob } from '@/queue/index.js'; import { createUpdateInstanceJob } from '@/queue/index.js';
@ -12,16 +11,16 @@ import Logger from './logger.js';
const logger = new Logger('metadata'); const logger = new Logger('metadata');
export async function fetchInstanceMetadata(instance: Instance, force = false): Promise<void> { export async function fetchInstanceMetadata(instance_host: string, force = false): Promise<void> {
const unlock = await getFetchInstanceMetadataLock(instance.host); const unlock = await getFetchInstanceMetadataLock(instance_host);
const instance = await Instances.findOneBy({ host: instance_host });
if (!force) { if (!force) {
const _instance = await Instances.findOneBy({ host: instance.host }); if (instance && !instance.isNotResponding && instance.infoUpdatedAt) {
if (_instance && _instance.infoUpdatedAt) { const threshold = Date.now() - instance.infoUpdatedAt.getTime();
const threshold = Date.now() - _instance.infoUpdatedAt.getTime();
if (threshold > DAY) { if (threshold > DAY) {
// queue a job to deal with it later // queue a job to deal with it later
createUpdateInstanceJob(instance.host); createUpdateInstanceJob(instance_host);
} }
// if the instance is already known and force is not specified, // if the instance is already known and force is not specified,
// never update the data in the same thread // never update the data in the same thread
@ -30,24 +29,24 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
} }
} }
logger.info(`Fetching metadata of ${instance.host} ...`); logger.info(`Fetching metadata of ${instance_host} ...`);
try { try {
const [info, dom, manifest] = await Promise.all([ const [info, dom, manifest] = await Promise.all([
fetchNodeinfo(instance).catch(() => null), fetchNodeinfo(instance_host).catch(() => null),
fetchDom(instance).catch(() => null), fetchDom(instance_host).catch(() => null),
fetchManifest(instance).catch(() => null), fetchManifest(instance_host).catch(() => null),
]); ]);
const [favicon, icon, themeColor, name, description] = await Promise.all([ const [favicon, icon, themeColor, name, description] = await Promise.all([
fetchFaviconUrl(instance, dom).catch(() => null), fetchFaviconUrl(instance_host, dom).catch(() => null),
fetchIconUrl(instance, dom, manifest).catch(() => null), fetchIconUrl(instance_host, dom, manifest).catch(() => null),
getThemeColor(info, dom, manifest).catch(() => null), getThemeColor(info, dom, manifest).catch(() => null),
getSiteName(info, dom, manifest).catch(() => null), getSiteName(info, dom, manifest).catch(() => null),
getDescription(info, dom, manifest).catch(() => null), getDescription(info, dom, manifest).catch(() => null),
]); ]);
logger.succ(`Successfuly fetched metadata of ${instance.host}`); logger.succ(`Successfuly fetched metadata of ${instance_host}`);
const updates = { const updates = {
infoUpdatedAt: new Date(), infoUpdatedAt: new Date(),
@ -69,9 +68,9 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
await Instances.update(instance.id, updates); await Instances.update(instance.id, updates);
logger.succ(`Successfuly updated metadata of ${instance.host}`); logger.succ(`Successfuly updated metadata of ${instance_host}`);
} catch (e) { } catch (e) {
logger.error(`Failed to update metadata of ${instance.host}: ${e}`); logger.error(`Failed to update metadata of ${instance_host}: ${e}`);
} finally { } finally {
unlock(); unlock();
} }
@ -96,11 +95,11 @@ type NodeInfo = {
}; };
}; };
async function fetchNodeinfo(instance: Instance): Promise<NodeInfo> { async function fetchNodeinfo(instance_host: string): Promise<NodeInfo> {
logger.info(`Fetching nodeinfo of ${instance.host} ...`); logger.info(`Fetching nodeinfo of ${instance_host} ...`);
try { try {
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo') const wellknown = await getJson('https://' + instance_host + '/.well-known/nodeinfo')
.catch(e => { .catch(e => {
if (e.statusCode === 404) { if (e.statusCode === 404) {
throw new Error('No nodeinfo provided'); throw new Error('No nodeinfo provided');
@ -129,21 +128,21 @@ async function fetchNodeinfo(instance: Instance): Promise<NodeInfo> {
throw new Error(e.statusCode || e.message); throw new Error(e.statusCode || e.message);
}); });
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`); logger.succ(`Successfuly fetched nodeinfo of ${instance_host}`);
return info as NodeInfo; return info as NodeInfo;
} catch (e) { } catch (e) {
const message = e instanceof Error ? e.message : e; const message = e instanceof Error ? e.message : e;
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${message}`); logger.error(`Failed to fetch nodeinfo of ${instance_host}: ${message}`);
throw e; throw e;
} }
} }
async function fetchDom(instance: Instance): Promise<DOMWindow['document']> { async function fetchDom(instance_host: string): Promise<DOMWindow['document']> {
logger.info(`Fetching HTML of ${instance.host} ...`); logger.info(`Fetching HTML of ${instance_host} ...`);
const url = 'https://' + instance.host; const url = 'https://' + instance_host;
const html = await getHtml(url); const html = await getHtml(url);
@ -153,8 +152,8 @@ async function fetchDom(instance: Instance): Promise<DOMWindow['document']> {
return doc; return doc;
} }
async function fetchManifest(instance: Instance): Promise<Record<string, unknown> | null> { async function fetchManifest(instance_host: string): Promise<Record<string, unknown> | null> {
const url = 'https://' + instance.host; const url = 'https://' + instance_host;
const manifestUrl = url + '/manifest.json'; const manifestUrl = url + '/manifest.json';
@ -163,8 +162,8 @@ async function fetchManifest(instance: Instance): Promise<Record<string, unknown
return manifest; return manifest;
} }
async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] | null): Promise<string | null> { async function fetchFaviconUrl(instance_host: string, doc: DOMWindow['document'] | null): Promise<string | null> {
const url = 'https://' + instance.host; const url = 'https://' + instance_host;
if (doc) { if (doc) {
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
@ -190,14 +189,14 @@ async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] |
return null; return null;
} }
async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { async function fetchIconUrl(instance_host: string, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
const url = 'https://' + instance.host; const url = 'https://' + instance_host;
return (new URL(manifest.icons[0].src, url)).href; return (new URL(manifest.icons[0].src, url)).href;
} }
if (doc) { if (doc) {
const url = 'https://' + instance.host; const url = 'https://' + instance_host;
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const links = Array.from(doc.getElementsByTagName('link')).reverse(); const links = Array.from(doc.getElementsByTagName('link')).reverse();