FoundKeyGang/FoundKey
FoundKeyGang/FoundKey
7
19
Fork 22
Code Issues 37 Pull requests 4 Releases Activity

server: refactor fetchInstanceMetadata

Browse source 
The function only uses the host aspect of the instance and re-fetches
the rest from the database anyway, so it does not make sense to require
to pass the whole instance object as an argument everywhere.
This commit is contained in:
Johann150 2024-09-07 09:21:20 +02:00
parent 886e21e2dd
commit d5cf305a3a
Signed by: Johann150
GPG key ID: 9EE6577A2A06F8F1
6 changed files with 36 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/src/queue/processors/deliver.ts
View file

@ -47,7 +47,7 @@ export default async (job: Bull.Job<DeliverJobData>) => {
isNotResponding: false,
});
fetchInstanceMetadata(i);
fetchInstanceMetadata(host);
});
} catch (res) {
// Update stats

2
packages/backend/src/queue/processors/inbox.ts
View file

@ -69,7 +69,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
isNotResponding: false,
});
fetchInstanceMetadata(i);
fetchInstanceMetadata(i.host);
instanceChart.requestReceived(i.host);
apRequestChart.inbox();

2
packages/backend/src/queue/processors/system/update-instance.ts
View file

@ -2,7 +2,7 @@ import Bull from 'bull';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
export async function updateInstance(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
await fetchInstanceMetadata({ host: job.data.host }, true);
await fetchInstanceMetadata(job.data.host, true);
done();
}

2
packages/backend/src/remote/activitypub/models/person.ts
View file

@ -253,7 +253,7 @@ export async function createPerson(value: string | IObject, resolver: Resolver):
registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host);
fetchInstanceMetadata(i);
fetchInstanceMetadata(i.host);
});
usersChart.update(user!, true);

12
packages/backend/src/server/api/endpoints/admin/federation/refresh-remote-instance-metadata.ts
View file

@ -1,6 +1,4 @@
import { Instances } from '@/models/index.js';
import { toPuny } from '@/misc/convert-host.js';
import { ApiError } from '@/server/api/error.js';
import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
import define from '@/server/api/define.js';
@ -9,8 +7,6 @@ export const meta = {
requireCredential: true,
requireModerator: true,
errors: ['NO_SUCH_OBJECT'],
} as const;
export const paramDef = {
@ -23,11 +19,5 @@ export const paramDef = {
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps) => {
const instance = await Instances.findOneBy({ host: toPuny(ps.host) });
if (instance == null) {
throw new ApiError('NO_SUCH_OBJECT');
}
fetchInstanceMetadata(instance, true);
fetchInstanceMetadata(toPuny(ps.host), true);
});

63
packages/backend/src/services/fetch-instance-metadata.ts
View file

@ -4,7 +4,6 @@ import fetch from 'node-fetch';
import tinycolor from 'tinycolor2';
import { DAY } from '@/const.js';
import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch.js';
import { Instance } from '@/models/entities/instance.js';
import { Instances } from '@/models/index.js';
import { getFetchInstanceMetadataLock } from '@/misc/app-lock.js';
import { createUpdateInstanceJob } from '@/queue/index.js';
@ -12,16 +11,16 @@ import Logger from './logger.js';
const logger = new Logger('metadata');
export async function fetchInstanceMetadata(instance: Instance, force = false): Promise<void> {
const unlock = await getFetchInstanceMetadataLock(instance.host);
export async function fetchInstanceMetadata(instance_host: string, force = false): Promise<void> {
const unlock = await getFetchInstanceMetadataLock(instance_host);
const instance = await Instances.findOneBy({ host: instance_host });
if (!force) {
const _instance = await Instances.findOneBy({ host: instance.host });
if (_instance && _instance.infoUpdatedAt) {
const threshold = Date.now() - _instance.infoUpdatedAt.getTime();
if (instance && !instance.isNotResponding && instance.infoUpdatedAt) {
const threshold = Date.now() - instance.infoUpdatedAt.getTime();
if (threshold > DAY) {
// queue a job to deal with it later
createUpdateInstanceJob(instance.host);
createUpdateInstanceJob(instance_host);
}
// if the instance is already known and force is not specified,
// never update the data in the same thread
@ -30,24 +29,24 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
}
}
logger.info(`Fetching metadata of ${instance.host} ...`);
logger.info(`Fetching metadata of ${instance_host} ...`);
try {
const [info, dom, manifest] = await Promise.all([
fetchNodeinfo(instance).catch(() => null),
fetchDom(instance).catch(() => null),
fetchManifest(instance).catch(() => null),
fetchNodeinfo(instance_host).catch(() => null),
fetchDom(instance_host).catch(() => null),
fetchManifest(instance_host).catch(() => null),
]);
const [favicon, icon, themeColor, name, description] = await Promise.all([
fetchFaviconUrl(instance, dom).catch(() => null),
fetchIconUrl(instance, dom, manifest).catch(() => null),
fetchFaviconUrl(instance_host, dom).catch(() => null),
fetchIconUrl(instance_host, dom, manifest).catch(() => null),
getThemeColor(info, dom, manifest).catch(() => null),
getSiteName(info, dom, manifest).catch(() => null),
getDescription(info, dom, manifest).catch(() => null),
]);
logger.succ(`Successfuly fetched metadata of ${instance.host}`);
logger.succ(`Successfuly fetched metadata of ${instance_host}`);
const updates = {
infoUpdatedAt: new Date(),
@ -69,9 +68,9 @@ export async function fetchInstanceMetadata(instance: Instance, force = false):
await Instances.update(instance.id, updates);
logger.succ(`Successfuly updated metadata of ${instance.host}`);
logger.succ(`Successfuly updated metadata of ${instance_host}`);
} catch (e) {
logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
logger.error(`Failed to update metadata of ${instance_host}: ${e}`);
} finally {
unlock();
}
@ -96,11 +95,11 @@ type NodeInfo = {
};
};
async function fetchNodeinfo(instance: Instance): Promise<NodeInfo> {
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
async function fetchNodeinfo(instance_host: string): Promise<NodeInfo> {
logger.info(`Fetching nodeinfo of ${instance_host} ...`);
try {
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
const wellknown = await getJson('https://' + instance_host + '/.well-known/nodeinfo')
.catch(e => {
if (e.statusCode === 404) {
throw new Error('No nodeinfo provided');
@ -129,21 +128,21 @@ async function fetchNodeinfo(instance: Instance): Promise<NodeInfo> {
throw new Error(e.statusCode || e.message);
});
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
logger.succ(`Successfuly fetched nodeinfo of ${instance_host}`);
return info as NodeInfo;
} catch (e) {
const message = e instanceof Error ? e.message : e;
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${message}`);
logger.error(`Failed to fetch nodeinfo of ${instance_host}: ${message}`);
throw e;
}
}
async function fetchDom(instance: Instance): Promise<DOMWindow['document']> {
logger.info(`Fetching HTML of ${instance.host} ...`);
async function fetchDom(instance_host: string): Promise<DOMWindow['document']> {
logger.info(`Fetching HTML of ${instance_host} ...`);
const url = 'https://' + instance.host;
const url = 'https://' + instance_host;
const html = await getHtml(url);
@ -153,8 +152,8 @@ async function fetchDom(instance: Instance): Promise<DOMWindow['document']> {
return doc;
}
async function fetchManifest(instance: Instance): Promise<Record<string, unknown> | null> {
const url = 'https://' + instance.host;
async function fetchManifest(instance_host: string): Promise<Record<string, unknown> | null> {
const url = 'https://' + instance_host;
const manifestUrl = url + '/manifest.json';
@ -163,8 +162,8 @@ async function fetchManifest(instance: Instance): Promise<Record<string, unknown
return manifest;
}
async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] | null): Promise<string | null> {
const url = 'https://' + instance.host;
async function fetchFaviconUrl(instance_host: string, doc: DOMWindow['document'] | null): Promise<string | null> {
const url = 'https://' + instance_host;
if (doc) {
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
@ -190,19 +189,19 @@ async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] |
return null;
}
async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
async function fetchIconUrl(instance_host: string, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
const url = 'https://' + instance.host;
const url = 'https://' + instance_host;
return (new URL(manifest.icons[0].src, url)).href;
}
if (doc) {
const url = 'https://' + instance.host;
const url = 'https://' + instance_host;
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const links = Array.from(doc.getElementsByTagName('link')).reverse();
// https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559
const href =
const href =
[
links.find(link => link.relList.contains('apple-touch-icon-precomposed'))?.href,
links.find(link => link.relList.contains('apple-touch-icon'))?.href,