feat(server): Fetch icon url of an instance (#6591)

* feat(server): Fetch icon url of an instance

Resolve #6589

* chore: Rename the function
This commit is contained in:
syuilo 2020-07-26 11:04:07 +09:00 committed by GitHub
parent cf9266eab9
commit f1ef85b636
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 183 additions and 80 deletions

View file

@ -0,0 +1,14 @@
import {MigrationInterface, QueryRunner} from "typeorm";
export class instanceIconUrl1595676934834 implements MigrationInterface {
name = 'instanceIconUrl1595676934834'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "instance" ADD "iconUrl" character varying(256) DEFAULT null`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "iconUrl"`);
}
}

View file

@ -21,8 +21,8 @@ export function getApLock(uri: string, timeout = 30 * 1000) {
return lock(`ap-object:${uri}`, timeout);
}
export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
return lock(`nodeinfo:${host}`, timeout);
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
return lock(`instance:${host}`, timeout);
}
export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {

View file

@ -27,6 +27,27 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
return await res.json();
}
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
const res = await fetch(url, {
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept
}, headers || {}),
timeout,
agent: getAgentByUrl,
});
if (!res.ok) {
throw {
name: `StatusError`,
statusCode: res.status,
message: `${res.status} ${res.statusText}`,
};
}
return await res.text();
}
/**
* Get http non-proxy agent
*/

View file

@ -158,6 +158,11 @@ export class Instance {
})
public maintainerEmail: string | null;
@Column('varchar', {
length: 256, nullable: true, default: null,
})
public iconUrl: string | null;
@Column('timestamp with time zone', {
nullable: true,
})

View file

@ -4,7 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
import Logger from '../../services/logger';
import { Instances } from '../../models';
import { instanceChart } from '../../services/chart';
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
import { fetchMeta } from '../../misc/fetch-meta';
import { toPuny } from '../../misc/convert-host';
@ -48,7 +48,7 @@ export default async (job: Bull.Job) => {
isNotResponding: false
});
fetchNodeinfo(i);
fetchInstanceMetadata(i);
instanceChart.requestSent(i.host, true);
});

View file

@ -8,7 +8,7 @@ import { instanceChart } from '../../services/chart';
import { fetchMeta } from '../../misc/fetch-meta';
import { toPuny, extractDbHost } from '../../misc/convert-host';
import { getApId } from '../../remote/activitypub/type';
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
import { InboxJobData } from '..';
import DbResolver from '../../remote/activitypub/db-resolver';
import { resolvePerson } from '../../remote/activitypub/models/person';
@ -126,7 +126,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
isNotResponding: false
});
fetchNodeinfo(i);
fetchInstanceMetadata(i);
instanceChart.requestReceived(i.host);
});

View file

@ -26,7 +26,7 @@ import { validActor } from '../../../remote/activitypub/type';
import { getConnection } from 'typeorm';
import { ensure } from '../../../prelude/ensure';
import { toArray } from '../../../prelude/array';
import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata';
const logger = apLogger;
@ -204,7 +204,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host);
fetchNodeinfo(i);
fetchInstanceMetadata(i);
});
usersChart.update(user!, true);

View file

@ -0,0 +1,135 @@
import { JSDOM } from 'jsdom';
import fetch from 'node-fetch';
import { getJson, getHtml, getAgentByUrl } from '../misc/fetch';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
import { getFetchInstanceMetadataLock } from '../misc/app-lock';
import Logger from './logger';
import { URL } from 'url';
const logger = new Logger('metadata', 'cyan');
export async function fetchInstanceMetadata(instance: Instance): Promise<void> {
const unlock = await getFetchInstanceMetadataLock(instance.host);
const _instance = await Instances.findOne({ host: instance.host });
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
unlock();
return;
}
logger.info(`Fetching metadata of ${instance.host} ...`);
try {
const [info, icon] = await Promise.all([
fetchNodeinfo(instance).catch(() => null),
fetchIconUrl(instance).catch(() => null),
]);
logger.succ(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
} as Record<string, any>;
if (info) {
updates.softwareName = info.software.name.toLowerCase();
updates.softwareVersion = info.software.version;
updates.openRegistrations = info.openRegistrations;
updates.name = info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null;
updates.description = info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null;
updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null;
updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null;
}
if (icon) {
updates.iconUrl = icon;
}
await Instances.update(instance.id, updates);
logger.succ(`Successfuly updated metadata of ${instance.host}`);
} catch (e) {
logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
} finally {
unlock();
}
}
async function fetchNodeinfo(instance: Instance): Promise<Record<string, any>> {
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
}
const links = wellknown.links as any[];
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
const link = lnik2_1 || lnik2_0 || lnik1_0;
if (link == null) {
throw 'No nodeinfo link provided';
}
const info = await getJson(link.href)
.catch(e => {
throw e.statusCode || e.message;
});
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
return info;
} catch (e) {
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
throw e;
}
}
async function fetchIconUrl(instance: Instance): Promise<string | null> {
logger.info(`Fetching icon URL of ${instance.host} ...`);
const url = 'https://' + instance.host;
const html = await getHtml(url);
const { window } = new JSDOM(html);
const doc = window.document;
const hrefAppleTouchIconPrecomposed = doc.querySelector('link[rel="apple-touch-icon-precomposed"]')?.getAttribute('href');
const hrefAppleTouchIcon = doc.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href');
const hrefIcon = doc.querySelector('link[rel="icon"]')?.getAttribute('href');
const href = hrefAppleTouchIconPrecomposed || hrefAppleTouchIcon || hrefIcon;
if (href) {
return (new URL(href, url)).href;
}
const faviconUrl = url + '/favicon.ico';
const favicon = await fetch(faviconUrl, {
timeout: 10000,
agent: getAgentByUrl,
});
if (favicon.ok) {
return faviconUrl;
}
return null;
}

View file

@ -1,72 +0,0 @@
import { getJson } from '../misc/fetch';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
import { getNodeinfoLock } from '../misc/app-lock';
import Logger from '../services/logger';
export const logger = new Logger('nodeinfo', 'cyan');
export async function fetchNodeinfo(instance: Instance) {
const unlock = await getNodeinfoLock(instance.host);
const _instance = await Instances.findOne({ host: instance.host });
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
unlock();
return;
}
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
}
const links = wellknown.links as any[];
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
const link = lnik2_1 || lnik2_0 || lnik1_0;
if (link == null) {
throw 'No nodeinfo link provided';
}
const info = await getJson(link.href)
.catch(e => {
throw e.statusCode || e.message;
});
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),
softwareName: info.software.name.toLowerCase(),
softwareVersion: info.software.version,
openRegistrations: info.openRegistrations,
name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
});
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
} catch (e) {
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),
});
} finally {
unlock();
}
}