From 68a9aac9573969311dd00a44536c3ee4c05b883d Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sat, 31 Mar 2018 19:55:00 +0900 Subject: [PATCH] Implement remote status retrieval --- package.json | 1 + src/{server/api => }/common/drive/add-file.ts | 12 +- .../api => }/common/drive/upload_from_url.ts | 2 +- src/{server/api => common}/event.ts | 4 +- src/{server/api => }/common/push-sw.ts | 4 +- src/common/remote/activitypub/act/create.ts | 9 + src/common/remote/activitypub/act/index.ts | 22 +++ src/common/remote/activitypub/create.ts | 86 ++++++++++ .../remote/activitypub/resolve-person.ts | 104 +++++++++++ src/common/remote/activitypub/resolver.ts | 97 +++++++++++ src/common/remote/activitypub/type.ts | 3 + src/common/remote/resolve-user.ts | 26 +++ src/common/remote/webfinger.ts | 25 +++ src/models/remote-user-object.ts | 15 ++ src/models/user.ts | 3 + src/processor/http/index.ts | 9 + src/processor/http/perform-activitypub.ts | 6 + .../{ => http}/report-github-failure.ts | 4 +- src/processor/index.ts | 13 +- src/server/api/common/notify.ts | 2 +- .../api/common/read-messaging-message.ts | 6 +- src/server/api/common/read-notification.ts | 2 +- .../api/endpoints/drive/files/create.ts | 2 +- .../api/endpoints/drive/files/update.ts | 2 +- .../endpoints/drive/files/upload_from_url.ts | 2 +- .../api/endpoints/drive/folders/create.ts | 2 +- .../api/endpoints/drive/folders/update.ts | 2 +- src/server/api/endpoints/following/create.ts | 2 +- src/server/api/endpoints/following/delete.ts | 2 +- .../api/endpoints/i/regenerate_token.ts | 2 +- src/server/api/endpoints/i/update.ts | 2 +- .../api/endpoints/i/update_client_setting.ts | 2 +- src/server/api/endpoints/i/update_home.ts | 2 +- .../api/endpoints/i/update_mobile_home.ts | 2 +- .../endpoints/messaging/messages/create.ts | 4 +- .../notifications/mark_as_read_all.ts | 2 +- src/server/api/endpoints/othello/match.ts | 2 +- src/server/api/endpoints/posts/create.ts | 2 +- src/server/api/endpoints/posts/polls/vote.ts | 2 +- .../api/endpoints/posts/reactions/create.ts | 2 +- src/server/api/endpoints/users/show.ts | 162 +----------------- src/server/api/private/signin.ts | 2 +- src/server/api/service/github.ts | 3 +- src/server/api/service/twitter.ts | 2 +- src/server/api/stream/othello-game.ts | 2 +- src/server/api/stream/othello.ts | 2 +- 46 files changed, 468 insertions(+), 198 deletions(-) rename src/{server/api => }/common/drive/add-file.ts (95%) rename src/{server/api => }/common/drive/upload_from_url.ts (92%) rename src/{server/api => common}/event.ts (97%) rename src/{server/api => }/common/push-sw.ts (92%) create mode 100644 src/common/remote/activitypub/act/create.ts create mode 100644 src/common/remote/activitypub/act/index.ts create mode 100644 src/common/remote/activitypub/create.ts create mode 100644 src/common/remote/activitypub/resolve-person.ts create mode 100644 src/common/remote/activitypub/resolver.ts create mode 100644 src/common/remote/activitypub/type.ts create mode 100644 src/common/remote/resolve-user.ts create mode 100644 src/common/remote/webfinger.ts create mode 100644 src/models/remote-user-object.ts create mode 100644 src/processor/http/index.ts create mode 100644 src/processor/http/perform-activitypub.ts rename src/processor/{ => http}/report-github-failure.ts (87%) diff --git a/package.json b/package.json index d1f544f86..4275c1c1c 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "deep-equal": "1.0.1", "deepcopy": "0.6.3", "diskusage": "0.2.4", + "dompurify": "^1.0.3", "elasticsearch": "14.2.2", "element-ui": "2.3.2", "emojilib": "2.2.12", diff --git a/src/server/api/common/drive/add-file.ts b/src/common/drive/add-file.ts similarity index 95% rename from src/server/api/common/drive/add-file.ts rename to src/common/drive/add-file.ts index 4551f5574..52a7713dd 100644 --- a/src/server/api/common/drive/add-file.ts +++ b/src/common/drive/add-file.ts @@ -10,12 +10,12 @@ import * as debug from 'debug'; import fileType = require('file-type'); import prominence = require('prominence'); -import DriveFile, { getGridFSBucket } from '../../../../models/drive-file'; -import DriveFolder from '../../../../models/drive-folder'; -import { pack } from '../../../../models/drive-file'; -import event, { publishDriveStream } from '../../event'; -import getAcct from '../../../../common/user/get-acct'; -import config from '../../../../conf'; +import DriveFile, { getGridFSBucket } from '../../models/drive-file'; +import DriveFolder from '../../models/drive-folder'; +import { pack } from '../../models/drive-file'; +import event, { publishDriveStream } from '../event'; +import getAcct from '../user/get-acct'; +import config from '../../conf'; const gm = _gm.subClass({ imageMagick: true diff --git a/src/server/api/common/drive/upload_from_url.ts b/src/common/drive/upload_from_url.ts similarity index 92% rename from src/server/api/common/drive/upload_from_url.ts rename to src/common/drive/upload_from_url.ts index b825e4c53..5dd969593 100644 --- a/src/server/api/common/drive/upload_from_url.ts +++ b/src/common/drive/upload_from_url.ts @@ -1,5 +1,5 @@ import * as URL from 'url'; -import { IDriveFile, validateFileName } from '../../../../models/drive-file'; +import { IDriveFile, validateFileName } from '../../models/drive-file'; import create from './add-file'; import * as debug from 'debug'; import * as tmp from 'tmp'; diff --git a/src/server/api/event.ts b/src/common/event.ts similarity index 97% rename from src/server/api/event.ts rename to src/common/event.ts index 98bf16113..53520f11c 100644 --- a/src/server/api/event.ts +++ b/src/common/event.ts @@ -1,7 +1,7 @@ import * as mongo from 'mongodb'; import * as redis from 'redis'; -import swPush from './common/push-sw'; -import config from '../../conf'; +import swPush from './push-sw'; +import config from '../conf'; type ID = string | mongo.ObjectID; diff --git a/src/server/api/common/push-sw.ts b/src/common/push-sw.ts similarity index 92% rename from src/server/api/common/push-sw.ts rename to src/common/push-sw.ts index 13227af8d..44c328e83 100644 --- a/src/server/api/common/push-sw.ts +++ b/src/common/push-sw.ts @@ -1,7 +1,7 @@ const push = require('web-push'); import * as mongo from 'mongodb'; -import Subscription from '../../../models/sw-subscription'; -import config from '../../../conf'; +import Subscription from '../models/sw-subscription'; +import config from '../conf'; if (config.sw) { // アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録 diff --git a/src/common/remote/activitypub/act/create.ts b/src/common/remote/activitypub/act/create.ts new file mode 100644 index 000000000..6c62f7ab9 --- /dev/null +++ b/src/common/remote/activitypub/act/create.ts @@ -0,0 +1,9 @@ +import create from '../create'; + +export default (resolver, actor, activity) => { + if ('actor' in activity && actor.account.uri !== activity.actor) { + throw new Error; + } + + return create(resolver, actor, activity.object); +}; diff --git a/src/common/remote/activitypub/act/index.ts b/src/common/remote/activitypub/act/index.ts new file mode 100644 index 000000000..0f4084a61 --- /dev/null +++ b/src/common/remote/activitypub/act/index.ts @@ -0,0 +1,22 @@ +import create from './create'; +import createObject from '../create'; +import Resolver from '../resolver'; + +export default (actor, value) => { + return (new Resolver).resolve(value).then(resolved => Promise.all(resolved.map(async asyncResult => { + const { resolver, object } = await asyncResult; + const created = await (await createObject(resolver, actor, [object]))[0]; + + if (created !== null) { + return created; + } + + switch (object.type) { + case 'Create': + return create(resolver, actor, object); + + default: + return null; + } + }))); +} diff --git a/src/common/remote/activitypub/create.ts b/src/common/remote/activitypub/create.ts new file mode 100644 index 000000000..4aaaeb306 --- /dev/null +++ b/src/common/remote/activitypub/create.ts @@ -0,0 +1,86 @@ +import { JSDOM } from 'jsdom'; +import config from '../../../conf'; +import Post from '../../../models/post'; +import RemoteUserObject, { IRemoteUserObject } from '../../../models/remote-user-object'; +import uploadFromUrl from '../../drive/upload_from_url'; +const createDOMPurify = require('dompurify'); + +function createRemoteUserObject($ref, $id, { id }) { + const object = { $ref, $id }; + + if (!id) { + return { object }; + } + + return RemoteUserObject.insert({ uri: id, object }); +} + +async function createImage(actor, object) { + if ('attributedTo' in object && actor.account.uri !== object.attributedTo) { + throw new Error; + } + + const { _id } = await uploadFromUrl(object.url, actor); + return createRemoteUserObject('driveFiles.files', _id, object); +} + +async function createNote(resolver, actor, object) { + if ('attributedTo' in object && actor.account.uri !== object.attributedTo) { + throw new Error; + } + + const mediaIds = 'attachment' in object && + (await Promise.all(await create(resolver, actor, object.attachment))) + .filter(media => media !== null && media.object.$ref === 'driveFiles.files') + .map(({ object }) => object.$id); + + const { window } = new JSDOM(object.content); + + const { _id } = await Post.insert({ + channelId: undefined, + index: undefined, + createdAt: new Date(object.published), + mediaIds, + replyId: undefined, + repostId: undefined, + poll: undefined, + text: window.document.body.textContent, + textHtml: object.content && createDOMPurify(window).sanitize(object.content), + userId: actor._id, + appId: null, + viaMobile: false, + geo: undefined + }); + + // Register to search database + if (object.content && config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + + es.index({ + index: 'misskey', + type: 'post', + id: _id.toString(), + body: { + text: window.document.body.textContent + } + }); + } + + return createRemoteUserObject('posts', _id, object); +} + +export default async function create(parentResolver, actor, value): Promise[]> { + const results = await parentResolver.resolveRemoteUserObjects(value); + + return results.map(asyncResult => asyncResult.then(({ resolver, object }) => { + switch (object.type) { + case 'Image': + return createImage(actor, object); + + case 'Note': + return createNote(resolver, actor, object); + } + + return null; + })); +}; diff --git a/src/common/remote/activitypub/resolve-person.ts b/src/common/remote/activitypub/resolve-person.ts new file mode 100644 index 000000000..c7c131b0e --- /dev/null +++ b/src/common/remote/activitypub/resolve-person.ts @@ -0,0 +1,104 @@ +import { JSDOM } from 'jsdom'; +import { toUnicode } from 'punycode'; +import User, { validateUsername, isValidName, isValidDescription } from '../../../models/user'; +import queue from '../../../queue'; +import webFinger from '../webfinger'; +import create from './create'; +import Resolver from './resolver'; + +async function isCollection(collection) { + return ['Collection', 'OrderedCollection'].includes(collection.type); +} + +export default async (value, usernameLower, hostLower, acctLower) => { + if (!validateUsername(usernameLower)) { + throw new Error; + } + + const { resolver, object } = await (new Resolver).resolveOne(value); + + if ( + object === null || + object.type !== 'Person' || + typeof object.preferredUsername !== 'string' || + object.preferredUsername.toLowerCase() !== usernameLower || + !isValidName(object.name) || + !isValidDescription(object.summary) + ) { + throw new Error; + } + + const [followers, following, outbox, finger] = await Promise.all([ + resolver.resolveOne(object.followers).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + resolver.resolveOne(object.following).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + resolver.resolveOne(object.outbox).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + webFinger(object.id, acctLower), + ]); + + const summaryDOM = JSDOM.fragment(object.summary); + + // Create user + const user = await User.insert({ + avatarId: null, + bannerId: null, + createdAt: Date.parse(object.published), + description: summaryDOM.textContent, + followersCount: followers.totalItem, + followingCount: following.totalItem, + name: object.name, + postsCount: outbox.totalItem, + driveCapacity: 1024 * 1024 * 8, // 8MiB + username: object.preferredUsername, + usernameLower, + host: toUnicode(finger.subject.replace(/^.*?@/, '')), + hostLower, + account: { + uri: object.id, + }, + }); + + queue.create('http', { + type: 'performActivityPub', + actor: user._id, + outbox + }).save(); + + const [avatarId, bannerId] = await Promise.all([ + object.icon, + object.image + ].map(async value => { + if (value === undefined) { + return null; + } + + try { + const created = await create(resolver, user, value); + + await Promise.all(created.map(asyncCreated => asyncCreated.then(created => { + if (created !== null && created.object.$ref === 'driveFiles.files') { + throw created.object.$id; + } + }, () => {}))); + + return null; + } catch (id) { + return id; + } + })); + + User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); + + user.avatarId = avatarId; + user.bannerId = bannerId; + + return user; +}; diff --git a/src/common/remote/activitypub/resolver.ts b/src/common/remote/activitypub/resolver.ts new file mode 100644 index 000000000..50ac1b0b1 --- /dev/null +++ b/src/common/remote/activitypub/resolver.ts @@ -0,0 +1,97 @@ +import RemoteUserObject from '../../../models/remote-user-object'; +import { IObject } from './type'; +const request = require('request-promise-native'); + +type IResult = { + resolver: Resolver; + object: IObject; +}; + +async function resolveUnrequestedOne(this: Resolver, value) { + if (typeof value !== 'string') { + return { resolver: this, object: value }; + } + + const resolver = new Resolver(this.requesting); + + resolver.requesting.add(value); + + const object = await request({ + url: value, + headers: { + Accept: 'application/activity+json, application/ld+json' + }, + json: true + }); + + if (object === null || ( + Array.isArray(object['@context']) ? + !object['@context'].includes('https://www.w3.org/ns/activitystreams') : + object['@context'] !== 'https://www.w3.org/ns/activitystreams' + )) { + throw new Error; + } + + return { resolver, object }; +} + +async function resolveCollection(this: Resolver, value) { + if (Array.isArray(value)) { + return value; + } + + const resolved = typeof value === 'string' ? + await resolveUnrequestedOne.call(this, value) : + value; + + switch (resolved.type) { + case 'Collection': + return resolved.items; + + case 'OrderedCollection': + return resolved.orderedItems; + + default: + return [resolved]; + } +} + +export default class Resolver { + requesting: Set; + + constructor(iterable?: Iterable) { + this.requesting = new Set(iterable); + } + + async resolve(value): Promise[]> { + const collection = await resolveCollection.call(this, value); + + return collection + .filter(element => !this.requesting.has(element)) + .map(resolveUnrequestedOne.bind(this)); + } + + resolveOne(value) { + if (this.requesting.has(value)) { + throw new Error; + } + + return resolveUnrequestedOne.call(this, value); + } + + async resolveRemoteUserObjects(value) { + const collection = await resolveCollection.call(this, value); + + return collection.filter(element => !this.requesting.has(element)).map(element => { + if (typeof element === 'string') { + const object = RemoteUserObject.findOne({ uri: element }); + + if (object !== null) { + return object; + } + } + + return resolveUnrequestedOne.call(this, element); + }); + } +} diff --git a/src/common/remote/activitypub/type.ts b/src/common/remote/activitypub/type.ts new file mode 100644 index 000000000..5c4750e14 --- /dev/null +++ b/src/common/remote/activitypub/type.ts @@ -0,0 +1,3 @@ +export type IObject = { + type: string; +} diff --git a/src/common/remote/resolve-user.ts b/src/common/remote/resolve-user.ts new file mode 100644 index 000000000..13d155830 --- /dev/null +++ b/src/common/remote/resolve-user.ts @@ -0,0 +1,26 @@ +import { toUnicode, toASCII } from 'punycode'; +import User from '../../models/user'; +import resolvePerson from './activitypub/resolve-person'; +import webFinger from './webfinger'; + +export default async (username, host, option) => { + const usernameLower = username.toLowerCase(); + const hostLowerAscii = toASCII(host).toLowerCase(); + const hostLower = toUnicode(hostLowerAscii); + + let user = await User.findOne({ usernameLower, hostLower }, option); + + if (user === null) { + const acctLower = `${usernameLower}@${hostLowerAscii}`; + + const finger = await webFinger(acctLower, acctLower); + const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self'); + if (!self) { + throw new Error; + } + + user = await resolvePerson(self.href, usernameLower, hostLower, acctLower); + } + + return user; +}; diff --git a/src/common/remote/webfinger.ts b/src/common/remote/webfinger.ts new file mode 100644 index 000000000..23f0aaa55 --- /dev/null +++ b/src/common/remote/webfinger.ts @@ -0,0 +1,25 @@ +const WebFinger = require('webfinger.js'); + +const webFinger = new WebFinger({}); + +type ILink = { + href: string; + rel: string; +} + +type IWebFinger = { + links: Array; + subject: string; +} + +export default (query, verifier): Promise => new Promise((res, rej) => webFinger.lookup(query, (error, result) => { + if (error) { + return rej(error); + } + + if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) { + return rej('WebFinger verfification failed'); + } + + res(result.object); +})); diff --git a/src/models/remote-user-object.ts b/src/models/remote-user-object.ts new file mode 100644 index 000000000..fb5b337c9 --- /dev/null +++ b/src/models/remote-user-object.ts @@ -0,0 +1,15 @@ +import * as mongodb from 'mongodb'; +import db from '../db/mongodb'; + +const RemoteUserObject = db.get('remoteUserObjects'); + +export default RemoteUserObject; + +export type IRemoteUserObject = { + _id: mongodb.ObjectID; + uri: string; + object: { + $ref: string; + $id: mongodb.ObjectID; + } +}; diff --git a/src/models/user.ts b/src/models/user.ts index 4fbfdec90..4728682d6 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -97,6 +97,9 @@ export type IUser = { account: ILocalAccount | IRemoteAccount; }; +export type ILocalUser = IUser & { account: ILocalAccount }; +export type IRemoteUser = IUser & { account: IRemoteAccount }; + export function init(user): IUser { user._id = new mongo.ObjectID(user._id); user.avatarId = new mongo.ObjectID(user.avatarId); diff --git a/src/processor/http/index.ts b/src/processor/http/index.ts new file mode 100644 index 000000000..da942ad2a --- /dev/null +++ b/src/processor/http/index.ts @@ -0,0 +1,9 @@ +import performActivityPub from './perform-activitypub'; +import reportGitHubFailure from './report-github-failure'; + +const handlers = { + performActivityPub, + reportGitHubFailure, +}; + +export default (job, done) => handlers[job.data.type](job).then(() => done(), done); diff --git a/src/processor/http/perform-activitypub.ts b/src/processor/http/perform-activitypub.ts new file mode 100644 index 000000000..5b1a02173 --- /dev/null +++ b/src/processor/http/perform-activitypub.ts @@ -0,0 +1,6 @@ +import User from '../../models/user'; +import act from '../../common/remote/activitypub/act'; + +export default ({ data }, done) => User.findOne({ _id: data.actor }) + .then(actor => act(actor, data.outbox)) + .then(() => done(), done); diff --git a/src/processor/report-github-failure.ts b/src/processor/http/report-github-failure.ts similarity index 87% rename from src/processor/report-github-failure.ts rename to src/processor/http/report-github-failure.ts index 610ffe276..53924a0fb 100644 --- a/src/processor/report-github-failure.ts +++ b/src/processor/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request'; -import User from '../models/user'; -const createPost = require('../server/api/endpoints/posts/create'); +import User from '../../models/user'; +const createPost = require('../../server/api/endpoints/posts/create'); export default ({ data }, done) => { const asyncBot = User.findOne({ _id: data.userId }); diff --git a/src/processor/index.ts b/src/processor/index.ts index f06cf24e8..cd271d372 100644 --- a/src/processor/index.ts +++ b/src/processor/index.ts @@ -1,4 +1,13 @@ import queue from '../queue'; -import reportGitHubFailure from './report-github-failure'; +import http from './http'; -export default () => queue.process('gitHubFailureReport', reportGitHubFailure); +/* + 256 is the default concurrency limit of Mozilla Firefox and Google + Chromium. + + a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google + https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff + Network.http.max-connections - MozillaZine Knowledge Base + http://kb.mozillazine.org/Network.http.max-connections +*/ +export default () => queue.process('http', 256, http); diff --git a/src/server/api/common/notify.ts b/src/server/api/common/notify.ts index f90506cf3..69bf8480b 100644 --- a/src/server/api/common/notify.ts +++ b/src/server/api/common/notify.ts @@ -1,7 +1,7 @@ import * as mongo from 'mongodb'; import Notification from '../../../models/notification'; import Mute from '../../../models/mute'; -import event from '../event'; +import event from '../../../common/event'; import { pack } from '../../../models/notification'; export default ( diff --git a/src/server/api/common/read-messaging-message.ts b/src/server/api/common/read-messaging-message.ts index f728130bb..127ea1865 100644 --- a/src/server/api/common/read-messaging-message.ts +++ b/src/server/api/common/read-messaging-message.ts @@ -1,9 +1,9 @@ import * as mongo from 'mongodb'; import Message from '../../../models/messaging-message'; import { IMessagingMessage as IMessage } from '../../../models/messaging-message'; -import publishUserStream from '../event'; -import { publishMessagingStream } from '../event'; -import { publishMessagingIndexStream } from '../event'; +import publishUserStream from '../../../common/event'; +import { publishMessagingStream } from '../../../common/event'; +import { publishMessagingIndexStream } from '../../../common/event'; /** * Mark as read message(s) diff --git a/src/server/api/common/read-notification.ts b/src/server/api/common/read-notification.ts index 27632c7ec..9b2012182 100644 --- a/src/server/api/common/read-notification.ts +++ b/src/server/api/common/read-notification.ts @@ -1,6 +1,6 @@ import * as mongo from 'mongodb'; import { default as Notification, INotification } from '../../../models/notification'; -import publishUserStream from '../event'; +import publishUserStream from '../../../common/event'; /** * Mark as read notification(s) diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts index 53c8c7067..cf4e35cd1 100644 --- a/src/server/api/endpoints/drive/files/create.ts +++ b/src/server/api/endpoints/drive/files/create.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import { validateFileName, pack } from '../../../../../models/drive-file'; -import create from '../../../common/drive/add-file'; +import create from '../../../../../common/drive/add-file'; /** * Create a file diff --git a/src/server/api/endpoints/drive/files/update.ts b/src/server/api/endpoints/drive/files/update.ts index 836b4cfcd..5d0b915f9 100644 --- a/src/server/api/endpoints/drive/files/update.ts +++ b/src/server/api/endpoints/drive/files/update.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import DriveFolder from '../../../../../models/drive-folder'; import DriveFile, { validateFileName, pack } from '../../../../../models/drive-file'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Update a file diff --git a/src/server/api/endpoints/drive/files/upload_from_url.ts b/src/server/api/endpoints/drive/files/upload_from_url.ts index 7262f09bb..01d875055 100644 --- a/src/server/api/endpoints/drive/files/upload_from_url.ts +++ b/src/server/api/endpoints/drive/files/upload_from_url.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import { pack } from '../../../../../models/drive-file'; -import uploadFromUrl from '../../../common/drive/upload_from_url'; +import uploadFromUrl from '../../../../../common/drive/upload_from_url'; /** * Create a file from a URL diff --git a/src/server/api/endpoints/drive/folders/create.ts b/src/server/api/endpoints/drive/folders/create.ts index 24e035930..bd3b0a0b1 100644 --- a/src/server/api/endpoints/drive/folders/create.ts +++ b/src/server/api/endpoints/drive/folders/create.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Create drive folder diff --git a/src/server/api/endpoints/drive/folders/update.ts b/src/server/api/endpoints/drive/folders/update.ts index 6c5a5c376..5ac81e5b5 100644 --- a/src/server/api/endpoints/drive/folders/update.ts +++ b/src/server/api/endpoints/drive/folders/update.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Update a folder diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index 1e24388a7..a689250e3 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -5,7 +5,7 @@ import $ from 'cafy'; import User, { pack as packUser } from '../../../../models/user'; import Following from '../../../../models/following'; import notify from '../../common/notify'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Follow a user diff --git a/src/server/api/endpoints/following/delete.ts b/src/server/api/endpoints/following/delete.ts index 7fc5f477f..ecca27d57 100644 --- a/src/server/api/endpoints/following/delete.ts +++ b/src/server/api/endpoints/following/delete.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User, { pack as packUser } from '../../../../models/user'; import Following from '../../../../models/following'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Unfollow a user diff --git a/src/server/api/endpoints/i/regenerate_token.ts b/src/server/api/endpoints/i/regenerate_token.ts index c35778ac0..0af622fd9 100644 --- a/src/server/api/endpoints/i/regenerate_token.ts +++ b/src/server/api/endpoints/i/regenerate_token.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import * as bcrypt from 'bcryptjs'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; import generateUserToken from '../../common/generate-native-user-token'; /** diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 8e198f3ad..b465e763e 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; import config from '../../../../conf'; /** diff --git a/src/server/api/endpoints/i/update_client_setting.ts b/src/server/api/endpoints/i/update_client_setting.ts index 03867b401..79789e664 100644 --- a/src/server/api/endpoints/i/update_client_setting.ts +++ b/src/server/api/endpoints/i/update_client_setting.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User, { pack } from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Update myself diff --git a/src/server/api/endpoints/i/update_home.ts b/src/server/api/endpoints/i/update_home.ts index 713cf9fcc..437f51d6f 100644 --- a/src/server/api/endpoints/i/update_home.ts +++ b/src/server/api/endpoints/i/update_home.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; module.exports = async (params, user) => new Promise(async (res, rej) => { // Get 'home' parameter diff --git a/src/server/api/endpoints/i/update_mobile_home.ts b/src/server/api/endpoints/i/update_mobile_home.ts index 6f28cebf9..783ca09d1 100644 --- a/src/server/api/endpoints/i/update_mobile_home.ts +++ b/src/server/api/endpoints/i/update_mobile_home.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; module.exports = async (params, user) => new Promise(async (res, rej) => { // Get 'home' parameter diff --git a/src/server/api/endpoints/messaging/messages/create.ts b/src/server/api/endpoints/messaging/messages/create.ts index 3d3b204da..b2b6c971d 100644 --- a/src/server/api/endpoints/messaging/messages/create.ts +++ b/src/server/api/endpoints/messaging/messages/create.ts @@ -9,8 +9,8 @@ import User from '../../../../../models/user'; import Mute from '../../../../../models/mute'; import DriveFile from '../../../../../models/drive-file'; import { pack } from '../../../../../models/messaging-message'; -import publishUserStream from '../../../event'; -import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event'; +import publishUserStream from '../../../../../common/event'; +import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../../../common/event'; import html from '../../../../../common/text/html'; import parse from '../../../../../common/text/parse'; import config from '../../../../../conf'; diff --git a/src/server/api/endpoints/notifications/mark_as_read_all.ts b/src/server/api/endpoints/notifications/mark_as_read_all.ts index 3693ba87b..f9bc6ebf7 100644 --- a/src/server/api/endpoints/notifications/mark_as_read_all.ts +++ b/src/server/api/endpoints/notifications/mark_as_read_all.ts @@ -2,7 +2,7 @@ * Module dependencies */ import Notification from '../../../../models/notification'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Mark as read all notifications diff --git a/src/server/api/endpoints/othello/match.ts b/src/server/api/endpoints/othello/match.ts index 03168095d..992b93d41 100644 --- a/src/server/api/endpoints/othello/match.ts +++ b/src/server/api/endpoints/othello/match.ts @@ -2,7 +2,7 @@ import $ from 'cafy'; import Matching, { pack as packMatching } from '../../../../models/othello-matching'; import OthelloGame, { pack as packGame } from '../../../../models/othello-game'; import User from '../../../../models/user'; -import publishUserStream, { publishOthelloStream } from '../../event'; +import publishUserStream, { publishOthelloStream } from '../../../../common/event'; import { eighteight } from '../../../../common/othello/maps'; module.exports = (params, user) => new Promise(async (res, rej) => { diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index 5342f7772..42901ebcb 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -16,7 +16,7 @@ import ChannelWatching from '../../../../models/channel-watching'; import { pack } from '../../../../models/post'; import notify from '../../common/notify'; import watch from '../../common/watch-post'; -import event, { pushSw, publishChannelStream } from '../../event'; +import event, { pushSw, publishChannelStream } from '../../../../common/event'; import getAcct from '../../../../common/user/get-acct'; import parseAcct from '../../../../common/user/parse-acct'; import config from '../../../../conf'; diff --git a/src/server/api/endpoints/posts/polls/vote.ts b/src/server/api/endpoints/posts/polls/vote.ts index b970c05e8..98df074e5 100644 --- a/src/server/api/endpoints/posts/polls/vote.ts +++ b/src/server/api/endpoints/posts/polls/vote.ts @@ -7,7 +7,7 @@ import Post from '../../../../../models/post'; import Watching from '../../../../../models/post-watching'; import notify from '../../../common/notify'; import watch from '../../../common/watch-post'; -import { publishPostStream } from '../../../event'; +import { publishPostStream } from '../../../../../common/event'; /** * Vote poll of a post diff --git a/src/server/api/endpoints/posts/reactions/create.ts b/src/server/api/endpoints/posts/reactions/create.ts index 5d2b5a7ed..8db76d643 100644 --- a/src/server/api/endpoints/posts/reactions/create.ts +++ b/src/server/api/endpoints/posts/reactions/create.ts @@ -8,7 +8,7 @@ import { pack as packUser } from '../../../../../models/user'; import Watching from '../../../../../models/post-watching'; import notify from '../../../common/notify'; import watch from '../../../common/watch-post'; -import { publishPostStream, pushSw } from '../../../event'; +import { publishPostStream, pushSw } from '../../../../../common/event'; /** * React to a post diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts index fd51d386b..9cd8716fe 100644 --- a/src/server/api/endpoints/users/show.ts +++ b/src/server/api/endpoints/users/show.ts @@ -2,49 +2,10 @@ * Module dependencies */ import $ from 'cafy'; -import { JSDOM } from 'jsdom'; -import { toUnicode, toASCII } from 'punycode'; -import uploadFromUrl from '../../common/drive/upload_from_url'; -import User, { pack, validateUsername, isValidName, isValidDescription } from '../../../../models/user'; -const request = require('request-promise-native'); -const WebFinger = require('webfinger.js'); +import User, { pack } from '../../../../models/user'; +import resolveRemoteUser from '../../../../common/remote/resolve-user'; -const webFinger = new WebFinger({}); - -async function getCollectionCount(url) { - if (!url) { - return null; - } - - try { - const collection = await request({ url, json: true }); - return collection ? collection.totalItems : null; - } catch (exception) { - return null; - } -} - -function findUser(q) { - return User.findOne(q, { - fields: { - data: false - } - }); -} - -function webFingerAndVerify(query, verifier) { - return new Promise((res, rej) => webFinger.lookup(query, (error, result) => { - if (error) { - return rej(error); - } - - if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) { - return rej('WebFinger verfification failed'); - } - - res(result.object); - })); -} +const cursorOption = { fields: { data: false } }; /** * Show a user @@ -74,124 +35,17 @@ module.exports = (params, me) => new Promise(async (res, rej) => { // Lookup user if (typeof host === 'string') { - const usernameLower = username.toLowerCase(); - const hostLowerAscii = toASCII(host).toLowerCase(); - const hostLower = toUnicode(hostLowerAscii); - - user = await findUser({ usernameLower, hostLower }); - - if (user === null) { - const acctLower = `${usernameLower}@${hostLowerAscii}`; - let activityStreams; - let finger; - let followersCount; - let followingCount; - let postsCount; - - if (!validateUsername(username)) { - return rej('username validation failed'); - } - - try { - finger = await webFingerAndVerify(acctLower, acctLower); - } catch (exception) { - return rej('WebFinger lookup failed'); - } - - const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self'); - if (!self) { - return rej('WebFinger has no reference to self representation'); - } - - try { - activityStreams = await request({ - url: self.href, - headers: { - Accept: 'application/activity+json, application/ld+json' - }, - json: true - }); - } catch (exception) { - return rej('failed to retrieve ActivityStreams representation'); - } - - if (!(activityStreams && - (Array.isArray(activityStreams['@context']) ? - activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') : - activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') && - activityStreams.type === 'Person' && - typeof activityStreams.preferredUsername === 'string' && - activityStreams.preferredUsername.toLowerCase() === usernameLower && - isValidName(activityStreams.name) && - isValidDescription(activityStreams.summary) - )) { - return rej('failed ActivityStreams validation'); - } - - try { - [followersCount, followingCount, postsCount] = await Promise.all([ - getCollectionCount(activityStreams.followers), - getCollectionCount(activityStreams.following), - getCollectionCount(activityStreams.outbox), - webFingerAndVerify(activityStreams.id, acctLower), - ]); - } catch (exception) { - return rej('failed to fetch assets'); - } - - const summaryDOM = JSDOM.fragment(activityStreams.summary); - - // Create user - user = await User.insert({ - avatarId: null, - bannerId: null, - createdAt: new Date(), - description: summaryDOM.textContent, - followersCount, - followingCount, - name: activityStreams.name, - postsCount, - driveCapacity: 1024 * 1024 * 8, // 8MiB - username, - usernameLower, - host: toUnicode(finger.subject.replace(/^.*?@/, '')), - hostLower, - account: { - uri: activityStreams.id, - }, - }); - - const [icon, image] = await Promise.all([ - activityStreams.icon, - activityStreams.image, - ].map(async image => { - if (!image || image.type !== 'Image') { - return { _id: null }; - } - - try { - return await uploadFromUrl(image.url, user); - } catch (exception) { - return { _id: null }; - } - })); - - User.update({ _id: user._id }, { - $set: { - avatarId: icon._id, - bannerId: image._id, - }, - }); - - user.avatarId = icon._id; - user.bannerId = icon._id; + try { + user = await resolveRemoteUser(username, host, cursorOption); + } catch (exception) { + return rej('failed to resolve remote user'); } } else { const q = userId !== undefined ? { _id: userId } : { usernameLower: username.toLowerCase(), host: null }; - user = await findUser(q); + user = await User.findOne(q, cursorOption); if (user === null) { return rej('user not found'); diff --git a/src/server/api/private/signin.ts b/src/server/api/private/signin.ts index d78fa11b8..4b7064491 100644 --- a/src/server/api/private/signin.ts +++ b/src/server/api/private/signin.ts @@ -3,7 +3,7 @@ import * as bcrypt from 'bcryptjs'; import * as speakeasy from 'speakeasy'; import { default as User, ILocalAccount, IUser } from '../../../models/user'; import Signin, { pack } from '../../../models/signin'; -import event from '../event'; +import event from '../../../common/event'; import signin from '../common/signin'; import config from '../../../conf'; diff --git a/src/server/api/service/github.ts b/src/server/api/service/github.ts index a2359cfb6..b4068c729 100644 --- a/src/server/api/service/github.ts +++ b/src/server/api/service/github.ts @@ -42,7 +42,8 @@ module.exports = async (app: express.Application) => { const commit = event.commit; const parent = commit.parents[0]; - queue.create('gitHubFailureReport', { + queue.create('http', { + type: 'gitHubFailureReport', userId: bot._id, parentUrl: parent.url, htmlUrl: commit.html_url, diff --git a/src/server/api/service/twitter.ts b/src/server/api/service/twitter.ts index d77341db2..73822b0bd 100644 --- a/src/server/api/service/twitter.ts +++ b/src/server/api/service/twitter.ts @@ -6,7 +6,7 @@ import * as uuid from 'uuid'; import autwh from 'autwh'; import redis from '../../../db/redis'; import User, { pack } from '../../../models/user'; -import event from '../event'; +import event from '../../../common/event'; import config from '../../../conf'; import signin from '../common/signin'; diff --git a/src/server/api/stream/othello-game.ts b/src/server/api/stream/othello-game.ts index b6a251c4c..b11915f8f 100644 --- a/src/server/api/stream/othello-game.ts +++ b/src/server/api/stream/othello-game.ts @@ -2,7 +2,7 @@ import * as websocket from 'websocket'; import * as redis from 'redis'; import * as CRC32 from 'crc-32'; import OthelloGame, { pack } from '../../../models/othello-game'; -import { publishOthelloGameStream } from '../event'; +import { publishOthelloGameStream } from '../../../common/event'; import Othello from '../../../common/othello/core'; import * as maps from '../../../common/othello/maps'; import { ParsedUrlQuery } from 'querystring'; diff --git a/src/server/api/stream/othello.ts b/src/server/api/stream/othello.ts index 4205afae7..1cf9a1494 100644 --- a/src/server/api/stream/othello.ts +++ b/src/server/api/stream/othello.ts @@ -2,7 +2,7 @@ import * as mongo from 'mongodb'; import * as websocket from 'websocket'; import * as redis from 'redis'; import Matching, { pack } from '../../../models/othello-matching'; -import publishUserStream from '../event'; +import publishUserStream from '../../../common/event'; export default function(request: websocket.request, connection: websocket.connection, subscriber: redis.RedisClient, user: any): void { // Subscribe othello stream