From e8b42d7e1668679e6a6ee0a7aea1e2ff7f37005b Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:12:35 +0900 Subject: [PATCH 01/58] wip --- src/post/create.ts | 107 +++++++++++- src/processor/http/deliver-post.ts | 93 ----------- src/processor/http/process-inbox.ts | 39 ----- src/queue.ts | 10 -- src/queue/index.ts | 37 ++++ .../processors}/db/delete-post-dependents.ts | 0 .../processors}/db/index.ts | 0 src/queue/processors/http/deliver.ts | 17 ++ .../processors}/http/follow.ts | 0 .../processors}/http/index.ts | 0 .../processors}/http/perform-activitypub.ts | 0 src/queue/processors/http/process-inbox.ts | 55 ++++++ .../processors}/http/report-github-failure.ts | 0 .../processors}/http/unfollow.ts | 0 src/{processor => queue/processors}/index.ts | 0 src/remote/activitypub/act/create.ts | 92 +++++++++- src/remote/activitypub/act/index.ts | 44 +++-- src/remote/activitypub/create.ts | 158 ------------------ src/remote/activitypub/resolver.ts | 77 ++++----- src/server/activitypub/inbox.ts | 2 +- 20 files changed, 354 insertions(+), 377 deletions(-) delete mode 100644 src/processor/http/deliver-post.ts delete mode 100644 src/processor/http/process-inbox.ts delete mode 100644 src/queue.ts create mode 100644 src/queue/index.ts rename src/{processor => queue/processors}/db/delete-post-dependents.ts (100%) rename src/{processor => queue/processors}/db/index.ts (100%) create mode 100644 src/queue/processors/http/deliver.ts rename src/{processor => queue/processors}/http/follow.ts (100%) rename src/{processor => queue/processors}/http/index.ts (100%) rename src/{processor => queue/processors}/http/perform-activitypub.ts (100%) create mode 100644 src/queue/processors/http/process-inbox.ts rename src/{processor => queue/processors}/http/report-github-failure.ts (100%) rename src/{processor => queue/processors}/http/unfollow.ts (100%) rename src/{processor => queue/processors}/index.ts (100%) delete mode 100644 src/remote/activitypub/create.ts diff --git a/src/post/create.ts b/src/post/create.ts index ecea37382..f78bbe752 100644 --- a/src/post/create.ts +++ b/src/post/create.ts @@ -1,8 +1,14 @@ import parseAcct from '../acct/parse'; -import Post from '../models/post'; -import User from '../models/user'; +import Post, { pack } from '../models/post'; +import User, { isLocalUser, isRemoteUser, IUser } from '../models/user'; +import stream from '../publishers/stream'; +import Following from '../models/following'; +import { createHttp } from '../queue'; +import renderNote from '../remote/activitypub/renderer/note'; +import renderCreate from '../remote/activitypub/renderer/create'; +import context from '../remote/activitypub/renderer/context'; -export default async (post, reply, repost, atMentions) => { +export default async (user: IUser, post, reply, repost, atMentions) => { post.mentions = []; function addMention(mentionee) { @@ -46,5 +52,98 @@ export default async (post, reply, repost, atMentions) => { addMention(_id); })); - return Post.insert(post); + const inserted = await Post.insert(post); + + User.update({ _id: user._id }, { + // Increment my posts count + $inc: { + postsCount: 1 + }, + + $set: { + latestPost: post._id + } + }); + + const postObj = await pack(inserted); + + // タイムラインへの投稿 + if (!post.channelId) { + // Publish event to myself's stream + stream(post.userId, 'post', postObj); + + // Fetch all followers + const followers = await Following.aggregate([{ + $lookup: { + from: 'users', + localField: 'followerId', + foreignField: '_id', + as: 'follower' + } + }, { + $match: { + followeeId: post.userId + } + }], { + _id: false + }); + + const note = await renderNote(user, post); + const content = renderCreate(note); + content['@context'] = context; + + Promise.all(followers.map(({ follower }) => { + if (isLocalUser(follower)) { + // Publish event to followers stream + stream(follower._id, 'post', postObj); + } else { + // フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信 + if (isLocalUser(user)) { + createHttp({ + type: 'deliver', + user, + content, + to: follower.account.inbox + }).save(); + } + } + })); + } + + // チャンネルへの投稿 + /* TODO + if (post.channelId) { + promises.push( + // Increment channel index(posts count) + Channel.update({ _id: post.channelId }, { + $inc: { + index: 1 + } + }), + + // Publish event to channel + promisedPostObj.then(postObj => { + publishChannelStream(post.channelId, 'post', postObj); + }), + + Promise.all([ + promisedPostObj, + + // Get channel watchers + ChannelWatching.find({ + channelId: post.channelId, + // 削除されたドキュメントは除く + deletedAt: { $exists: false } + }) + ]).then(([postObj, watches]) => { + // チャンネルの視聴者(のタイムライン)に配信 + watches.forEach(w => { + stream(w.userId, 'post', postObj); + }); + }) + ); + }*/ + + return Promise.all(promises); + }; diff --git a/src/processor/http/deliver-post.ts b/src/processor/http/deliver-post.ts deleted file mode 100644 index c00ab912c..000000000 --- a/src/processor/http/deliver-post.ts +++ /dev/null @@ -1,93 +0,0 @@ -import Channel from '../../models/channel'; -import Following from '../../models/following'; -import ChannelWatching from '../../models/channel-watching'; -import Post, { pack } from '../../models/post'; -import User, { isLocalUser } from '../../models/user'; -import stream, { publishChannelStream } from '../../publishers/stream'; -import context from '../../remote/activitypub/renderer/context'; -import renderCreate from '../../remote/activitypub/renderer/create'; -import renderNote from '../../remote/activitypub/renderer/note'; -import request from '../../remote/request'; - -export default ({ data }) => Post.findOne({ _id: data.id }).then(post => { - const promisedPostObj = pack(post); - const promises = []; - - // タイムラインへの投稿 - if (!post.channelId) { - promises.push( - // Publish event to myself's stream - promisedPostObj.then(postObj => { - stream(post.userId, 'post', postObj); - }), - - Promise.all([ - User.findOne({ _id: post.userId }), - - // Fetch all followers - Following.aggregate([{ - $lookup: { - from: 'users', - localField: 'followerId', - foreignField: '_id', - as: 'follower' - } - }, { - $match: { - followeeId: post.userId - } - }], { - _id: false - }) - ]).then(([user, followers]) => Promise.all(followers.map(following => { - if (isLocalUser(following.follower)) { - // Publish event to followers stream - return promisedPostObj.then(postObj => { - stream(following.followerId, 'post', postObj); - }); - } - - return renderNote(user, post).then(note => { - const create = renderCreate(note); - create['@context'] = context; - return request(user, following.follower[0].account.inbox, create); - }); - }))) - ); - } - - // チャンネルへの投稿 - if (post.channelId) { - promises.push( - // Increment channel index(posts count) - Channel.update({ _id: post.channelId }, { - $inc: { - index: 1 - } - }), - - // Publish event to channel - promisedPostObj.then(postObj => { - publishChannelStream(post.channelId, 'post', postObj); - }), - - Promise.all([ - promisedPostObj, - - // Get channel watchers - ChannelWatching.find({ - channelId: post.channelId, - // 削除されたドキュメントは除く - deletedAt: { $exists: false } - }) - ]).then(([postObj, watches]) => { - // チャンネルの視聴者(のタイムライン)に配信 - watches.forEach(w => { - stream(w.userId, 'post', postObj); - }); - }) - ); - } - - return Promise.all(promises); -}); diff --git a/src/processor/http/process-inbox.ts b/src/processor/http/process-inbox.ts deleted file mode 100644 index f102f8d6b..000000000 --- a/src/processor/http/process-inbox.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { verifySignature } from 'http-signature'; -import parseAcct from '../../acct/parse'; -import User, { IRemoteUser } from '../../models/user'; -import act from '../../remote/activitypub/act'; -import resolvePerson from '../../remote/activitypub/resolve-person'; -import Resolver from '../../remote/activitypub/resolver'; - -export default async ({ data }): Promise => { - const keyIdLower = data.signature.keyId.toLowerCase(); - let user; - - if (keyIdLower.startsWith('acct:')) { - const { username, host } = parseAcct(keyIdLower.slice('acct:'.length)); - if (host === null) { - throw 'request was made by local user'; - } - - user = await User.findOne({ usernameLower: username, hostLower: host }) as IRemoteUser; - } else { - user = await User.findOne({ - host: { $ne: null }, - 'account.publicKey.id': data.signature.keyId - }) as IRemoteUser; - - if (user === null) { - user = await resolvePerson(data.signature.keyId); - } - } - - if (user === null) { - throw 'failed to resolve user'; - } - - if (!verifySignature(data.signature, user.account.publicKey.publicKeyPem)) { - throw 'signature verification failed'; - } - - await Promise.all(await act(new Resolver(), user, data.inbox, true)); -}; diff --git a/src/queue.ts b/src/queue.ts deleted file mode 100644 index 08ea13c2a..000000000 --- a/src/queue.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { createQueue } from 'kue'; -import config from './config'; - -export default createQueue({ - redis: { - port: config.redis.port, - host: config.redis.host, - auth: config.redis.pass - } -}); diff --git a/src/queue/index.ts b/src/queue/index.ts new file mode 100644 index 000000000..c8c436b18 --- /dev/null +++ b/src/queue/index.ts @@ -0,0 +1,37 @@ +import { createQueue } from 'kue'; +import config from '../config'; +import db from './processors/db'; +import http from './processors/http'; + +const queue = createQueue({ + redis: { + port: config.redis.port, + host: config.redis.host, + auth: config.redis.pass + } +}); + +export function createHttp(data) { + return queue + .create('http', data) + .attempts(16) + .backoff({ delay: 16384, type: 'exponential' }); +} + +export function createDb(data) { + return queue.create('db', data); +} + +export function process() { + queue.process('db', db); + + /* + 256 is the default concurrency limit of Mozilla Firefox and Google + Chromium. + a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google + https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff + Network.http.max-connections - MozillaZine Knowledge Base + http://kb.mozillazine.org/Network.http.max-connections + */ + queue.process('http', 256, http); +} diff --git a/src/processor/db/delete-post-dependents.ts b/src/queue/processors/db/delete-post-dependents.ts similarity index 100% rename from src/processor/db/delete-post-dependents.ts rename to src/queue/processors/db/delete-post-dependents.ts diff --git a/src/processor/db/index.ts b/src/queue/processors/db/index.ts similarity index 100% rename from src/processor/db/index.ts rename to src/queue/processors/db/index.ts diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts new file mode 100644 index 000000000..8cd9eb624 --- /dev/null +++ b/src/queue/processors/http/deliver.ts @@ -0,0 +1,17 @@ +import * as kue from 'kue'; + +import Channel from '../../models/channel'; +import Following from '../../models/following'; +import ChannelWatching from '../../models/channel-watching'; +import Post, { pack } from '../../models/post'; +import User, { isLocalUser } from '../../models/user'; +import stream, { publishChannelStream } from '../../publishers/stream'; +import context from '../../remote/activitypub/renderer/context'; +import renderCreate from '../../remote/activitypub/renderer/create'; +import renderNote from '../../remote/activitypub/renderer/note'; +import request from '../../remote/request'; + +export default async (job: kue.Job, done): Promise => { + + request(user, following.follower[0].account.inbox, create); +} diff --git a/src/processor/http/follow.ts b/src/queue/processors/http/follow.ts similarity index 100% rename from src/processor/http/follow.ts rename to src/queue/processors/http/follow.ts diff --git a/src/processor/http/index.ts b/src/queue/processors/http/index.ts similarity index 100% rename from src/processor/http/index.ts rename to src/queue/processors/http/index.ts diff --git a/src/processor/http/perform-activitypub.ts b/src/queue/processors/http/perform-activitypub.ts similarity index 100% rename from src/processor/http/perform-activitypub.ts rename to src/queue/processors/http/perform-activitypub.ts diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts new file mode 100644 index 000000000..fff1fbf66 --- /dev/null +++ b/src/queue/processors/http/process-inbox.ts @@ -0,0 +1,55 @@ +import * as kue from 'kue'; + +import { verifySignature } from 'http-signature'; +import parseAcct from '../../acct/parse'; +import User, { IRemoteUser } from '../../models/user'; +import act from '../../remote/activitypub/act'; +import resolvePerson from '../../remote/activitypub/resolve-person'; +import Resolver from '../../remote/activitypub/resolver'; + +// ユーザーのinboxにアクティビティが届いた時の処理 +export default async (job: kue.Job, done): Promise => { + const signature = job.data.signature; + const activity = job.data.activity; + + const keyIdLower = signature.keyId.toLowerCase(); + let user; + + if (keyIdLower.startsWith('acct:')) { + const { username, host } = parseAcct(keyIdLower.slice('acct:'.length)); + if (host === null) { + console.warn(`request was made by local user: @${username}`); + done(); + } + + user = await User.findOne({ usernameLower: username, hostLower: host }) as IRemoteUser; + } else { + user = await User.findOne({ + host: { $ne: null }, + 'account.publicKey.id': signature.keyId + }) as IRemoteUser; + + // アクティビティを送信してきたユーザーがまだMisskeyサーバーに登録されていなかったら登録する + if (user === null) { + user = await resolvePerson(signature.keyId); + } + } + + if (user === null) { + done(new Error('failed to resolve user')); + return; + } + + if (!verifySignature(signature, user.account.publicKey.publicKeyPem)) { + done(new Error('signature verification failed')); + return; + } + + // アクティビティを処理 + try { + await act(new Resolver(), user, activity); + done(); + } catch (e) { + done(e); + } +}; diff --git a/src/processor/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts similarity index 100% rename from src/processor/http/report-github-failure.ts rename to src/queue/processors/http/report-github-failure.ts diff --git a/src/processor/http/unfollow.ts b/src/queue/processors/http/unfollow.ts similarity index 100% rename from src/processor/http/unfollow.ts rename to src/queue/processors/http/unfollow.ts diff --git a/src/processor/index.ts b/src/queue/processors/index.ts similarity index 100% rename from src/processor/index.ts rename to src/queue/processors/index.ts diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index fa681982c..c1a30ce7d 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -1,10 +1,92 @@ -import create from '../create'; -import Resolver from '../resolver'; +import { JSDOM } from 'jsdom'; +const createDOMPurify = require('dompurify'); -export default (resolver: Resolver, actor, activity, distribute) => { +import Resolver from '../resolver'; +import DriveFile from '../../../models/drive-file'; +import Post from '../../../models/post'; +import uploadFromUrl from '../../../drive/upload-from-url'; +import createPost from '../../../post/create'; + +export default async (resolver: Resolver, actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { - throw new Error(); + throw new Error('invalid actor'); } - return create(resolver, actor, activity.object, distribute); + const uri = activity.id || activity; + + try { + await Promise.all([ + DriveFile.findOne({ 'metadata.uri': uri }).then(file => { + if (file !== null) { + throw new Error(); + } + }, () => {}), + Post.findOne({ uri }).then(post => { + if (post !== null) { + throw new Error(); + } + }, () => {}) + ]); + } catch (object) { + throw new Error(`already registered: ${uri}`); + } + + const object = await resolver.resolve(activity); + + switch (object.type) { + case 'Image': + createImage(resolver, object); + break; + + case 'Note': + createNote(resolver, object); + break; + } + + /// + + async function createImage(resolver: Resolver, image) { + if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { + throw new Error('invalid image'); + } + + return await uploadFromUrl(image.url, actor); + } + + async function createNote(resolver: Resolver, note) { + if ( + ('attributedTo' in note && actor.account.uri !== note.attributedTo) || + typeof note.id !== 'string' + ) { + throw new Error('invalid note'); + } + + const mediaIds = []; + + if ('attachment' in note) { + note.attachment.forEach(async media => { + const created = await createImage(resolver, media); + mediaIds.push(created._id); + }); + } + + const { window } = new JSDOM(note.content); + + await createPost(actor, { + channelId: undefined, + index: undefined, + createdAt: new Date(note.published), + mediaIds, + replyId: undefined, + repostId: undefined, + poll: undefined, + text: window.document.body.textContent, + textHtml: note.content && createDOMPurify(window).sanitize(note.content), + userId: actor._id, + appId: null, + viaMobile: false, + geo: undefined, + uri: note.id + }, null, null, []); + } }; diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index d282e1288..d78335f16 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -2,35 +2,29 @@ import create from './create'; import performDeleteActivity from './delete'; import follow from './follow'; import undo from './undo'; -import createObject from '../create'; import Resolver from '../resolver'; +import { IObject } from '../type'; -export default async (parentResolver: Resolver, actor, value, distribute?: boolean) => { - const collection = await parentResolver.resolveCollection(value); +export default async (parentResolver: Resolver, actor, activity: IObject): Promise => { + switch (activity.type) { + case 'Create': + await create(parentResolver, actor, activity); + break; - return collection.object.map(async element => { - const { resolver, object } = await collection.resolver.resolveOne(element); - const created = await (await createObject(resolver, actor, [object], distribute))[0]; + case 'Delete': + await performDeleteActivity(parentResolver, actor, activity); + break; - if (created !== null) { - return created; - } + case 'Follow': + await follow(parentResolver, actor, activity); + break; - switch (object.type) { - case 'Create': - return create(resolver, actor, object, distribute); + case 'Undo': + await undo(parentResolver, actor, activity); + break; - case 'Delete': - return performDeleteActivity(resolver, actor, object); - - case 'Follow': - return follow(resolver, actor, object, distribute); - - case 'Undo': - return undo(resolver, actor, object); - - default: - return null; - } - }); + default: + console.warn(`unknown activity type: ${activity.type}`); + return null; + } }; diff --git a/src/remote/activitypub/create.ts b/src/remote/activitypub/create.ts deleted file mode 100644 index 97c72860f..000000000 --- a/src/remote/activitypub/create.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { JSDOM } from 'jsdom'; -import { ObjectID } from 'mongodb'; -import config from '../../config'; -import DriveFile from '../../models/drive-file'; -import Post from '../../models/post'; -import { IRemoteUser } from '../../models/user'; -import uploadFromUrl from '../../drive/upload-from-url'; -import createPost from '../../post/create'; -import distributePost from '../../post/distribute'; -import Resolver from './resolver'; -const createDOMPurify = require('dompurify'); - -type IResult = { - resolver: Resolver; - object: { - $ref: string; - $id: ObjectID; - }; -}; - -class Creator { - private actor: IRemoteUser; - private distribute: boolean; - - constructor(actor, distribute) { - this.actor = actor; - this.distribute = distribute; - } - - private async createImage(resolver: Resolver, image) { - if ('attributedTo' in image && this.actor.account.uri !== image.attributedTo) { - throw new Error(); - } - - const { _id } = await uploadFromUrl(image.url, this.actor, image.id || null); - return { - resolver, - object: { $ref: 'driveFiles.files', $id: _id } - }; - } - - private async createNote(resolver: Resolver, note) { - if ( - ('attributedTo' in note && this.actor.account.uri !== note.attributedTo) || - typeof note.id !== 'string' - ) { - throw new Error(); - } - - const mediaIds = 'attachment' in note && - (await Promise.all(await this.create(resolver, note.attachment))) - .filter(media => media !== null && media.object.$ref === 'driveFiles.files') - .map(({ object }) => object.$id); - - const { window } = new JSDOM(note.content); - - const inserted = await createPost({ - channelId: undefined, - index: undefined, - createdAt: new Date(note.published), - mediaIds, - replyId: undefined, - repostId: undefined, - poll: undefined, - text: window.document.body.textContent, - textHtml: note.content && createDOMPurify(window).sanitize(note.content), - userId: this.actor._id, - appId: null, - viaMobile: false, - geo: undefined, - uri: note.id - }, null, null, []); - - const promises = []; - - if (this.distribute) { - promises.push(distributePost(this.actor, inserted.mentions, inserted)); - } - - // Register to search database - if (note.content && config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - - promises.push(new Promise((resolve, reject) => { - es.index({ - index: 'misskey', - type: 'post', - id: inserted._id.toString(), - body: { - text: window.document.body.textContent - } - }, resolve); - })); - } - - await Promise.all(promises); - - return { - resolver, - object: { $ref: 'posts', id: inserted._id } - }; - } - - public async create(parentResolver: Resolver, value): Promise>> { - const collection = await parentResolver.resolveCollection(value); - - return collection.object.map(async element => { - const uri = element.id || element; - - try { - await Promise.all([ - DriveFile.findOne({ 'metadata.uri': uri }).then(file => { - if (file === null) { - return; - } - - throw { - $ref: 'driveFile.files', - $id: file._id - }; - }, () => {}), - Post.findOne({ uri }).then(post => { - if (post === null) { - return; - } - - throw { - $ref: 'posts', - $id: post._id - }; - }, () => {}) - ]); - } catch (object) { - return { - resolver: collection.resolver, - object - }; - } - - const { resolver, object } = await collection.resolver.resolveOne(element); - - switch (object.type) { - case 'Image': - return this.createImage(resolver, object); - - case 'Note': - return this.createNote(resolver, object); - } - - return null; - }); - } -} - -export default (resolver: Resolver, actor, value, distribute?: boolean) => { - const creator = new Creator(actor, distribute); - return creator.create(resolver, value); -}; diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 371ccdcc3..de0bba268 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -1,20 +1,45 @@ +import { IObject } from "./type"; + const request = require('request-promise-native'); export default class Resolver { - private requesting: Set; + private history: Set; - constructor(iterable?: Iterable) { - this.requesting = new Set(iterable); + constructor() { + this.history = new Set(); } - private async resolveUnrequestedOne(value) { - if (typeof value !== 'string') { - return { resolver: this, object: value }; + public async resolveCollection(value) { + const collection = typeof value === 'string' + ? await this.resolve(value) + : value; + + switch (collection.type) { + case 'Collection': + collection.objects = collection.object.items; + break; + + case 'OrderedCollection': + collection.objects = collection.object.orderedItems; + break; + + default: + throw new Error(`unknown collection type: ${collection.type}`); } - const resolver = new Resolver(this.requesting); + return collection; + } - resolver.requesting.add(value); + public async resolve(value): Promise { + if (typeof value !== 'string') { + return value; + } + + if (this.history.has(value)) { + throw new Error('cannot resolve already resolved one'); + } + + this.history.add(value); const object = await request({ url: value, @@ -29,41 +54,9 @@ export default class Resolver { !object['@context'].includes('https://www.w3.org/ns/activitystreams') : object['@context'] !== 'https://www.w3.org/ns/activitystreams' )) { - throw new Error(); + throw new Error('invalid response'); } - return { resolver, object }; - } - - public async resolveCollection(value) { - const resolved = typeof value === 'string' ? - await this.resolveUnrequestedOne(value) : - { resolver: this, object: value }; - - switch (resolved.object.type) { - case 'Collection': - resolved.object = resolved.object.items; - break; - - case 'OrderedCollection': - resolved.object = resolved.object.orderedItems; - break; - - default: - if (!Array.isArray(value)) { - resolved.object = [resolved.object]; - } - break; - } - - return resolved; - } - - public resolveOne(value) { - if (this.requesting.has(value)) { - throw new Error(); - } - - return this.resolveUnrequestedOne(value); + return object; } } diff --git a/src/server/activitypub/inbox.ts b/src/server/activitypub/inbox.ts index 5de843385..847dc19af 100644 --- a/src/server/activitypub/inbox.ts +++ b/src/server/activitypub/inbox.ts @@ -24,7 +24,7 @@ app.post('/@:user/inbox', bodyParser.json({ queue.create('http', { type: 'processInbox', - inbox: req.body, + activity: req.body, signature, }).save(); From 5bd1451b610c134c056991e05327990180cbb8d5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:22:48 +0900 Subject: [PATCH 02/58] wip --- src/queue/processors/http/deliver.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 8cd9eb624..1700063a5 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -1,17 +1,7 @@ import * as kue from 'kue'; -import Channel from '../../models/channel'; -import Following from '../../models/following'; -import ChannelWatching from '../../models/channel-watching'; -import Post, { pack } from '../../models/post'; -import User, { isLocalUser } from '../../models/user'; -import stream, { publishChannelStream } from '../../publishers/stream'; -import context from '../../remote/activitypub/renderer/context'; -import renderCreate from '../../remote/activitypub/renderer/create'; -import renderNote from '../../remote/activitypub/renderer/note'; -import request from '../../remote/request'; +import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - - request(user, following.follower[0].account.inbox, create); -} + await request(job.data.user, job.data.to, job.data.content); +}; From 77f056b4fcdf74da8b6a8cc4a923eb8789d6f5ae Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:59:38 +0900 Subject: [PATCH 03/58] wip --- src/{ => api}/drive/add-file.ts | 0 src/{ => api}/drive/upload-from-url.ts | 0 src/api/following/create.ts | 82 ++++++++++++++++++++++ src/{ => api}/post/create.ts | 0 src/{ => api}/post/distribute.ts | 0 src/{ => api}/post/watch.ts | 0 src/queue/processors/http/unfollow.ts | 97 ++++++++++++++------------ src/remote/activitypub/act/create.ts | 8 ++- src/remote/activitypub/act/follow.ts | 59 +--------------- 9 files changed, 142 insertions(+), 104 deletions(-) rename src/{ => api}/drive/add-file.ts (100%) rename src/{ => api}/drive/upload-from-url.ts (100%) create mode 100644 src/api/following/create.ts rename src/{ => api}/post/create.ts (100%) rename src/{ => api}/post/distribute.ts (100%) rename src/{ => api}/post/watch.ts (100%) diff --git a/src/drive/add-file.ts b/src/api/drive/add-file.ts similarity index 100% rename from src/drive/add-file.ts rename to src/api/drive/add-file.ts diff --git a/src/drive/upload-from-url.ts b/src/api/drive/upload-from-url.ts similarity index 100% rename from src/drive/upload-from-url.ts rename to src/api/drive/upload-from-url.ts diff --git a/src/api/following/create.ts b/src/api/following/create.ts new file mode 100644 index 000000000..353a6c892 --- /dev/null +++ b/src/api/following/create.ts @@ -0,0 +1,82 @@ +import User, { isLocalUser, isRemoteUser, pack as packUser, IUser } from '../../models/user'; +import Following from '../../models/following'; +import FollowingLog from '../../models/following-log'; +import FollowedLog from '../../models/followed-log'; +import event from '../../publishers/stream'; +import notify from '../../publishers/notify'; +import context from '../../remote/activitypub/renderer/context'; +import renderFollow from '../../remote/activitypub/renderer/follow'; +import renderAccept from '../../remote/activitypub/renderer/accept'; +import { createHttp } from '../../queue'; + +export default async function(follower: IUser, followee: IUser, activity?) { + const following = await Following.insert({ + createdAt: new Date(), + followerId: follower._id, + followeeId: followee._id + }); + + //#region Increment following count + User.update({ _id: follower._id }, { + $inc: { + followingCount: 1 + } + }); + + FollowingLog.insert({ + createdAt: following.createdAt, + userId: follower._id, + count: follower.followingCount + 1 + }); + //#endregion + + //#region Increment followers count + User.update({ _id: followee._id }, { + $inc: { + followersCount: 1 + } + }); + FollowedLog.insert({ + createdAt: following.createdAt, + userId: followee._id, + count: followee.followersCount + 1 + }); + //#endregion + + // Publish follow event + if (isLocalUser(follower)) { + packUser(followee, follower).then(packed => event(follower._id, 'follow', packed)); + } + + // Publish followed event + if (isLocalUser(followee)) { + packUser(follower, followee).then(packed => event(followee._id, 'followed', packed)), + + // 通知を作成 + notify(followee._id, follower._id, 'follow'); + } + + if (isLocalUser(follower) && isRemoteUser(followee)) { + const content = renderFollow(follower, followee); + content['@context'] = context; + + createHttp({ + type: 'deliver', + user: follower, + content, + to: followee.account.inbox + }).save(); + } + + if (isRemoteUser(follower) && isLocalUser(followee)) { + const content = renderAccept(activity); + content['@context'] = context; + + createHttp({ + type: 'deliver', + user: followee, + content, + to: follower.account.inbox + }).save(); + } +} diff --git a/src/post/create.ts b/src/api/post/create.ts similarity index 100% rename from src/post/create.ts rename to src/api/post/create.ts diff --git a/src/post/distribute.ts b/src/api/post/distribute.ts similarity index 100% rename from src/post/distribute.ts rename to src/api/post/distribute.ts diff --git a/src/post/watch.ts b/src/api/post/watch.ts similarity index 100% rename from src/post/watch.ts rename to src/api/post/watch.ts diff --git a/src/queue/processors/http/unfollow.ts b/src/queue/processors/http/unfollow.ts index d3d5f2246..801a3612a 100644 --- a/src/queue/processors/http/unfollow.ts +++ b/src/queue/processors/http/unfollow.ts @@ -1,56 +1,63 @@ -import FollowedLog from '../../models/followed-log'; -import Following from '../../models/following'; -import FollowingLog from '../../models/following-log'; -import User, { isRemoteUser, pack as packUser } from '../../models/user'; -import stream from '../../publishers/stream'; -import renderFollow from '../../remote/activitypub/renderer/follow'; -import renderUndo from '../../remote/activitypub/renderer/undo'; -import context from '../../remote/activitypub/renderer/context'; -import request from '../../remote/request'; +import FollowedLog from '../../../models/followed-log'; +import Following from '../../../models/following'; +import FollowingLog from '../../../models/following-log'; +import User, { isLocalUser, isRemoteUser, pack as packUser } from '../../../models/user'; +import stream from '../../../publishers/stream'; +import renderFollow from '../../../remote/activitypub/renderer/follow'; +import renderUndo from '../../../remote/activitypub/renderer/undo'; +import context from '../../../remote/activitypub/renderer/context'; +import request from '../../../remote/request'; +import Logger from '../../../utils/logger'; export default async ({ data }) => { - // Delete following - const following = await Following.findOneAndDelete({ _id: data.id }); + const following = await Following.findOne({ _id: data.id }); if (following === null) { return; } - const promisedFollower = User.findOne({ _id: following.followerId }); - const promisedFollowee = User.findOne({ _id: following.followeeId }); + const [follower, followee] = await Promise.all([ + User.findOne({ _id: following.followerId }), + User.findOne({ _id: following.followeeId }) + ]); - await Promise.all([ - // Decrement following count - User.update({ _id: following.followerId }, { $inc: { followingCount: -1 } }), - promisedFollower.then(({ followingCount }) => FollowingLog.insert({ - createdAt: new Date(), - userId: following.followerId, - count: followingCount - 1 - })), + if (isLocalUser(follower) && isRemoteUser(followee)) { + const undo = renderUndo(renderFollow(follower, followee)); + undo['@context'] = context; - // Decrement followers count - User.update({ _id: following.followeeId }, { $inc: { followersCount: -1 } }), - promisedFollowee.then(({ followersCount }) => FollowedLog.insert({ - createdAt: new Date(), - userId: following.followeeId, - count: followersCount - 1 - })), + await request(follower, followee.account.inbox, undo); + } + + try { + await Promise.all([ + // Delete following + Following.findOneAndDelete({ _id: data.id }), + + // Decrement following count + User.update({ _id: follower._id }, { $inc: { followingCount: -1 } }), + FollowingLog.insert({ + createdAt: new Date(), + userId: follower._id, + count: follower.followingCount - 1 + }), + + // Decrement followers count + User.update({ _id: followee._id }, { $inc: { followersCount: -1 } }), + FollowedLog.insert({ + createdAt: new Date(), + userId: followee._id, + count: followee.followersCount - 1 + }) + ]); + + if (isLocalUser(follower)) { + return; + } + + const promisedPackedUser = packUser(followee, follower); // Publish follow event - Promise.all([promisedFollower, promisedFollowee]).then(async ([follower, followee]) => { - if (isRemoteUser(follower)) { - return; - } - - const promisedPackedUser = packUser(followee, follower); - - if (isRemoteUser(followee)) { - const undo = renderUndo(renderFollow(follower, followee)); - undo['@context'] = context; - - await request(follower, followee.account.inbox, undo); - } - - stream(follower._id, 'unfollow', promisedPackedUser); - }) - ]); + stream(follower._id, 'unfollow', promisedPackedUser); + } catch (error) { + Logger.error(error.toString()); + } }; diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index c1a30ce7d..7ee9f8dfb 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -4,10 +4,10 @@ const createDOMPurify = require('dompurify'); import Resolver from '../resolver'; import DriveFile from '../../../models/drive-file'; import Post from '../../../models/post'; -import uploadFromUrl from '../../../drive/upload-from-url'; -import createPost from '../../../post/create'; +import uploadFromUrl from '../../../api/drive/upload-from-url'; +import createPost from '../../../api/post/create'; -export default async (resolver: Resolver, actor, activity): Promise => { +export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } @@ -31,6 +31,8 @@ export default async (resolver: Resolver, actor, activity): Promise => { throw new Error(`already registered: ${uri}`); } + const resolver = new Resolver(); + const object = await resolver.resolve(activity); switch (object.type) { diff --git a/src/remote/activitypub/act/follow.ts b/src/remote/activitypub/act/follow.ts index 23fa41df8..dc173a0ac 100644 --- a/src/remote/activitypub/act/follow.ts +++ b/src/remote/activitypub/act/follow.ts @@ -1,15 +1,9 @@ -import { MongoError } from 'mongodb'; import parseAcct from '../../../acct/parse'; -import Following, { IFollowing } from '../../../models/following'; import User from '../../../models/user'; import config from '../../../config'; -import queue from '../../../queue'; -import context from '../renderer/context'; -import renderAccept from '../renderer/accept'; -import request from '../../request'; -import Resolver from '../resolver'; +import follow from '../../../api/following/create'; -export default async (resolver: Resolver, actor, activity, distribute) => { +export default async (actor, activity): Promise => { const prefix = config.url + '/@'; const id = activity.object.id || activity.object; @@ -27,52 +21,5 @@ export default async (resolver: Resolver, actor, activity, distribute) => { throw new Error(); } - if (!distribute) { - const { _id } = await Following.findOne({ - followerId: actor._id, - followeeId: followee._id - }); - - return { - resolver, - object: { $ref: 'following', $id: _id } - }; - } - - const promisedFollowing = Following.insert({ - createdAt: new Date(), - followerId: actor._id, - followeeId: followee._id - }).then(following => new Promise((resolve, reject) => { - queue.create('http', { - type: 'follow', - following: following._id - }).save(error => { - if (error) { - reject(error); - } else { - resolve(following); - } - }); - }) as Promise, async error => { - // duplicate key error - if (error instanceof MongoError && error.code === 11000) { - return Following.findOne({ - followerId: actor._id, - followeeId: followee._id - }); - } - - throw error; - }); - - const accept = renderAccept(activity); - accept['@context'] = context; - - await request(followee, actor.account.inbox, accept); - - return promisedFollowing.then(({ _id }) => ({ - resolver, - object: { $ref: 'following', $id: _id } - })); + await follow(actor, followee, activity); }; From 0ac695edbda10231ee70f0a0debedb4c8f708954 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 00:01:01 +0900 Subject: [PATCH 04/58] wip --- src/queue/processors/http/follow.ts | 69 ----------------------------- 1 file changed, 69 deletions(-) delete mode 100644 src/queue/processors/http/follow.ts diff --git a/src/queue/processors/http/follow.ts b/src/queue/processors/http/follow.ts deleted file mode 100644 index 8bf890efb..000000000 --- a/src/queue/processors/http/follow.ts +++ /dev/null @@ -1,69 +0,0 @@ -import User, { isLocalUser, pack as packUser } from '../../models/user'; -import Following from '../../models/following'; -import FollowingLog from '../../models/following-log'; -import FollowedLog from '../../models/followed-log'; -import event from '../../publishers/stream'; -import notify from '../../publishers/notify'; -import context from '../../remote/activitypub/renderer/context'; -import render from '../../remote/activitypub/renderer/follow'; -import request from '../../remote/request'; - -export default ({ data }) => Following.findOne({ _id: data.following }).then(({ followerId, followeeId }) => { - const promisedFollower = User.findOne({ _id: followerId }); - const promisedFollowee = User.findOne({ _id: followeeId }); - - return Promise.all([ - // Increment following count - User.update(followerId, { - $inc: { - followingCount: 1 - } - }), - - promisedFollower.then(({ followingCount }) => FollowingLog.insert({ - createdAt: data.following.createdAt, - userId: followerId, - count: followingCount + 1 - })), - - // Increment followers count - User.update({ _id: followeeId }, { - $inc: { - followersCount: 1 - } - }), - - promisedFollowee.then(({ followersCount }) => FollowedLog.insert({ - createdAt: data.following.createdAt, - userId: followerId, - count: followersCount + 1 - })), - - // Notify - promisedFollowee.then(followee => followee.host === null ? - notify(followeeId, followerId, 'follow') : null), - - // Publish follow event - Promise.all([promisedFollower, promisedFollowee]).then(([follower, followee]) => { - let followerEvent; - let followeeEvent; - - if (isLocalUser(follower)) { - followerEvent = packUser(followee, follower) - .then(packed => event(follower._id, 'follow', packed)); - } - - if (isLocalUser(followee)) { - followeeEvent = packUser(follower, followee) - .then(packed => event(followee._id, 'followed', packed)); - } else if (isLocalUser(follower)) { - const rendered = render(follower, followee); - rendered['@context'] = context; - - followeeEvent = request(follower, followee.account.inbox, rendered); - } - - return Promise.all([followerEvent, followeeEvent]); - }) - ]); -}); From 0009a371c86097fd1be117c1eb7e9fa2fcce7400 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 00:40:34 +0900 Subject: [PATCH 05/58] wip --- src/api/post/create.ts | 290 ++++++++++++++++++++++++++++++------- src/api/post/distribute.ts | 190 ------------------------ 2 files changed, 235 insertions(+), 245 deletions(-) delete mode 100644 src/api/post/distribute.ts diff --git a/src/api/post/create.ts b/src/api/post/create.ts index f78bbe752..af94c6d81 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -1,76 +1,90 @@ -import parseAcct from '../acct/parse'; -import Post, { pack } from '../models/post'; -import User, { isLocalUser, isRemoteUser, IUser } from '../models/user'; -import stream from '../publishers/stream'; -import Following from '../models/following'; -import { createHttp } from '../queue'; -import renderNote from '../remote/activitypub/renderer/note'; -import renderCreate from '../remote/activitypub/renderer/create'; -import context from '../remote/activitypub/renderer/context'; +import parseAcct from '../../acct/parse'; +import Post, { pack, IPost } from '../../models/post'; +import User, { isLocalUser, isRemoteUser, IUser } from '../../models/user'; +import stream from '../../publishers/stream'; +import Following from '../../models/following'; +import { createHttp } from '../../queue'; +import renderNote from '../../remote/activitypub/renderer/note'; +import renderCreate from '../../remote/activitypub/renderer/create'; +import context from '../../remote/activitypub/renderer/context'; +import { IDriveFile } from '../../models/drive-file'; +import notify from '../../publishers/notify'; +import PostWatching from '../../models/post-watching'; +import watch from './watch'; +import Mute from '../../models/mute'; +import pushSw from '../../publishers/push-sw'; +import event from '../../publishers/stream'; +import parse from '../../text/parse'; -export default async (user: IUser, post, reply, repost, atMentions) => { - post.mentions = []; +export default async (user: IUser, content: { + createdAt: Date; + text: string; + reply: IPost; + repost: IPost; + media: IDriveFile[]; + geo: any; + viaMobile: boolean; + tags: string[]; +}) => new Promise(async (res, rej) => { + const tags = content.tags || []; - function addMention(mentionee) { - // Reject if already added - if (post.mentions.some(x => x.equals(mentionee))) return; + let tokens = null; - // Add mention - post.mentions.push(mentionee); + if (content.text) { + // Analyze + tokens = parse(content.text); + + // Extract hashtags + const hashtags = tokens + .filter(t => t.type == 'hashtag') + .map(t => t.hashtag); + + hashtags.forEach(tag => { + if (tags.indexOf(tag) == -1) { + tags.push(tag); + } + }); } - if (reply) { - // Add mention - addMention(reply.userId); - post.replyId = reply._id; - post._reply = { userId: reply.userId }; - } else { - post.replyId = null; - post._reply = null; - } + // 投稿を作成 + const post = await Post.insert({ + createdAt: content.createdAt, + mediaIds: content.media ? content.media.map(file => file._id) : [], + replyId: content.reply ? content.reply._id : null, + repostId: content.repost ? content.repost._id : null, + text: content.text, + tags, + userId: user._id, + viaMobile: content.viaMobile, + geo: content.geo || null, - if (repost) { - if (post.text) { - // Add mention - addMention(repost.userId); - } + // 以下非正規化データ + _reply: content.reply ? { userId: content.reply.userId } : null, + _repost: content.repost ? { userId: content.repost.userId } : null, + }); - post.repostId = repost._id; - post._repost = { userId: repost.userId }; - } else { - post.repostId = null; - post._repost = null; - } - - await Promise.all(atMentions.map(async mention => { - // Fetch mentioned user - // SELECT _id - const { _id } = await User - .findOne(parseAcct(mention), { _id: true }); - - // Add mention - addMention(_id); - })); - - const inserted = await Post.insert(post); + res(post); User.update({ _id: user._id }, { - // Increment my posts count + // Increment posts count $inc: { postsCount: 1 }, - + // Update latest post $set: { - latestPost: post._id + latestPost: post } }); - const postObj = await pack(inserted); + // Serialize + const postObj = await pack(post); // タイムラインへの投稿 if (!post.channelId) { // Publish event to myself's stream - stream(post.userId, 'post', postObj); + if (isLocalUser(user)) { + stream(post.userId, 'post', postObj); + } // Fetch all followers const followers = await Following.aggregate([{ @@ -144,6 +158,172 @@ export default async (user: IUser, post, reply, repost, atMentions) => { ); }*/ - return Promise.all(promises); + const mentions = []; -}; + async function addMention(mentionee, reason) { + // Reject if already added + if (mentions.some(x => x.equals(mentionee))) return; + + // Add mention + mentions.push(mentionee); + + // Publish event + if (!user._id.equals(mentionee)) { + const mentioneeMutes = await Mute.find({ + muter_id: mentionee, + deleted_at: { $exists: false } + }); + const mentioneesMutedUserIds = mentioneeMutes.map(m => m.muteeId.toString()); + if (mentioneesMutedUserIds.indexOf(user._id.toString()) == -1) { + event(mentionee, reason, postObj); + pushSw(mentionee, reason, postObj); + } + } + } + + // If has in reply to post + if (content.reply) { + // Increment replies count + Post.update({ _id: content.reply._id }, { + $inc: { + repliesCount: 1 + } + }); + + // (自分自身へのリプライでない限りは)通知を作成 + notify(content.reply.userId, user._id, 'reply', { + postId: post._id + }); + + // Fetch watchers + PostWatching.find({ + postId: content.reply._id, + userId: { $ne: user._id }, + // 削除されたドキュメントは除く + deletedAt: { $exists: false } + }, { + fields: { + userId: true + } + }).then(watchers => { + watchers.forEach(watcher => { + notify(watcher.userId, user._id, 'reply', { + postId: post._id + }); + }); + }); + + // この投稿をWatchする + if (isLocalUser(user) && user.account.settings.autoWatch !== false) { + watch(user._id, content.reply); + } + + // Add mention + addMention(content.reply.userId, 'reply'); + } + + // If it is repost + if (content.repost) { + // Notify + const type = content.text ? 'quote' : 'repost'; + notify(content.repost.userId, user._id, type, { + post_id: post._id + }); + + // Fetch watchers + PostWatching.find({ + postId: content.repost._id, + userId: { $ne: user._id }, + // 削除されたドキュメントは除く + deletedAt: { $exists: false } + }, { + fields: { + userId: true + } + }).then(watchers => { + watchers.forEach(watcher => { + notify(watcher.userId, user._id, type, { + postId: post._id + }); + }); + }); + + // この投稿をWatchする + if (isLocalUser(user) && user.account.settings.autoWatch !== false) { + watch(user._id, content.repost); + } + + // If it is quote repost + if (content.text) { + // Add mention + addMention(content.repost.userId, 'quote'); + } else { + // Publish event + if (!user._id.equals(content.repost.userId)) { + event(content.repost.userId, 'repost', postObj); + } + } + + // 今までで同じ投稿をRepostしているか + const existRepost = await Post.findOne({ + userId: user._id, + repostId: content.repost._id, + _id: { + $ne: post._id + } + }); + + if (!existRepost) { + // Update repostee status + Post.update({ _id: content.repost._id }, { + $inc: { + repostCount: 1 + } + }); + } + } + + // If has text content + if (content.text) { + // Extract an '@' mentions + const atMentions = tokens + .filter(t => t.type == 'mention') + .map(m => m.username) + // Drop dupulicates + .filter((v, i, s) => s.indexOf(v) == i); + + // Resolve all mentions + await Promise.all(atMentions.map(async mention => { + // Fetch mentioned user + // SELECT _id + const mentionee = await User + .findOne({ + usernameLower: mention.toLowerCase() + }, { _id: true }); + + // When mentioned user not found + if (mentionee == null) return; + + // 既に言及されたユーザーに対する返信や引用repostの場合も無視 + if (content.reply && content.reply.userId.equals(mentionee._id)) return; + if (content.repost && content.repost.userId.equals(mentionee._id)) return; + + // Add mention + addMention(mentionee._id, 'mention'); + + // Create notification + notify(mentionee._id, user._id, 'mention', { + post_id: post._id + }); + })); + } + + // Append mentions data + if (mentions.length > 0) { + Post.update({ _id: post._id }, { + $set: { + mentions + } + }); + } +}); diff --git a/src/api/post/distribute.ts b/src/api/post/distribute.ts deleted file mode 100644 index 49c6eb22d..000000000 --- a/src/api/post/distribute.ts +++ /dev/null @@ -1,190 +0,0 @@ -import Mute from '../models/mute'; -import Post, { pack } from '../models/post'; -import Watching from '../models/post-watching'; -import User from '../models/user'; -import stream from '../publishers/stream'; -import notify from '../publishers/notify'; -import pushSw from '../publishers/push-sw'; -import queue from '../queue'; -import watch from './watch'; - -export default async (user, mentions, post) => { - const promisedPostObj = pack(post); - const promises = [ - User.update({ _id: user._id }, { - // Increment my posts count - $inc: { - postsCount: 1 - }, - - $set: { - latestPost: post._id - } - }), - new Promise((resolve, reject) => queue.create('http', { - type: 'deliverPost', - id: post._id, - }).save(error => error ? reject(error) : resolve())), - ] as Array>; - - function addMention(promisedMentionee, reason) { - // Publish event - promises.push(promisedMentionee.then(mentionee => { - if (user._id.equals(mentionee)) { - return Promise.resolve(); - } - - return Promise.all([ - promisedPostObj, - Mute.find({ - muterId: mentionee, - deletedAt: { $exists: false } - }) - ]).then(([postObj, mentioneeMutes]) => { - const mentioneesMutedUserIds = mentioneeMutes.map(m => m.muteeId.toString()); - if (mentioneesMutedUserIds.indexOf(user._id.toString()) == -1) { - stream(mentionee, reason, postObj); - pushSw(mentionee, reason, postObj); - } - }); - })); - } - - // If has in reply to post - if (post.replyId) { - promises.push( - // Increment replies count - Post.update({ _id: post.replyId }, { - $inc: { - repliesCount: 1 - } - }), - - // 自分自身へのリプライでない限りは通知を作成 - promisedPostObj.then(({ reply }) => { - return notify(reply.userId, user._id, 'reply', { - postId: post._id - }); - }), - - // Fetch watchers - Watching - .find({ - postId: post.replyId, - userId: { $ne: user._id }, - // 削除されたドキュメントは除く - deletedAt: { $exists: false } - }, { - fields: { - userId: true - } - }) - .then(watchers => { - watchers.forEach(watcher => { - notify(watcher.userId, user._id, 'reply', { - postId: post._id - }); - }); - }) - ); - - // Add mention - addMention(promisedPostObj.then(({ reply }) => reply.userId), 'reply'); - - // この投稿をWatchする - if (user.account.settings.autoWatch !== false) { - promises.push(promisedPostObj.then(({ reply }) => { - return watch(user._id, reply); - })); - } - } - - // If it is repost - if (post.repostId) { - const type = post.text ? 'quote' : 'repost'; - - promises.push( - promisedPostObj.then(({ repost }) => Promise.all([ - // Notify - notify(repost.userId, user._id, type, { - postId: post._id - }), - - // この投稿をWatchする - // TODO: ユーザーが「Repostしたときに自動でWatchする」設定を - // オフにしていた場合はしない - watch(user._id, repost) - ])), - - // Fetch watchers - Watching - .find({ - postId: post.repostId, - userId: { $ne: user._id }, - // 削除されたドキュメントは除く - deletedAt: { $exists: false } - }, { - fields: { - userId: true - } - }) - .then(watchers => { - watchers.forEach(watcher => { - notify(watcher.userId, user._id, type, { - postId: post._id - }); - }); - }) - ); - - // If it is quote repost - if (post.text) { - // Add mention - addMention(promisedPostObj.then(({ repost }) => repost.userId), 'quote'); - } else { - promises.push(promisedPostObj.then(postObj => { - // Publish event - if (!user._id.equals(postObj.repost.userId)) { - stream(postObj.repost.userId, 'repost', postObj); - } - })); - } - - // 今までで同じ投稿をRepostしているか - const existRepost = await Post.findOne({ - userId: user._id, - repostId: post.repostId, - _id: { - $ne: post._id - } - }); - - if (!existRepost) { - // Update repostee status - promises.push(Post.update({ _id: post.repostId }, { - $inc: { - repostCount: 1 - } - })); - } - } - - // Resolve all mentions - await promisedPostObj.then(({ reply, repost }) => Promise.all(mentions.map(async mention => { - // 既に言及されたユーザーに対する返信や引用repostの場合も無視 - if (reply && reply.userId.equals(mention)) return; - if (repost && repost.userId.equals(mention)) return; - - // Add mention - addMention(mention, 'mention'); - - // Create notification - await notify(mention, user._id, 'mention', { - postId: post._id - }); - }))); - - await Promise.all(promises); - - return promisedPostObj; -}; From fd73fad148a500bd95a575fe6e4b73a25882fb89 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 00:50:57 +0900 Subject: [PATCH 06/58] wip --- src/api/post/create.ts | 15 ++++- src/remote/activitypub/act/create.ts | 18 ++---- src/server/api/endpoints/posts/create.ts | 82 ++++-------------------- 3 files changed, 31 insertions(+), 84 deletions(-) diff --git a/src/api/post/create.ts b/src/api/post/create.ts index af94c6d81..8256cbc35 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -15,6 +15,8 @@ import Mute from '../../models/mute'; import pushSw from '../../publishers/push-sw'; import event from '../../publishers/stream'; import parse from '../../text/parse'; +import html from '../../text/html'; +import { IApp } from '../../models/app'; export default async (user: IUser, content: { createdAt: Date; @@ -23,9 +25,14 @@ export default async (user: IUser, content: { repost: IPost; media: IDriveFile[]; geo: any; + poll: any; viaMobile: boolean; tags: string[]; -}) => new Promise(async (res, rej) => { + cw: string; + visibility: string; + uri?: string; + app?: IApp; +}) => new Promise(async (res, rej) => { const tags = content.tags || []; let tokens = null; @@ -53,10 +60,16 @@ export default async (user: IUser, content: { replyId: content.reply ? content.reply._id : null, repostId: content.repost ? content.repost._id : null, text: content.text, + textHtml: tokens === null ? null : html(tokens), + poll: content.poll, + cw: content.cw, tags, userId: user._id, viaMobile: content.viaMobile, geo: content.geo || null, + uri: content.uri, + appId: content.app ? content.app._id : null, + visibility: content.visibility, // 以下非正規化データ _reply: content.reply ? { userId: content.reply.userId } : null, diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 7ee9f8dfb..957900900 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -63,32 +63,26 @@ export default async (actor, activity): Promise => { throw new Error('invalid note'); } - const mediaIds = []; + const media = []; if ('attachment' in note) { note.attachment.forEach(async media => { const created = await createImage(resolver, media); - mediaIds.push(created._id); + media.push(created); }); } const { window } = new JSDOM(note.content); await createPost(actor, { - channelId: undefined, - index: undefined, createdAt: new Date(note.published), - mediaIds, - replyId: undefined, - repostId: undefined, - poll: undefined, + media, + reply: undefined, + repost: undefined, text: window.document.body.textContent, - textHtml: note.content && createDOMPurify(window).sanitize(note.content), - userId: actor._id, - appId: null, viaMobile: false, geo: undefined, uri: note.id - }, null, null, []); + }); } }; diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index 03af7ee76..d241c8c38 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -3,16 +3,12 @@ */ import $ from 'cafy'; import deepEqual = require('deep-equal'); -import renderAcct from '../../../../acct/render'; -import config from '../../../../config'; -import html from '../../../../text/html'; -import parse from '../../../../text/parse'; -import Post, { IPost, isValidText, isValidCw } from '../../../../models/post'; +import Post, { IPost, isValidText, isValidCw, pack } from '../../../../models/post'; import { ILocalUser } from '../../../../models/user'; import Channel, { IChannel } from '../../../../models/channel'; import DriveFile from '../../../../models/drive-file'; -import create from '../../../../post/create'; -import distribute from '../../../../post/distribute'; +import create from '../../../../api/post/create'; +import { IApp } from '../../../../models/app'; /** * Create a post @@ -22,7 +18,7 @@ import distribute from '../../../../post/distribute'; * @param {any} app * @return {Promise} */ -module.exports = (params, user: ILocalUser, app) => new Promise(async (res, rej) => { +module.exports = (params, user: ILocalUser, app: IApp) => new Promise(async (res, rej) => { // Get 'visibility' parameter const [visibility = 'public', visibilityErr] = $(params.visibility).optional.string().or(['public', 'unlisted', 'private', 'direct']).$; if (visibilityErr) return rej('invalid visibility'); @@ -230,82 +226,26 @@ module.exports = (params, user: ILocalUser, app) => new Promise(async (res, rej) } } - let tokens = null; - if (text) { - // Analyze - tokens = parse(text); - - // Extract hashtags - const hashtags = tokens - .filter(t => t.type == 'hashtag') - .map(t => t.hashtag); - - hashtags.forEach(tag => { - if (tags.indexOf(tag) == -1) { - tags.push(tag); - } - }); - } - - let atMentions = []; - - // If has text content - if (text) { - /* - // Extract a hashtags - const hashtags = tokens - .filter(t => t.type == 'hashtag') - .map(t => t.hashtag) - // Drop dupulicates - .filter((v, i, s) => s.indexOf(v) == i); - - // ハッシュタグをデータベースに登録 - registerHashtags(user, hashtags); - */ - // Extract an '@' mentions - atMentions = tokens - .filter(t => t.type == 'mention') - .map(renderAcct) - // Drop dupulicates - .filter((v, i, s) => s.indexOf(v) == i); - } - // 投稿を作成 - const post = await create({ + const post = await create(user, { createdAt: new Date(), - channelId: channel ? channel._id : undefined, - index: channel ? channel.index + 1 : undefined, - mediaIds: files ? files.map(file => file._id) : [], + media: files, poll: poll, text: text, - textHtml: tokens === null ? null : html(tokens), + reply, + repost, cw: cw, tags: tags, - userId: user._id, - appId: app ? app._id : null, + app: app, viaMobile: viaMobile, visibility, geo - }, reply, repost, atMentions); + }); - const postObj = await distribute(user, post.mentions, post); + const postObj = await pack(post, user); // Reponse res({ createdPost: postObj }); - - // Register to search database - if (post.text && config.elasticsearch.enable) { - const es = require('../../../db/elasticsearch'); - - es.index({ - index: 'misskey', - type: 'post', - id: post._id.toString(), - body: { - text: post.text - } - }); - } }); From fba46b4c7f5751cd31463e922edb19f5652ca408 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 01:07:07 +0900 Subject: [PATCH 07/58] wip --- src/queue/processors/http/index.ts | 10 +--- .../processors/http/perform-activitypub.ts | 7 --- src/remote/activitypub/resolve-person.ts | 58 +++++-------------- 3 files changed, 15 insertions(+), 60 deletions(-) delete mode 100644 src/queue/processors/http/perform-activitypub.ts diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 8f9aa717c..06c6b1d1a 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -1,17 +1,11 @@ -import deliverPost from './deliver-post'; -import follow from './follow'; -import performActivityPub from './perform-activitypub'; +import deliver from './deliver'; import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; -import unfollow from './unfollow'; const handlers = { - deliverPost, - follow, - performActivityPub, + deliver, processInbox, reportGitHubFailure, - unfollow }; export default (job, done) => handlers[job.data.type](job).then(() => done(), done); diff --git a/src/queue/processors/http/perform-activitypub.ts b/src/queue/processors/http/perform-activitypub.ts deleted file mode 100644 index 963e532fe..000000000 --- a/src/queue/processors/http/perform-activitypub.ts +++ /dev/null @@ -1,7 +0,0 @@ -import User from '../../models/user'; -import act from '../../remote/activitypub/act'; -import Resolver from '../../remote/activitypub/resolver'; - -export default ({ data }) => User.findOne({ _id: data.actor }) - .then(actor => act(new Resolver(), actor, data.outbox)) - .then(Promise.all); diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 59be65908..77d08398b 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -1,17 +1,15 @@ import { JSDOM } from 'jsdom'; import { toUnicode } from 'punycode'; import User, { validateUsername, isValidName, isValidDescription } from '../../models/user'; -import queue from '../../queue'; import webFinger from '../webfinger'; import create from './create'; import Resolver from './resolver'; - -async function isCollection(collection) { - return ['Collection', 'OrderedCollection'].includes(collection.type); -} +import uploadFromUrl from '../../api/drive/upload-from-url'; export default async (value, verifier?: string) => { - const { resolver, object } = await new Resolver().resolveOne(value); + const resolver = new Resolver(); + + const object = await resolver.resolve(value) as any; if ( object === null || @@ -21,24 +19,10 @@ export default async (value, verifier?: string) => { !isValidName(object.name) || !isValidDescription(object.summary) ) { - throw new Error(); + throw new Error('invalid person'); } - const [followers, following, outbox, finger] = await Promise.all([ - resolver.resolveOne(object.followers).then( - resolved => isCollection(resolved.object) ? resolved.object : null, - () => null - ), - resolver.resolveOne(object.following).then( - resolved => isCollection(resolved.object) ? resolved.object : null, - () => null - ), - resolver.resolveOne(object.outbox).then( - resolved => isCollection(resolved.object) ? resolved.object : null, - () => null - ), - webFinger(object.id, verifier), - ]); + const finger = await webFinger(object.id, verifier); const host = toUnicode(finger.subject.replace(/^.*?@/, '')); const hostLower = host.replace(/[A-Z]+/, matched => matched.toLowerCase()); @@ -50,10 +34,10 @@ export default async (value, verifier?: string) => { bannerId: null, createdAt: Date.parse(object.published), description: summaryDOM.textContent, - followersCount: followers ? followers.totalItem || 0 : 0, - followingCount: following ? following.totalItem || 0 : 0, + followersCount: 0, + followingCount: 0, name: object.name, - postsCount: outbox ? outbox.totalItem || 0 : 0, + postsCount: 0, driveCapacity: 1024 * 1024 * 8, // 8MiB username: object.preferredUsername, usernameLower: object.preferredUsername.toLowerCase(), @@ -69,33 +53,17 @@ export default async (value, verifier?: string) => { }, }); - queue.create('http', { - type: 'performActivityPub', - actor: user._id, - outbox - }).save(); - const [avatarId, bannerId] = await Promise.all([ object.icon, object.image - ].map(async value => { - if (value === undefined) { + ].map(async url => { + if (url === undefined) { return null; } - try { - const created = await create(resolver, user, value); + const img = await uploadFromUrl(url, user); - await Promise.all(created.map(asyncCreated => asyncCreated.then(created => { - if (created !== null && created.object.$ref === 'driveFiles.files') { - throw created.object.$id; - } - }, () => {}))); - - return null; - } catch (id) { - return id; - } + return img._id; })); User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); From eb304cb5fb588a3da8742f234cdf05ce6deeaa59 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 01:22:41 +0900 Subject: [PATCH 08/58] wip --- src/api/following/delete.ts | 69 +++++++++++++++++++++ src/queue/processors/http/unfollow.ts | 63 ------------------- src/remote/activitypub/act/create.ts | 1 - src/remote/activitypub/act/index.ts | 11 ++-- src/remote/activitypub/act/undo.ts | 15 +++++ src/remote/activitypub/act/undo/index.ts | 27 -------- src/remote/activitypub/act/undo/unfollow.ts | 11 ---- src/remote/activitypub/act/unfollow.ts | 25 ++++++++ 8 files changed, 114 insertions(+), 108 deletions(-) create mode 100644 src/api/following/delete.ts delete mode 100644 src/queue/processors/http/unfollow.ts create mode 100644 src/remote/activitypub/act/undo.ts delete mode 100644 src/remote/activitypub/act/undo/index.ts delete mode 100644 src/remote/activitypub/act/undo/unfollow.ts create mode 100644 src/remote/activitypub/act/unfollow.ts diff --git a/src/api/following/delete.ts b/src/api/following/delete.ts new file mode 100644 index 000000000..4cdff7ce1 --- /dev/null +++ b/src/api/following/delete.ts @@ -0,0 +1,69 @@ +import User, { isLocalUser, isRemoteUser, pack as packUser, IUser } from '../../models/user'; +import Following from '../../models/following'; +import FollowingLog from '../../models/following-log'; +import FollowedLog from '../../models/followed-log'; +import event from '../../publishers/stream'; +import context from '../../remote/activitypub/renderer/context'; +import renderFollow from '../../remote/activitypub/renderer/follow'; +import renderUndo from '../../remote/activitypub/renderer/undo'; +import { createHttp } from '../../queue'; + +export default async function(follower: IUser, followee: IUser, activity?) { + const following = await Following.findOne({ + followerId: follower._id, + followeeId: followee._id + }); + + if (following == null) { + console.warn('フォロー解除がリクエストされましたがフォローしていませんでした'); + return; + } + + Following.remove({ + _id: following._id + }); + + //#region Decrement following count + User.update({ _id: follower._id }, { + $inc: { + followingCount: -1 + } + }); + + FollowingLog.insert({ + createdAt: following.createdAt, + userId: follower._id, + count: follower.followingCount - 1 + }); + //#endregion + + //#region Decrement followers count + User.update({ _id: followee._id }, { + $inc: { + followersCount: -1 + } + }); + FollowedLog.insert({ + createdAt: following.createdAt, + userId: followee._id, + count: followee.followersCount - 1 + }); + //#endregion + + // Publish unfollow event + if (isLocalUser(follower)) { + packUser(followee, follower).then(packed => event(follower._id, 'unfollow', packed)); + } + + if (isLocalUser(follower) && isRemoteUser(followee)) { + const content = renderUndo(renderFollow(follower, followee)); + content['@context'] = context; + + createHttp({ + type: 'deliver', + user: follower, + content, + to: followee.account.inbox + }).save(); + } +} diff --git a/src/queue/processors/http/unfollow.ts b/src/queue/processors/http/unfollow.ts deleted file mode 100644 index 801a3612a..000000000 --- a/src/queue/processors/http/unfollow.ts +++ /dev/null @@ -1,63 +0,0 @@ -import FollowedLog from '../../../models/followed-log'; -import Following from '../../../models/following'; -import FollowingLog from '../../../models/following-log'; -import User, { isLocalUser, isRemoteUser, pack as packUser } from '../../../models/user'; -import stream from '../../../publishers/stream'; -import renderFollow from '../../../remote/activitypub/renderer/follow'; -import renderUndo from '../../../remote/activitypub/renderer/undo'; -import context from '../../../remote/activitypub/renderer/context'; -import request from '../../../remote/request'; -import Logger from '../../../utils/logger'; - -export default async ({ data }) => { - const following = await Following.findOne({ _id: data.id }); - if (following === null) { - return; - } - - const [follower, followee] = await Promise.all([ - User.findOne({ _id: following.followerId }), - User.findOne({ _id: following.followeeId }) - ]); - - if (isLocalUser(follower) && isRemoteUser(followee)) { - const undo = renderUndo(renderFollow(follower, followee)); - undo['@context'] = context; - - await request(follower, followee.account.inbox, undo); - } - - try { - await Promise.all([ - // Delete following - Following.findOneAndDelete({ _id: data.id }), - - // Decrement following count - User.update({ _id: follower._id }, { $inc: { followingCount: -1 } }), - FollowingLog.insert({ - createdAt: new Date(), - userId: follower._id, - count: follower.followingCount - 1 - }), - - // Decrement followers count - User.update({ _id: followee._id }, { $inc: { followersCount: -1 } }), - FollowedLog.insert({ - createdAt: new Date(), - userId: followee._id, - count: followee.followersCount - 1 - }) - ]); - - if (isLocalUser(follower)) { - return; - } - - const promisedPackedUser = packUser(followee, follower); - - // Publish follow event - stream(follower._id, 'unfollow', promisedPackedUser); - } catch (error) { - Logger.error(error.toString()); - } -}; diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 957900900..c486571fc 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -1,5 +1,4 @@ import { JSDOM } from 'jsdom'; -const createDOMPurify = require('dompurify'); import Resolver from '../resolver'; import DriveFile from '../../../models/drive-file'; diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index d78335f16..f22500ace 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -2,25 +2,24 @@ import create from './create'; import performDeleteActivity from './delete'; import follow from './follow'; import undo from './undo'; -import Resolver from '../resolver'; import { IObject } from '../type'; -export default async (parentResolver: Resolver, actor, activity: IObject): Promise => { +export default async (actor, activity: IObject): Promise => { switch (activity.type) { case 'Create': - await create(parentResolver, actor, activity); + await create(actor, activity); break; case 'Delete': - await performDeleteActivity(parentResolver, actor, activity); + await performDeleteActivity(actor, activity); break; case 'Follow': - await follow(parentResolver, actor, activity); + await follow(actor, activity); break; case 'Undo': - await undo(parentResolver, actor, activity); + await undo(actor, activity); break; default: diff --git a/src/remote/activitypub/act/undo.ts b/src/remote/activitypub/act/undo.ts new file mode 100644 index 000000000..b3b83777d --- /dev/null +++ b/src/remote/activitypub/act/undo.ts @@ -0,0 +1,15 @@ +import unfollow from './unfollow'; + +export default async (actor, activity): Promise => { + if ('actor' in activity && actor.account.uri !== activity.actor) { + throw new Error('invalid actor'); + } + + switch (activity.object.type) { + case 'Follow': + unfollow(activity.object); + break; + } + + return null; +}; diff --git a/src/remote/activitypub/act/undo/index.ts b/src/remote/activitypub/act/undo/index.ts deleted file mode 100644 index aa60d3a4f..000000000 --- a/src/remote/activitypub/act/undo/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import act from '../../act'; -import deleteObject from '../../delete'; -import unfollow from './unfollow'; -import Resolver from '../../resolver'; - -export default async (resolver: Resolver, actor, activity): Promise => { - if ('actor' in activity && actor.account.uri !== activity.actor) { - throw new Error(); - } - - const results = await act(resolver, actor, activity.object); - - await Promise.all(results.map(async promisedResult => { - const result = await promisedResult; - - if (result === null || await deleteObject(result) !== null) { - return; - } - - switch (result.object.$ref) { - case 'following': - await unfollow(result.object); - } - })); - - return null; -}; diff --git a/src/remote/activitypub/act/undo/unfollow.ts b/src/remote/activitypub/act/undo/unfollow.ts deleted file mode 100644 index c17e06e8a..000000000 --- a/src/remote/activitypub/act/undo/unfollow.ts +++ /dev/null @@ -1,11 +0,0 @@ -import queue from '../../../../queue'; - -export default ({ $id }) => new Promise((resolve, reject) => { - queue.create('http', { type: 'unfollow', id: $id }).save(error => { - if (error) { - reject(error); - } else { - resolve(); - } - }); -}); diff --git a/src/remote/activitypub/act/unfollow.ts b/src/remote/activitypub/act/unfollow.ts new file mode 100644 index 000000000..e3c9e1c1c --- /dev/null +++ b/src/remote/activitypub/act/unfollow.ts @@ -0,0 +1,25 @@ +import parseAcct from '../../../acct/parse'; +import User from '../../../models/user'; +import config from '../../../config'; +import unfollow from '../../../api/following/delete'; + +export default async (actor, activity): Promise => { + const prefix = config.url + '/@'; + const id = activity.object.id || activity.object; + + if (!id.startsWith(prefix)) { + return null; + } + + const { username, host } = parseAcct(id.slice(prefix.length)); + if (host !== null) { + throw new Error(); + } + + const followee = await User.findOne({ username, host }); + if (followee === null) { + throw new Error(); + } + + await unfollow(actor, followee, activity); +}; From b6b98752053042b3b37d5b94bc823635edc2061f Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 01:24:01 +0900 Subject: [PATCH 09/58] wip --- src/queue/processors/http/process-inbox.ts | 12 ++++++------ src/queue/processors/index.ts | 18 ------------------ 2 files changed, 6 insertions(+), 24 deletions(-) delete mode 100644 src/queue/processors/index.ts diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index fff1fbf66..82585d3a6 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -1,11 +1,11 @@ import * as kue from 'kue'; import { verifySignature } from 'http-signature'; -import parseAcct from '../../acct/parse'; -import User, { IRemoteUser } from '../../models/user'; -import act from '../../remote/activitypub/act'; -import resolvePerson from '../../remote/activitypub/resolve-person'; -import Resolver from '../../remote/activitypub/resolver'; +import parseAcct from '../../../acct/parse'; +import User, { IRemoteUser } from '../../../models/user'; +import act from '../../../remote/activitypub/act'; +import resolvePerson from '../../../remote/activitypub/resolve-person'; +import Resolver from '../../../remote/activitypub/resolver'; // ユーザーのinboxにアクティビティが届いた時の処理 export default async (job: kue.Job, done): Promise => { @@ -47,7 +47,7 @@ export default async (job: kue.Job, done): Promise => { // アクティビティを処理 try { - await act(new Resolver(), user, activity); + await act(user, activity); done(); } catch (e) { done(e); diff --git a/src/queue/processors/index.ts b/src/queue/processors/index.ts deleted file mode 100644 index 172048dda..000000000 --- a/src/queue/processors/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -import queue from '../queue'; -import db from './db'; -import http from './http'; - -export default () => { - queue.process('db', db); - - /* - 256 is the default concurrency limit of Mozilla Firefox and Google - Chromium. - - a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google - https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff - Network.http.max-connections - MozillaZine Knowledge Base - http://kb.mozillazine.org/Network.http.max-connections - */ - queue.process('http', 256, http); -}; From c2c03a1c650c0395fc4d77334ec86d00b1bb24c2 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 03:21:11 +0900 Subject: [PATCH 10/58] wip --- src/api/drive/add-file.ts | 12 +++---- src/api/drive/upload-from-url.ts | 2 +- src/api/post/create.ts | 13 ++++---- src/api/post/watch.ts | 2 +- .../processors/db/delete-post-dependents.ts | 12 +++---- src/queue/processors/http/process-inbox.ts | 1 - src/remote/activitypub/act/create.ts | 25 ++++++++++----- src/remote/activitypub/act/delete.ts | 31 ++++++++++++------- src/remote/activitypub/act/undo.ts | 2 +- src/remote/activitypub/delete/index.ts | 10 ------ src/remote/activitypub/delete/post.ts | 13 -------- src/remote/activitypub/resolve-person.ts | 1 - src/server/activitypub/inbox.ts | 4 +-- 13 files changed, 60 insertions(+), 68 deletions(-) delete mode 100644 src/remote/activitypub/delete/index.ts delete mode 100644 src/remote/activitypub/delete/post.ts diff --git a/src/api/drive/add-file.ts b/src/api/drive/add-file.ts index 24eb5208d..64a2f1834 100644 --- a/src/api/drive/add-file.ts +++ b/src/api/drive/add-file.ts @@ -10,12 +10,12 @@ import * as debug from 'debug'; import fileType = require('file-type'); import prominence = require('prominence'); -import DriveFile, { IMetadata, getGridFSBucket } from '../models/drive-file'; -import DriveFolder from '../models/drive-folder'; -import { pack } from '../models/drive-file'; -import event, { publishDriveStream } from '../publishers/stream'; -import getAcct from '../acct/render'; -import config from '../config'; +import DriveFile, { IMetadata, getGridFSBucket } from '../../models/drive-file'; +import DriveFolder from '../../models/drive-folder'; +import { pack } from '../../models/drive-file'; +import event, { publishDriveStream } from '../../publishers/stream'; +import getAcct from '../../acct/render'; +import config from '../../config'; const gm = _gm.subClass({ imageMagick: true diff --git a/src/api/drive/upload-from-url.ts b/src/api/drive/upload-from-url.ts index f96af0f26..26c890d15 100644 --- a/src/api/drive/upload-from-url.ts +++ b/src/api/drive/upload-from-url.ts @@ -1,5 +1,5 @@ import * as URL from 'url'; -import { IDriveFile, validateFileName } from '../models/drive-file'; +import { IDriveFile, validateFileName } from '../../models/drive-file'; import create from './add-file'; import * as debug from 'debug'; import * as tmp from 'tmp'; diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 8256cbc35..36819ec2b 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -1,6 +1,5 @@ -import parseAcct from '../../acct/parse'; import Post, { pack, IPost } from '../../models/post'; -import User, { isLocalUser, isRemoteUser, IUser } from '../../models/user'; +import User, { isLocalUser, IUser } from '../../models/user'; import stream from '../../publishers/stream'; import Following from '../../models/following'; import { createHttp } from '../../queue'; @@ -25,14 +24,16 @@ export default async (user: IUser, content: { repost: IPost; media: IDriveFile[]; geo: any; - poll: any; + poll?: any; viaMobile: boolean; - tags: string[]; - cw: string; - visibility: string; + tags?: string[]; + cw?: string; + visibility?: string; uri?: string; app?: IApp; }) => new Promise(async (res, rej) => { + if (content.visibility == null) content.visibility = 'public'; + const tags = content.tags || []; let tokens = null; diff --git a/src/api/post/watch.ts b/src/api/post/watch.ts index 61ea44443..bbd9976f4 100644 --- a/src/api/post/watch.ts +++ b/src/api/post/watch.ts @@ -1,5 +1,5 @@ import * as mongodb from 'mongodb'; -import Watching from '../models/post-watching'; +import Watching from '../../models/post-watching'; export default async (me: mongodb.ObjectID, post: object) => { // 自分の投稿はwatchできない diff --git a/src/queue/processors/db/delete-post-dependents.ts b/src/queue/processors/db/delete-post-dependents.ts index 879c41ec9..6de21eb05 100644 --- a/src/queue/processors/db/delete-post-dependents.ts +++ b/src/queue/processors/db/delete-post-dependents.ts @@ -1,9 +1,9 @@ -import Favorite from '../../models/favorite'; -import Notification from '../../models/notification'; -import PollVote from '../../models/poll-vote'; -import PostReaction from '../../models/post-reaction'; -import PostWatching from '../../models/post-watching'; -import Post from '../../models/post'; +import Favorite from '../../../models/favorite'; +import Notification from '../../../models/notification'; +import PollVote from '../../../models/poll-vote'; +import PostReaction from '../../../models/post-reaction'; +import PostWatching from '../../../models/post-watching'; +import Post from '../../../models/post'; export default async ({ data }) => Promise.all([ Favorite.remove({ postId: data._id }), diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index 82585d3a6..c3074429f 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -5,7 +5,6 @@ import parseAcct from '../../../acct/parse'; import User, { IRemoteUser } from '../../../models/user'; import act from '../../../remote/activitypub/act'; import resolvePerson from '../../../remote/activitypub/resolve-person'; -import Resolver from '../../../remote/activitypub/resolver'; // ユーザーのinboxにアクティビティが届いた時の処理 export default async (job: kue.Job, done): Promise => { diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index c486571fc..f97832a98 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -36,17 +36,17 @@ export default async (actor, activity): Promise => { switch (object.type) { case 'Image': - createImage(resolver, object); + createImage(object); break; case 'Note': - createNote(resolver, object); + createNote(object); break; } /// - async function createImage(resolver: Resolver, image) { + async function createImage(image) { if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { throw new Error('invalid image'); } @@ -54,7 +54,7 @@ export default async (actor, activity): Promise => { return await uploadFromUrl(image.url, actor); } - async function createNote(resolver: Resolver, note) { + async function createNote(note) { if ( ('attributedTo' in note && actor.account.uri !== note.attributedTo) || typeof note.id !== 'string' @@ -63,20 +63,29 @@ export default async (actor, activity): Promise => { } const media = []; - if ('attachment' in note) { note.attachment.forEach(async media => { - const created = await createImage(resolver, media); + const created = await createImage(media); media.push(created); }); } + let reply = null; + if ('inReplyTo' in note) { + const inReplyToPost = await Post.findOne({ uri: note.id || note }); + if (inReplyToPost) { + reply = inReplyToPost; + } else { + reply = await createNote(await resolver.resolve(note)); + } + } + const { window } = new JSDOM(note.content); - await createPost(actor, { + return await createPost(actor, { createdAt: new Date(note.published), media, - reply: undefined, + reply, repost: undefined, text: window.document.body.textContent, viaMobile: false, diff --git a/src/remote/activitypub/act/delete.ts b/src/remote/activitypub/act/delete.ts index f9eb4dd08..334ca47ed 100644 --- a/src/remote/activitypub/act/delete.ts +++ b/src/remote/activitypub/act/delete.ts @@ -1,21 +1,28 @@ -import create from '../create'; -import deleteObject from '../delete'; +import Resolver from '../resolver'; +import Post from '../../../models/post'; +import { createDb } from '../../../queue'; -export default async (resolver, actor, activity) => { +export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error(); } - const results = await create(resolver, actor, activity.object); + const resolver = new Resolver(); - await Promise.all(results.map(async promisedResult => { - const result = await promisedResult; - if (result === null) { - return; - } + const object = await resolver.resolve(activity); - await deleteObject(result); - })); + switch (object.type) { + case 'Note': + deleteNote(object); + break; + } - return null; + async function deleteNote(note) { + const post = await Post.findOneAndDelete({ uri: note.id }); + + createDb({ + type: 'deletePostDependents', + id: post._id + }).delay(65536).save(); + } }; diff --git a/src/remote/activitypub/act/undo.ts b/src/remote/activitypub/act/undo.ts index b3b83777d..9d9f6b035 100644 --- a/src/remote/activitypub/act/undo.ts +++ b/src/remote/activitypub/act/undo.ts @@ -7,7 +7,7 @@ export default async (actor, activity): Promise => { switch (activity.object.type) { case 'Follow': - unfollow(activity.object); + unfollow(actor, activity.object); break; } diff --git a/src/remote/activitypub/delete/index.ts b/src/remote/activitypub/delete/index.ts deleted file mode 100644 index bc9104284..000000000 --- a/src/remote/activitypub/delete/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -import deletePost from './post'; - -export default async ({ object }) => { - switch (object.$ref) { - case 'posts': - return deletePost(object); - } - - return null; -}; diff --git a/src/remote/activitypub/delete/post.ts b/src/remote/activitypub/delete/post.ts deleted file mode 100644 index f6c816647..000000000 --- a/src/remote/activitypub/delete/post.ts +++ /dev/null @@ -1,13 +0,0 @@ -import Post from '../../../models/post'; -import queue from '../../../queue'; - -export default async ({ $id }) => { - const promisedDeletion = Post.findOneAndDelete({ _id: $id }); - - await new Promise((resolve, reject) => queue.create('db', { - type: 'deletePostDependents', - id: $id - }).delay(65536).save(error => error ? reject(error) : resolve())); - - return promisedDeletion; -}; diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 77d08398b..28162497f 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -2,7 +2,6 @@ import { JSDOM } from 'jsdom'; import { toUnicode } from 'punycode'; import User, { validateUsername, isValidName, isValidDescription } from '../../models/user'; import webFinger from '../webfinger'; -import create from './create'; import Resolver from './resolver'; import uploadFromUrl from '../../api/drive/upload-from-url'; diff --git a/src/server/activitypub/inbox.ts b/src/server/activitypub/inbox.ts index 847dc19af..b0015409a 100644 --- a/src/server/activitypub/inbox.ts +++ b/src/server/activitypub/inbox.ts @@ -1,7 +1,7 @@ import * as bodyParser from 'body-parser'; import * as express from 'express'; import { parseRequest } from 'http-signature'; -import queue from '../../queue'; +import { createHttp } from '../../queue'; const app = express(); @@ -22,7 +22,7 @@ app.post('/@:user/inbox', bodyParser.json({ return res.sendStatus(401); } - queue.create('http', { + createHttp({ type: 'processInbox', activity: req.body, signature, From fc15249aa43471e73eb9dda46e1807084f4d2cd3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 15:50:52 +0900 Subject: [PATCH 11/58] wip --- src/api/post/create.ts | 11 +++++++---- src/client/app/init.ts | 6 +----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 36819ec2b..549511753 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -54,8 +54,7 @@ export default async (user: IUser, content: { }); } - // 投稿を作成 - const post = await Post.insert({ + const data: any = { createdAt: content.createdAt, mediaIds: content.media ? content.media.map(file => file._id) : [], replyId: content.reply ? content.reply._id : null, @@ -68,14 +67,18 @@ export default async (user: IUser, content: { userId: user._id, viaMobile: content.viaMobile, geo: content.geo || null, - uri: content.uri, appId: content.app ? content.app._id : null, visibility: content.visibility, // 以下非正規化データ _reply: content.reply ? { userId: content.reply.userId } : null, _repost: content.repost ? { userId: content.repost.userId } : null, - }); + }; + + if (content.uri != null) data.uri = content.uri; + + // 投稿を作成 + const post = await Post.insert(data); res(post); diff --git a/src/client/app/init.ts b/src/client/app/init.ts index 3e5c38961..2fb8f15cf 100644 --- a/src/client/app/init.ts +++ b/src/client/app/init.ts @@ -14,7 +14,7 @@ import ElementLocaleJa from 'element-ui/lib/locale/lang/ja'; import App from './app.vue'; import checkForUpdate from './common/scripts/check-for-update'; import MiOS, { API } from './common/mios'; -import { version, codename, hostname, lang } from './config'; +import { version, codename, lang } from './config'; let elementLocale; switch (lang) { @@ -60,10 +60,6 @@ console.info( window.clearTimeout((window as any).mkBootTimer); delete (window as any).mkBootTimer; -if (hostname != 'localhost') { - document.domain = hostname; -} - //#region Set lang attr const html = document.documentElement; html.setAttribute('lang', lang); From d3ed2761b9f007fae6977fdbf136619bce498c98 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 15:54:12 +0900 Subject: [PATCH 12/58] wip --- src/server/webfinger.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/server/webfinger.ts b/src/server/webfinger.ts index 20057da31..fd7ebc3fb 100644 --- a/src/server/webfinger.ts +++ b/src/server/webfinger.ts @@ -1,11 +1,12 @@ +import * as express from 'express'; + import config from '../config'; import parseAcct from '../acct/parse'; import User from '../models/user'; -const express = require('express'); const app = express(); -app.get('/.well-known/webfinger', async (req, res) => { +app.get('/.well-known/webfinger', async (req: express.Request, res: express.Response) => { if (typeof req.query.resource !== 'string') { return res.sendStatus(400); } @@ -34,13 +35,15 @@ app.get('/.well-known/webfinger', async (req, res) => { return res.json({ subject: `acct:${user.username}@${config.host}`, - links: [ - { - rel: 'self', - type: 'application/activity+json', - href: `${config.url}/@${user.username}` - } - ] + links: [{ + rel: 'self', + type: 'application/activity+json', + href: `${config.url}/@${user.username}` + }, { + rel: 'http://webfinger.net/rel/profile-page', + type: 'text/html', + href: `${config.url}/@${user.username}` + }] }); }); From f5c55d46b71af6d3f346192a014212c747d1321c Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 17:52:46 +0900 Subject: [PATCH 13/58] wip --- src/server/activitypub/inbox.ts | 4 +--- src/server/activitypub/outbox.ts | 3 +-- src/server/activitypub/post.ts | 3 +-- src/server/activitypub/publickey.ts | 3 +-- src/server/activitypub/user.ts | 3 +-- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/server/activitypub/inbox.ts b/src/server/activitypub/inbox.ts index b0015409a..1b6cc0c00 100644 --- a/src/server/activitypub/inbox.ts +++ b/src/server/activitypub/inbox.ts @@ -3,9 +3,7 @@ import * as express from 'express'; import { parseRequest } from 'http-signature'; import { createHttp } from '../../queue'; -const app = express(); - -app.disable('x-powered-by'); +const app = express.Router(); app.post('/@:user/inbox', bodyParser.json({ type() { diff --git a/src/server/activitypub/outbox.ts b/src/server/activitypub/outbox.ts index 9ecb0c071..976908d1f 100644 --- a/src/server/activitypub/outbox.ts +++ b/src/server/activitypub/outbox.ts @@ -6,8 +6,7 @@ import config from '../../config'; import Post from '../../models/post'; import withUser from './with-user'; -const app = express(); -app.disable('x-powered-by'); +const app = express.Router(); app.get('/@:user/outbox', withUser(username => { return `${config.url}/@${username}/inbox`; diff --git a/src/server/activitypub/post.ts b/src/server/activitypub/post.ts index 91d91aeb9..355c60356 100644 --- a/src/server/activitypub/post.ts +++ b/src/server/activitypub/post.ts @@ -5,8 +5,7 @@ import parseAcct from '../../acct/parse'; import Post from '../../models/post'; import User from '../../models/user'; -const app = express(); -app.disable('x-powered-by'); +const app = express.Router(); app.get('/@:user/:post', async (req, res, next) => { const accepted = req.accepts(['html', 'application/activity+json', 'application/ld+json']); diff --git a/src/server/activitypub/publickey.ts b/src/server/activitypub/publickey.ts index c564c437e..b48504927 100644 --- a/src/server/activitypub/publickey.ts +++ b/src/server/activitypub/publickey.ts @@ -4,8 +4,7 @@ import render from '../../remote/activitypub/renderer/key'; import config from '../../config'; import withUser from './with-user'; -const app = express(); -app.disable('x-powered-by'); +const app = express.Router(); app.get('/@:user/publickey', withUser(username => { return `${config.url}/@${username}/publickey`; diff --git a/src/server/activitypub/user.ts b/src/server/activitypub/user.ts index baf2dc9a0..f05497451 100644 --- a/src/server/activitypub/user.ts +++ b/src/server/activitypub/user.ts @@ -11,8 +11,7 @@ const respond = withUser(username => `${config.url}/@${username}`, (user, req, r res.json(rendered); }); -const app = express(); -app.disable('x-powered-by'); +const app = express.Router(); app.get('/@:user', (req, res, next) => { const accepted = req.accepts(['html', 'application/activity+json', 'application/ld+json']); From 06347cd71e46ce2b991bc8b872cd0725c8862954 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:08:51 +0900 Subject: [PATCH 14/58] wip --- src/api/drive/upload-from-url.ts | 8 +++++++- src/index.ts | 4 ++++ src/remote/activitypub/resolve-person.ts | 10 +++++----- src/remote/activitypub/resolver.ts | 8 ++++++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/api/drive/upload-from-url.ts b/src/api/drive/upload-from-url.ts index 26c890d15..676586cd1 100644 --- a/src/api/drive/upload-from-url.ts +++ b/src/api/drive/upload-from-url.ts @@ -6,14 +6,18 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import * as request from 'request'; -const log = debug('misskey:common:drive:upload_from_url'); +const log = debug('misskey:drive:upload-from-url'); export default async (url, user, folderId = null, uri = null): Promise => { + log(`REQUESTED: ${url}`); + let name = URL.parse(url).pathname.split('/').pop(); if (!validateFileName(name)) { name = null; } + log(`name: ${name}`); + // Create temp file const path = await new Promise((res: (string) => void, rej) => { tmp.file((e, path) => { @@ -37,6 +41,8 @@ export default async (url, user, folderId = null, uri = null): Promise { if (e) log(e.stack); diff --git a/src/index.ts b/src/index.ts index 29c4f3431..e35c917a4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,10 @@ const ev = new Xev(); process.title = 'Misskey'; +if (process.env.NODE_ENV != 'production') { + process.env.DEBUG = 'misskey:*'; +} + // https://github.com/Automattic/kue/issues/822 require('events').EventEmitter.prototype._maxListeners = 256; diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 28162497f..c288a2f00 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -31,7 +31,7 @@ export default async (value, verifier?: string) => { const user = await User.insert({ avatarId: null, bannerId: null, - createdAt: Date.parse(object.published), + createdAt: Date.parse(object.published) || null, description: summaryDOM.textContent, followersCount: 0, followingCount: 0, @@ -55,14 +55,14 @@ export default async (value, verifier?: string) => { const [avatarId, bannerId] = await Promise.all([ object.icon, object.image - ].map(async url => { - if (url === undefined) { + ].map(async img => { + if (img === undefined) { return null; } - const img = await uploadFromUrl(url, user); + const file = await uploadFromUrl(img.url, user); - return img._id; + return file._id; })); User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index de0bba268..09a6e7005 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -1,6 +1,8 @@ -import { IObject } from "./type"; +import * as request from 'request-promise-native'; +import * as debug from 'debug'; +import { IObject } from './type'; -const request = require('request-promise-native'); +const log = debug('misskey:activitypub:resolver'); export default class Resolver { private history: Set; @@ -57,6 +59,8 @@ export default class Resolver { throw new Error('invalid response'); } + log(`resolved: ${JSON.stringify(object)}`); + return object; } } From 7403f38fb43b0ad747236061a591cbf94e198ba6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:43:06 +0900 Subject: [PATCH 15/58] wip --- src/api/post/create.ts | 15 ++++++++------- src/index.ts | 2 +- src/queue/index.ts | 8 +++++++- src/queue/processors/http/deliver.ts | 9 ++++++++- src/queue/processors/http/index.ts | 16 ++++++++++++---- .../processors/http/report-github-failure.ts | 6 +++--- src/remote/activitypub/resolver.ts | 2 +- src/remote/request.ts | 8 ++++++++ src/server/api/endpoints/following/create.ts | 11 ++--------- 9 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 549511753..dbeb87ae8 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -18,20 +18,21 @@ import html from '../../text/html'; import { IApp } from '../../models/app'; export default async (user: IUser, content: { - createdAt: Date; - text: string; - reply: IPost; - repost: IPost; - media: IDriveFile[]; - geo: any; + createdAt?: Date; + text?: string; + reply?: IPost; + repost?: IPost; + media?: IDriveFile[]; + geo?: any; poll?: any; - viaMobile: boolean; + viaMobile?: boolean; tags?: string[]; cw?: string; visibility?: string; uri?: string; app?: IApp; }) => new Promise(async (res, rej) => { + if (content.createdAt == null) content.createdAt = new Date(); if (content.visibility == null) content.visibility = 'public'; const tags = content.tags || []; diff --git a/src/index.ts b/src/index.ts index e35c917a4..f45bcaa6a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -103,7 +103,7 @@ async function workerMain(opt) { if (!opt['only-server']) { // start processor - require('./processor').default(); + require('./queue').default(); } // Send a 'ready' message to parent process diff --git a/src/queue/index.ts b/src/queue/index.ts index c8c436b18..86600dc26 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,8 +1,12 @@ import { createQueue } from 'kue'; +import * as debug from 'debug'; + import config from '../config'; import db from './processors/db'; import http from './processors/http'; +const log = debug('misskey:queue'); + const queue = createQueue({ redis: { port: config.redis.port, @@ -12,6 +16,8 @@ const queue = createQueue({ }); export function createHttp(data) { + log(`HTTP job created: ${JSON.stringify(data)}`); + return queue .create('http', data) .attempts(16) @@ -22,7 +28,7 @@ export function createDb(data) { return queue.create('db', data); } -export function process() { +export default function() { queue.process('db', db); /* diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 1700063a5..da7e8bc36 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -3,5 +3,12 @@ import * as kue from 'kue'; import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - await request(job.data.user, job.data.to, job.data.content); + try { + await request(job.data.user, job.data.to, job.data.content); + done(); + } catch (e) { + console.warn(`deliver failed: ${e}`); + + done(e); + } }; diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 06c6b1d1a..3d7d941b1 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -3,9 +3,17 @@ import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; const handlers = { - deliver, - processInbox, - reportGitHubFailure, + deliver, + processInbox, + reportGitHubFailure }; -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); +export default (job, done) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job).then(() => done(), done); + } else { + console.warn(`Unknown job: ${job.data.type}`); + } +}; diff --git a/src/queue/processors/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts index 4f6f5ccee..e747d062d 100644 --- a/src/queue/processors/http/report-github-failure.ts +++ b/src/queue/processors/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request-promise-native'; -import User from '../../models/user'; -const createPost = require('../../server/api/endpoints/posts/create'); +import User from '../../../models/user'; +import createPost from '../../../api/post/create'; export default async ({ data }) => { const asyncBot = User.findOne({ _id: data.userId }); @@ -20,5 +20,5 @@ export default async ({ data }) => { `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; - createPost({ text }, await asyncBot); + createPost(await asyncBot, { text }); }; diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 09a6e7005..38639c681 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -59,7 +59,7 @@ export default class Resolver { throw new Error('invalid response'); } - log(`resolved: ${JSON.stringify(object)}`); + log(`resolved: ${JSON.stringify(object, null, 2)}`); return object; } diff --git a/src/remote/request.ts b/src/remote/request.ts index 72262cbf6..a375aebfb 100644 --- a/src/remote/request.ts +++ b/src/remote/request.ts @@ -1,9 +1,15 @@ import { request } from 'https'; import { sign } from 'http-signature'; import { URL } from 'url'; +import * as debug from 'debug'; + import config from '../config'; +const log = debug('misskey:activitypub:deliver'); + export default ({ account, username }, url, object) => new Promise((resolve, reject) => { + log(`--> ${url}`); + const { protocol, hostname, port, pathname, search } = new URL(url); const req = request({ @@ -14,6 +20,8 @@ export default ({ account, username }, url, object) => new Promise((resolve, rej path: pathname + search, }, res => { res.on('end', () => { + log(`${url} --> ${res.statusCode}`); + if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } else { diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index e56859521..fae686ce5 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User from '../../../../models/user'; import Following from '../../../../models/following'; -import queue from '../../../../queue'; +import create from '../../../../api/following/create'; /** * Follow a user @@ -50,15 +50,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => { } // Create following - const { _id } = await Following.insert({ - createdAt: new Date(), - followerId: follower._id, - followeeId: followee._id - }); - - queue.create('http', { type: 'follow', following: _id }).save(); + create(follower, followee); // Send response res(); - }); From fd87a63e573581544a169143fad4ef2180be22bd Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:50:52 +0900 Subject: [PATCH 16/58] wip --- src/remote/activitypub/resolve-person.ts | 2 +- src/remote/resolve-user.ts | 2 +- src/server/api/endpoints/users/show.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index c288a2f00..b979bb1cd 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -15,7 +15,7 @@ export default async (value, verifier?: string) => { object.type !== 'Person' || typeof object.preferredUsername !== 'string' || !validateUsername(object.preferredUsername) || - !isValidName(object.name) || + (object.name != '' && !isValidName(object.name)) || !isValidDescription(object.summary) ) { throw new Error('invalid person'); diff --git a/src/remote/resolve-user.ts b/src/remote/resolve-user.ts index 48219e8cb..9e1ae5195 100644 --- a/src/remote/resolve-user.ts +++ b/src/remote/resolve-user.ts @@ -16,7 +16,7 @@ export default async (username, host, option) => { const finger = await webFinger(acctLower, acctLower); const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self'); if (!self) { - throw new Error(); + throw new Error('self link not found'); } user = await resolvePerson(self.href, acctLower); diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts index 2b0279937..d272ce463 100644 --- a/src/server/api/endpoints/users/show.ts +++ b/src/server/api/endpoints/users/show.ts @@ -37,7 +37,8 @@ module.exports = (params, me) => new Promise(async (res, rej) => { if (typeof host === 'string') { try { user = await resolveRemoteUser(username, host, cursorOption); - } catch (exception) { + } catch (e) { + console.warn(`failed to resolve remote user: ${e}`); return rej('failed to resolve remote user'); } } else { From 2a80fdeafe295896b99a499b499873d7d8b55a3d Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 19:19:00 +0900 Subject: [PATCH 17/58] wip --- src/remote/activitypub/act/create.ts | 46 ++++++++++++++----------- src/remote/activitypub/act/index.ts | 4 +++ src/remote/activitypub/renderer/note.ts | 13 ++++--- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index f97832a98..80afb61bd 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -1,11 +1,13 @@ import { JSDOM } from 'jsdom'; +import * as debug from 'debug'; import Resolver from '../resolver'; -import DriveFile from '../../../models/drive-file'; import Post from '../../../models/post'; import uploadFromUrl from '../../../api/drive/upload-from-url'; import createPost from '../../../api/post/create'; +const log = debug('misskey:activitypub'); + export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); @@ -13,26 +15,20 @@ export default async (actor, activity): Promise => { const uri = activity.id || activity; - try { - await Promise.all([ - DriveFile.findOne({ 'metadata.uri': uri }).then(file => { - if (file !== null) { - throw new Error(); - } - }, () => {}), - Post.findOne({ uri }).then(post => { - if (post !== null) { - throw new Error(); - } - }, () => {}) - ]); - } catch (object) { - throw new Error(`already registered: ${uri}`); - } + log(`Create: ${uri}`); + + // TODO: 同じURIをもつものが既に登録されていないかチェック const resolver = new Resolver(); - const object = await resolver.resolve(activity); + let object; + + try { + object = await resolver.resolve(activity.object); + } catch (e) { + log(`Resolve failed: ${e}`); + throw e; + } switch (object.type) { case 'Image': @@ -42,15 +38,22 @@ export default async (actor, activity): Promise => { case 'Note': createNote(object); break; + + default: + console.warn(`Unknown type: ${object.type}`); + break; } /// async function createImage(image) { if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { + log(`invalid image: ${JSON.stringify(image, null, 2)}`); throw new Error('invalid image'); } + log(`Creating the Image: ${uri}`); + return await uploadFromUrl(image.url, actor); } @@ -59,11 +62,14 @@ export default async (actor, activity): Promise => { ('attributedTo' in note && actor.account.uri !== note.attributedTo) || typeof note.id !== 'string' ) { + log(`invalid note: ${JSON.stringify(note, null, 2)}`); throw new Error('invalid note'); } + log(`Creating the Note: ${uri}`); + const media = []; - if ('attachment' in note) { + if ('attachment' in note && note.attachment != null) { note.attachment.forEach(async media => { const created = await createImage(media); media.push(created); @@ -71,7 +77,7 @@ export default async (actor, activity): Promise => { } let reply = null; - if ('inReplyTo' in note) { + if ('inReplyTo' in note && note.inReplyTo != null) { const inReplyToPost = await Post.findOne({ uri: note.id || note }); if (inReplyToPost) { reply = inReplyToPost; diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index f22500ace..584022709 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -18,6 +18,10 @@ export default async (actor, activity: IObject): Promise => { await follow(actor, activity); break; + case 'Accept': + // noop + break; + case 'Undo': await undo(actor, activity); break; diff --git a/src/remote/activitypub/renderer/note.ts b/src/remote/activitypub/renderer/note.ts index 43531b121..e45b10215 100644 --- a/src/remote/activitypub/renderer/note.ts +++ b/src/remote/activitypub/renderer/note.ts @@ -2,11 +2,14 @@ import renderDocument from './document'; import renderHashtag from './hashtag'; import config from '../../../config'; import DriveFile from '../../../models/drive-file'; -import Post from '../../../models/post'; -import User from '../../../models/user'; +import Post, { IPost } from '../../../models/post'; +import User, { IUser } from '../../../models/user'; + +export default async (user: IUser, post: IPost) => { + const promisedFiles = post.mediaIds + ? DriveFile.find({ _id: { $in: post.mediaIds } }) + : Promise.resolve([]); -export default async (user, post) => { - const promisedFiles = DriveFile.find({ _id: { $in: post.mediaIds } }); let inReplyTo; if (post.replyId) { @@ -39,6 +42,6 @@ export default async (user, post) => { cc: `${attributedTo}/followers`, inReplyTo, attachment: (await promisedFiles).map(renderDocument), - tag: post.tags.map(renderHashtag) + tag: (post.tags || []).map(renderHashtag) }; }; From a6abcd1aa5edca98d8cc2b974cc63b06b3dd75bf Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 19:23:42 +0900 Subject: [PATCH 18/58] wip --- src/remote/activitypub/act/create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 80afb61bd..7d5a9d427 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -78,7 +78,7 @@ export default async (actor, activity): Promise => { let reply = null; if ('inReplyTo' in note && note.inReplyTo != null) { - const inReplyToPost = await Post.findOne({ uri: note.id || note }); + const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); if (inReplyToPost) { reply = inReplyToPost; } else { From 5f8ab584464386b67bff6f9b7f60525e3e50cecc Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 22:49:41 +0900 Subject: [PATCH 19/58] wip --- src/remote/activitypub/act/create.ts | 106 ++++++++++++----------- src/remote/activitypub/act/index.ts | 3 +- src/remote/activitypub/resolve-person.ts | 2 +- src/remote/activitypub/resolver.ts | 4 + 4 files changed, 62 insertions(+), 53 deletions(-) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 7d5a9d427..9669348d5 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -5,10 +5,12 @@ import Resolver from '../resolver'; import Post from '../../../models/post'; import uploadFromUrl from '../../../api/drive/upload-from-url'; import createPost from '../../../api/post/create'; +import { IRemoteUser, isRemoteUser } from '../../../models/user'; +import resolvePerson from '../resolve-person'; const log = debug('misskey:activitypub'); -export default async (actor, activity): Promise => { +export default async (actor: IRemoteUser, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } @@ -32,71 +34,73 @@ export default async (actor, activity): Promise => { switch (object.type) { case 'Image': - createImage(object); + createImage(resolver, actor, object); break; case 'Note': - createNote(object); + createNote(resolver, actor, object); break; default: console.warn(`Unknown type: ${object.type}`); break; } +}; - /// - - async function createImage(image) { - if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { - log(`invalid image: ${JSON.stringify(image, null, 2)}`); - throw new Error('invalid image'); - } - - log(`Creating the Image: ${uri}`); - - return await uploadFromUrl(image.url, actor); +async function createImage(resolver: Resolver, actor: IRemoteUser, image) { + if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { + log(`invalid image: ${JSON.stringify(image, null, 2)}`); + throw new Error('invalid image'); } - async function createNote(note) { - if ( - ('attributedTo' in note && actor.account.uri !== note.attributedTo) || - typeof note.id !== 'string' - ) { - log(`invalid note: ${JSON.stringify(note, null, 2)}`); - throw new Error('invalid note'); - } + log(`Creating the Image: ${image.id}`); - log(`Creating the Note: ${uri}`); + return await uploadFromUrl(image.url, actor); +} - const media = []; - if ('attachment' in note && note.attachment != null) { - note.attachment.forEach(async media => { - const created = await createImage(media); - media.push(created); - }); - } +async function createNote(resolver: Resolver, actor: IRemoteUser, note) { + if ( + ('attributedTo' in note && actor.account.uri !== note.attributedTo) || + typeof note.id !== 'string' + ) { + log(`invalid note: ${JSON.stringify(note, null, 2)}`); + throw new Error('invalid note'); + } - let reply = null; - if ('inReplyTo' in note && note.inReplyTo != null) { - const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); - if (inReplyToPost) { - reply = inReplyToPost; - } else { - reply = await createNote(await resolver.resolve(note)); - } - } + log(`Creating the Note: ${note.id}`); - const { window } = new JSDOM(note.content); - - return await createPost(actor, { - createdAt: new Date(note.published), - media, - reply, - repost: undefined, - text: window.document.body.textContent, - viaMobile: false, - geo: undefined, - uri: note.id + const media = []; + if ('attachment' in note && note.attachment != null) { + note.attachment.forEach(async media => { + const created = await createImage(resolver, note.actor, media); + media.push(created); }); } -}; + + let reply = null; + if ('inReplyTo' in note && note.inReplyTo != null) { + const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); + if (inReplyToPost) { + reply = inReplyToPost; + } else { + const inReplyTo = await resolver.resolve(note.inReplyTo) as any; + const actor = await resolvePerson(inReplyTo.attributedTo); + if (isRemoteUser(actor)) { + reply = await createNote(resolver, actor, inReplyTo); + } + } + } + + const { window } = new JSDOM(note.content); + + return await createPost(actor, { + createdAt: new Date(note.published), + media, + reply, + repost: undefined, + text: window.document.body.textContent, + viaMobile: false, + geo: undefined, + uri: note.id + }); +} diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index 584022709..f58505b0a 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -3,8 +3,9 @@ import performDeleteActivity from './delete'; import follow from './follow'; import undo from './undo'; import { IObject } from '../type'; +import { IUser } from '../../../models/user'; -export default async (actor, activity: IObject): Promise => { +export default async (actor: IUser, activity: IObject): Promise => { switch (activity.type) { case 'Create': await create(actor, activity); diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index b979bb1cd..2bf7a1354 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -11,7 +11,7 @@ export default async (value, verifier?: string) => { const object = await resolver.resolve(value) as any; if ( - object === null || + object == null || object.type !== 'Person' || typeof object.preferredUsername !== 'string' || !validateUsername(object.preferredUsername) || diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 38639c681..4a97e2ef6 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -33,6 +33,10 @@ export default class Resolver { } public async resolve(value): Promise { + if (value == null) { + throw new Error('resolvee is null (or undefined)'); + } + if (typeof value !== 'string') { return value; } From 30bd467b7143502b5aacc86de7e724ac4aa9c6c3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 23:19:47 +0900 Subject: [PATCH 20/58] wip --- src/api/post/create.ts | 40 +++++++++++++++------------- src/remote/activitypub/act/create.ts | 6 ++--- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/api/post/create.ts b/src/api/post/create.ts index dbeb87ae8..7b7fceda2 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -31,7 +31,7 @@ export default async (user: IUser, content: { visibility?: string; uri?: string; app?: IApp; -}) => new Promise(async (res, rej) => { +}, silent = false) => new Promise(async (res, rej) => { if (content.createdAt == null) content.createdAt = new Date(); if (content.visibility == null) content.visibility = 'public'; @@ -120,26 +120,28 @@ export default async (user: IUser, content: { _id: false }); - const note = await renderNote(user, post); - const content = renderCreate(note); - content['@context'] = context; + if (!silent) { + const note = await renderNote(user, post); + const content = renderCreate(note); + content['@context'] = context; - Promise.all(followers.map(({ follower }) => { - if (isLocalUser(follower)) { - // Publish event to followers stream - stream(follower._id, 'post', postObj); - } else { - // フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信 - if (isLocalUser(user)) { - createHttp({ - type: 'deliver', - user, - content, - to: follower.account.inbox - }).save(); + Promise.all(followers.map(({ follower }) => { + if (isLocalUser(follower)) { + // Publish event to followers stream + stream(follower._id, 'post', postObj); + } else { + // フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信 + if (isLocalUser(user)) { + createHttp({ + type: 'deliver', + user, + content, + to: follower.account.inbox + }).save(); + } } - } - })); + })); + } } // チャンネルへの投稿 diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 9669348d5..fe58f58f8 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -58,7 +58,7 @@ async function createImage(resolver: Resolver, actor: IRemoteUser, image) { return await uploadFromUrl(image.url, actor); } -async function createNote(resolver: Resolver, actor: IRemoteUser, note) { +async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = false) { if ( ('attributedTo' in note && actor.account.uri !== note.attributedTo) || typeof note.id !== 'string' @@ -86,7 +86,7 @@ async function createNote(resolver: Resolver, actor: IRemoteUser, note) { const inReplyTo = await resolver.resolve(note.inReplyTo) as any; const actor = await resolvePerson(inReplyTo.attributedTo); if (isRemoteUser(actor)) { - reply = await createNote(resolver, actor, inReplyTo); + reply = await createNote(resolver, actor, inReplyTo, true); } } } @@ -102,5 +102,5 @@ async function createNote(resolver: Resolver, actor: IRemoteUser, note) { viaMobile: false, geo: undefined, uri: note.id - }); + }, silent); } From 0de40f3a76ef159b099a90b65ded3073dbbd2b78 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 23:24:51 +0900 Subject: [PATCH 21/58] wip --- src/api/following/create.ts | 16 +++------------- src/api/following/delete.ts | 9 ++------- src/api/post/create.ts | 9 ++------- src/queue/index.ts | 9 +++++++++ 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/src/api/following/create.ts b/src/api/following/create.ts index 353a6c892..d919f4487 100644 --- a/src/api/following/create.ts +++ b/src/api/following/create.ts @@ -7,7 +7,7 @@ import notify from '../../publishers/notify'; import context from '../../remote/activitypub/renderer/context'; import renderFollow from '../../remote/activitypub/renderer/follow'; import renderAccept from '../../remote/activitypub/renderer/accept'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; export default async function(follower: IUser, followee: IUser, activity?) { const following = await Following.insert({ @@ -60,23 +60,13 @@ export default async function(follower: IUser, followee: IUser, activity?) { const content = renderFollow(follower, followee); content['@context'] = context; - createHttp({ - type: 'deliver', - user: follower, - content, - to: followee.account.inbox - }).save(); + deliver(follower, content, followee.account.inbox).save(); } if (isRemoteUser(follower) && isLocalUser(followee)) { const content = renderAccept(activity); content['@context'] = context; - createHttp({ - type: 'deliver', - user: followee, - content, - to: follower.account.inbox - }).save(); + deliver(followee, content, follower.account.inbox).save(); } } diff --git a/src/api/following/delete.ts b/src/api/following/delete.ts index 4cdff7ce1..364a4803b 100644 --- a/src/api/following/delete.ts +++ b/src/api/following/delete.ts @@ -6,7 +6,7 @@ import event from '../../publishers/stream'; import context from '../../remote/activitypub/renderer/context'; import renderFollow from '../../remote/activitypub/renderer/follow'; import renderUndo from '../../remote/activitypub/renderer/undo'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; export default async function(follower: IUser, followee: IUser, activity?) { const following = await Following.findOne({ @@ -59,11 +59,6 @@ export default async function(follower: IUser, followee: IUser, activity?) { const content = renderUndo(renderFollow(follower, followee)); content['@context'] = context; - createHttp({ - type: 'deliver', - user: follower, - content, - to: followee.account.inbox - }).save(); + deliver(follower, content, followee.account.inbox).save(); } } diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 7b7fceda2..9723dbe45 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -2,7 +2,7 @@ import Post, { pack, IPost } from '../../models/post'; import User, { isLocalUser, IUser } from '../../models/user'; import stream from '../../publishers/stream'; import Following from '../../models/following'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; import renderNote from '../../remote/activitypub/renderer/note'; import renderCreate from '../../remote/activitypub/renderer/create'; import context from '../../remote/activitypub/renderer/context'; @@ -132,12 +132,7 @@ export default async (user: IUser, content: { } else { // フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信 if (isLocalUser(user)) { - createHttp({ - type: 'deliver', - user, - content, - to: follower.account.inbox - }).save(); + deliver(user, content, follower.account.inbox).save(); } } })); diff --git a/src/queue/index.ts b/src/queue/index.ts index 86600dc26..689985e0e 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -28,6 +28,15 @@ export function createDb(data) { return queue.create('db', data); } +export function deliver(user, content, to) { + return createHttp({ + type: 'deliver', + user, + content, + to + }); +} + export default function() { queue.process('db', db); From b6aeacdeb942beb7b5b2f6ac8cf4a89163e59153 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 03:10:25 +0900 Subject: [PATCH 22/58] RENAME: api --> services --- src/queue/processors/http/report-github-failure.ts | 2 +- src/remote/activitypub/act/create.ts | 4 ++-- src/remote/activitypub/act/follow.ts | 2 +- src/remote/activitypub/act/unfollow.ts | 2 +- src/remote/activitypub/resolve-person.ts | 2 +- src/server/api/endpoints/following/create.ts | 2 +- src/server/api/endpoints/posts/create.ts | 2 +- src/{api => services}/drive/add-file.ts | 0 src/{api => services}/drive/upload-from-url.ts | 0 src/{api => services}/following/create.ts | 0 src/{api => services}/following/delete.ts | 0 src/{api => services}/post/create.ts | 0 src/services/post/reaction/create.ts | 0 src/{api => services}/post/watch.ts | 0 14 files changed, 8 insertions(+), 8 deletions(-) rename src/{api => services}/drive/add-file.ts (100%) rename src/{api => services}/drive/upload-from-url.ts (100%) rename src/{api => services}/following/create.ts (100%) rename src/{api => services}/following/delete.ts (100%) rename src/{api => services}/post/create.ts (100%) create mode 100644 src/services/post/reaction/create.ts rename src/{api => services}/post/watch.ts (100%) diff --git a/src/queue/processors/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts index e747d062d..1e0b51f89 100644 --- a/src/queue/processors/http/report-github-failure.ts +++ b/src/queue/processors/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request-promise-native'; import User from '../../../models/user'; -import createPost from '../../../api/post/create'; +import createPost from '../../../services/post/create'; export default async ({ data }) => { const asyncBot = User.findOne({ _id: data.userId }); diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index fe58f58f8..139c98f3b 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -3,8 +3,8 @@ import * as debug from 'debug'; import Resolver from '../resolver'; import Post from '../../../models/post'; -import uploadFromUrl from '../../../api/drive/upload-from-url'; -import createPost from '../../../api/post/create'; +import uploadFromUrl from '../../../services/drive/upload-from-url'; +import createPost from '../../../services/post/create'; import { IRemoteUser, isRemoteUser } from '../../../models/user'; import resolvePerson from '../resolve-person'; diff --git a/src/remote/activitypub/act/follow.ts b/src/remote/activitypub/act/follow.ts index dc173a0ac..4fc423d15 100644 --- a/src/remote/activitypub/act/follow.ts +++ b/src/remote/activitypub/act/follow.ts @@ -1,7 +1,7 @@ import parseAcct from '../../../acct/parse'; import User from '../../../models/user'; import config from '../../../config'; -import follow from '../../../api/following/create'; +import follow from '../../../services/following/create'; export default async (actor, activity): Promise => { const prefix = config.url + '/@'; diff --git a/src/remote/activitypub/act/unfollow.ts b/src/remote/activitypub/act/unfollow.ts index e3c9e1c1c..66c15e9a9 100644 --- a/src/remote/activitypub/act/unfollow.ts +++ b/src/remote/activitypub/act/unfollow.ts @@ -1,7 +1,7 @@ import parseAcct from '../../../acct/parse'; import User from '../../../models/user'; import config from '../../../config'; -import unfollow from '../../../api/following/delete'; +import unfollow from '../../../services/following/delete'; export default async (actor, activity): Promise => { const prefix = config.url + '/@'; diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 2bf7a1354..907f19834 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -3,7 +3,7 @@ import { toUnicode } from 'punycode'; import User, { validateUsername, isValidName, isValidDescription } from '../../models/user'; import webFinger from '../webfinger'; import Resolver from './resolver'; -import uploadFromUrl from '../../api/drive/upload-from-url'; +import uploadFromUrl from '../../services/drive/upload-from-url'; export default async (value, verifier?: string) => { const resolver = new Resolver(); diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index fae686ce5..0ccac8d83 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User from '../../../../models/user'; import Following from '../../../../models/following'; -import create from '../../../../api/following/create'; +import create from '../../../../services/following/create'; /** * Follow a user diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index d241c8c38..003a892bc 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -7,7 +7,7 @@ import Post, { IPost, isValidText, isValidCw, pack } from '../../../../models/po import { ILocalUser } from '../../../../models/user'; import Channel, { IChannel } from '../../../../models/channel'; import DriveFile from '../../../../models/drive-file'; -import create from '../../../../api/post/create'; +import create from '../../../../services/post/create'; import { IApp } from '../../../../models/app'; /** diff --git a/src/api/drive/add-file.ts b/src/services/drive/add-file.ts similarity index 100% rename from src/api/drive/add-file.ts rename to src/services/drive/add-file.ts diff --git a/src/api/drive/upload-from-url.ts b/src/services/drive/upload-from-url.ts similarity index 100% rename from src/api/drive/upload-from-url.ts rename to src/services/drive/upload-from-url.ts diff --git a/src/api/following/create.ts b/src/services/following/create.ts similarity index 100% rename from src/api/following/create.ts rename to src/services/following/create.ts diff --git a/src/api/following/delete.ts b/src/services/following/delete.ts similarity index 100% rename from src/api/following/delete.ts rename to src/services/following/delete.ts diff --git a/src/api/post/create.ts b/src/services/post/create.ts similarity index 100% rename from src/api/post/create.ts rename to src/services/post/create.ts diff --git a/src/services/post/reaction/create.ts b/src/services/post/reaction/create.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/api/post/watch.ts b/src/services/post/watch.ts similarity index 100% rename from src/api/post/watch.ts rename to src/services/post/watch.ts From 0154e44e1d02829e8f35fa131005448f694e745e Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 03:42:55 +0900 Subject: [PATCH 23/58] Fix bugs --- src/queue/processors/http/index.ts | 3 ++- src/services/post/create.ts | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 3d7d941b1..61d7f9ac9 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -12,8 +12,9 @@ export default (job, done) => { const handler = handlers[job.data.type]; if (handler) { - handler(job).then(() => done(), done); + handler(job, done); } else { console.warn(`Unknown job: ${job.data.type}`); + done(); } }; diff --git a/src/services/post/create.ts b/src/services/post/create.ts index 9723dbe45..405e4a2f7 100644 --- a/src/services/post/create.ts +++ b/src/services/post/create.ts @@ -98,7 +98,7 @@ export default async (user: IUser, content: { const postObj = await pack(post); // タイムラインへの投稿 - if (!post.channelId) { + if (post.channelId == null) { // Publish event to myself's stream if (isLocalUser(user)) { stream(post.userId, 'post', postObj); @@ -110,7 +110,7 @@ export default async (user: IUser, content: { from: 'users', localField: 'followerId', foreignField: '_id', - as: 'follower' + as: 'user' } }, { $match: { @@ -125,7 +125,9 @@ export default async (user: IUser, content: { const content = renderCreate(note); content['@context'] = context; - Promise.all(followers.map(({ follower }) => { + Promise.all(followers.map(follower => { + follower = follower.user[0]; + if (isLocalUser(follower)) { // Publish event to followers stream stream(follower._id, 'post', postObj); From a6fb4f2e33b5bcbda5c9461d3c55e3d1c956b2c9 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 04:04:50 +0900 Subject: [PATCH 24/58] wip --- src/models/post.ts | 14 ++++++ src/services/post/create.ts | 98 +++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/src/models/post.ts b/src/models/post.ts index 2f2b51b94..68a638fa2 100644 --- a/src/models/post.ts +++ b/src/models/post.ts @@ -52,6 +52,20 @@ export type IPost = { speed: number; }; uri: string; + + _reply?: { + userId: mongo.ObjectID; + }; + _repost?: { + userId: mongo.ObjectID; + }; + _user: { + host: string; + hostLower: string; + account: { + inbox?: string; + }; + }; }; /** diff --git a/src/services/post/create.ts b/src/services/post/create.ts index 405e4a2f7..745683b51 100644 --- a/src/services/post/create.ts +++ b/src/services/post/create.ts @@ -1,5 +1,5 @@ import Post, { pack, IPost } from '../../models/post'; -import User, { isLocalUser, IUser } from '../../models/user'; +import User, { isLocalUser, IUser, isRemoteUser } from '../../models/user'; import stream from '../../publishers/stream'; import Following from '../../models/following'; import { deliver } from '../../queue'; @@ -17,7 +17,7 @@ import parse from '../../text/parse'; import html from '../../text/html'; import { IApp } from '../../models/app'; -export default async (user: IUser, content: { +export default async (user: IUser, data: { createdAt?: Date; text?: string; reply?: IPost; @@ -32,16 +32,16 @@ export default async (user: IUser, content: { uri?: string; app?: IApp; }, silent = false) => new Promise(async (res, rej) => { - if (content.createdAt == null) content.createdAt = new Date(); - if (content.visibility == null) content.visibility = 'public'; + if (data.createdAt == null) data.createdAt = new Date(); + if (data.visibility == null) data.visibility = 'public'; - const tags = content.tags || []; + const tags = data.tags || []; let tokens = null; - if (content.text) { + if (data.text) { // Analyze - tokens = parse(content.text); + tokens = parse(data.text); // Extract hashtags const hashtags = tokens @@ -55,31 +55,38 @@ export default async (user: IUser, content: { }); } - const data: any = { - createdAt: content.createdAt, - mediaIds: content.media ? content.media.map(file => file._id) : [], - replyId: content.reply ? content.reply._id : null, - repostId: content.repost ? content.repost._id : null, - text: content.text, + const insert: any = { + createdAt: data.createdAt, + mediaIds: data.media ? data.media.map(file => file._id) : [], + replyId: data.reply ? data.reply._id : null, + repostId: data.repost ? data.repost._id : null, + text: data.text, textHtml: tokens === null ? null : html(tokens), - poll: content.poll, - cw: content.cw, + poll: data.poll, + cw: data.cw, tags, userId: user._id, - viaMobile: content.viaMobile, - geo: content.geo || null, - appId: content.app ? content.app._id : null, - visibility: content.visibility, + viaMobile: data.viaMobile, + geo: data.geo || null, + appId: data.app ? data.app._id : null, + visibility: data.visibility, // 以下非正規化データ - _reply: content.reply ? { userId: content.reply.userId } : null, - _repost: content.repost ? { userId: content.repost.userId } : null, + _reply: data.reply ? { userId: data.reply.userId } : null, + _repost: data.repost ? { userId: data.repost.userId } : null, + _user: { + host: user.host, + hostLower: user.hostLower, + account: isLocalUser(user) ? {} : { + inbox: user.account.inbox + } + } }; - if (content.uri != null) data.uri = content.uri; + if (data.uri != null) insert.uri = data.uri; // 投稿を作成 - const post = await Post.insert(data); + const post = await Post.insert(insert); res(post); @@ -125,6 +132,11 @@ export default async (user: IUser, content: { const content = renderCreate(note); content['@context'] = context; + // 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送 + if (data.reply && isLocalUser(user) && isRemoteUser(data.reply._user)) { + deliver(user, content, data.reply._user.account.inbox).save(); + } + Promise.all(followers.map(follower => { follower = follower.user[0]; @@ -199,22 +211,22 @@ export default async (user: IUser, content: { } // If has in reply to post - if (content.reply) { + if (data.reply) { // Increment replies count - Post.update({ _id: content.reply._id }, { + Post.update({ _id: data.reply._id }, { $inc: { repliesCount: 1 } }); // (自分自身へのリプライでない限りは)通知を作成 - notify(content.reply.userId, user._id, 'reply', { + notify(data.reply.userId, user._id, 'reply', { postId: post._id }); // Fetch watchers PostWatching.find({ - postId: content.reply._id, + postId: data.reply._id, userId: { $ne: user._id }, // 削除されたドキュメントは除く deletedAt: { $exists: false } @@ -232,24 +244,24 @@ export default async (user: IUser, content: { // この投稿をWatchする if (isLocalUser(user) && user.account.settings.autoWatch !== false) { - watch(user._id, content.reply); + watch(user._id, data.reply); } // Add mention - addMention(content.reply.userId, 'reply'); + addMention(data.reply.userId, 'reply'); } // If it is repost - if (content.repost) { + if (data.repost) { // Notify - const type = content.text ? 'quote' : 'repost'; - notify(content.repost.userId, user._id, type, { + const type = data.text ? 'quote' : 'repost'; + notify(data.repost.userId, user._id, type, { post_id: post._id }); // Fetch watchers PostWatching.find({ - postId: content.repost._id, + postId: data.repost._id, userId: { $ne: user._id }, // 削除されたドキュメントは除く deletedAt: { $exists: false } @@ -267,24 +279,24 @@ export default async (user: IUser, content: { // この投稿をWatchする if (isLocalUser(user) && user.account.settings.autoWatch !== false) { - watch(user._id, content.repost); + watch(user._id, data.repost); } // If it is quote repost - if (content.text) { + if (data.text) { // Add mention - addMention(content.repost.userId, 'quote'); + addMention(data.repost.userId, 'quote'); } else { // Publish event - if (!user._id.equals(content.repost.userId)) { - event(content.repost.userId, 'repost', postObj); + if (!user._id.equals(data.repost.userId)) { + event(data.repost.userId, 'repost', postObj); } } // 今までで同じ投稿をRepostしているか const existRepost = await Post.findOne({ userId: user._id, - repostId: content.repost._id, + repostId: data.repost._id, _id: { $ne: post._id } @@ -292,7 +304,7 @@ export default async (user: IUser, content: { if (!existRepost) { // Update repostee status - Post.update({ _id: content.repost._id }, { + Post.update({ _id: data.repost._id }, { $inc: { repostCount: 1 } @@ -301,7 +313,7 @@ export default async (user: IUser, content: { } // If has text content - if (content.text) { + if (data.text) { // Extract an '@' mentions const atMentions = tokens .filter(t => t.type == 'mention') @@ -322,8 +334,8 @@ export default async (user: IUser, content: { if (mentionee == null) return; // 既に言及されたユーザーに対する返信や引用repostの場合も無視 - if (content.reply && content.reply.userId.equals(mentionee._id)) return; - if (content.repost && content.repost.userId.equals(mentionee._id)) return; + if (data.reply && data.reply.userId.equals(mentionee._id)) return; + if (data.repost && data.repost.userId.equals(mentionee._id)) return; // Add mention addMention(mentionee._id, 'mention'); From 862463a13cc952919322b4e2f06c196bf2850517 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 14:35:17 +0900 Subject: [PATCH 25/58] Fix bug --- src/queue/processors/http/process-inbox.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index c3074429f..4666e7f37 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -19,6 +19,7 @@ export default async (job: kue.Job, done): Promise => { if (host === null) { console.warn(`request was made by local user: @${username}`); done(); + return; } user = await User.findOne({ usernameLower: username, hostLower: host }) as IRemoteUser; @@ -40,7 +41,8 @@ export default async (job: kue.Job, done): Promise => { } if (!verifySignature(signature, user.account.publicKey.publicKeyPem)) { - done(new Error('signature verification failed')); + console.warn('signature verification failed'); + done(); return; } From 0856f4cd12ba2e053ab3ef30acdbec7e9a59fc61 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 14:39:28 +0900 Subject: [PATCH 26/58] Use error instaed of warn --- src/queue/processors/http/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 61d7f9ac9..3dc259537 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -14,7 +14,7 @@ export default (job, done) => { if (handler) { handler(job, done); } else { - console.warn(`Unknown job: ${job.data.type}`); + console.error(`Unknown job: ${job.data.type}`); done(); } }; From 1e8fe4c6a89c9f117ad7a359b44ff49410b694d6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 14:47:00 +0900 Subject: [PATCH 27/58] Better English --- src/remote/activitypub/act/create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 139c98f3b..10083995d 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -28,7 +28,7 @@ export default async (actor: IRemoteUser, activity): Promise => { try { object = await resolver.resolve(activity.object); } catch (e) { - log(`Resolve failed: ${e}`); + log(`Resolution failed: ${e}`); throw e; } From eb0809ebb422849e2e362d64ee0efb7cb75dc3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=93=E3=81=B4=E3=81=AA=E3=81=9F=E3=81=BF=E3=81=BD?= Date: Fri, 6 Apr 2018 14:57:01 +0900 Subject: [PATCH 28/58] Remove needless log --- src/queue/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/queue/index.ts b/src/queue/index.ts index 689985e0e..691223de2 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,12 +1,9 @@ import { createQueue } from 'kue'; -import * as debug from 'debug'; import config from '../config'; import db from './processors/db'; import http from './processors/http'; -const log = debug('misskey:queue'); - const queue = createQueue({ redis: { port: config.redis.port, @@ -16,8 +13,6 @@ const queue = createQueue({ }); export function createHttp(data) { - log(`HTTP job created: ${JSON.stringify(data)}`); - return queue .create('http', data) .attempts(16) From 9c15c94f801de7019f014446aa3b8ea42980a1da Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 19:18:38 +0900 Subject: [PATCH 29/58] Remove silent flag --- src/remote/activitypub/act/create.ts | 6 +++--- src/services/post/create.ts | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 10083995d..1b9bad8ff 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -58,7 +58,7 @@ async function createImage(resolver: Resolver, actor: IRemoteUser, image) { return await uploadFromUrl(image.url, actor); } -async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = false) { +async function createNote(resolver: Resolver, actor: IRemoteUser, note) { if ( ('attributedTo' in note && actor.account.uri !== note.attributedTo) || typeof note.id !== 'string' @@ -86,7 +86,7 @@ async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = const inReplyTo = await resolver.resolve(note.inReplyTo) as any; const actor = await resolvePerson(inReplyTo.attributedTo); if (isRemoteUser(actor)) { - reply = await createNote(resolver, actor, inReplyTo, true); + reply = await createNote(resolver, actor, inReplyTo); } } } @@ -102,5 +102,5 @@ async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = viaMobile: false, geo: undefined, uri: note.id - }, silent); + }); } diff --git a/src/services/post/create.ts b/src/services/post/create.ts index 745683b51..0bede2772 100644 --- a/src/services/post/create.ts +++ b/src/services/post/create.ts @@ -31,7 +31,7 @@ export default async (user: IUser, data: { visibility?: string; uri?: string; app?: IApp; -}, silent = false) => new Promise(async (res, rej) => { +}) => new Promise(async (res, rej) => { if (data.createdAt == null) data.createdAt = new Date(); if (data.visibility == null) data.visibility = 'public'; @@ -127,7 +127,10 @@ export default async (user: IUser, data: { _id: false }); - if (!silent) { + // この投稿が3分以内に作成されたものであるならストリームに配信 + const shouldDistribute = new Date().getTime() - post.createdAt.getTime() < 1000 * 60 * 3; + + if (shouldDistribute) { const note = await renderNote(user, post); const content = renderCreate(note); content['@context'] = context; From 0ecbafe5eb6c77651a5c149f331ed013d5ac75c1 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 19:23:13 +0900 Subject: [PATCH 30/58] Add todos --- src/remote/activitypub/act/create.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 1b9bad8ff..d1eeacbc3 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -71,6 +71,8 @@ async function createNote(resolver: Resolver, actor: IRemoteUser, note) { const media = []; if ('attachment' in note && note.attachment != null) { + // TODO: attachmentは必ずしもImageではない + // TODO: ループの中でawaitはすべきでない note.attachment.forEach(async media => { const created = await createImage(resolver, note.actor, media); media.push(created); From 574e3b0bfd33ac101c00556bb948861ae9b64e72 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 19:26:17 +0900 Subject: [PATCH 31/58] Fix type annotation --- src/remote/activitypub/act/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index f58505b0a..c1d64b7c7 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -3,9 +3,9 @@ import performDeleteActivity from './delete'; import follow from './follow'; import undo from './undo'; import { IObject } from '../type'; -import { IUser } from '../../../models/user'; +import { IRemoteUser } from '../../../models/user'; -export default async (actor: IUser, activity: IObject): Promise => { +export default async (actor: IRemoteUser, activity: IObject): Promise => { switch (activity.type) { case 'Create': await create(actor, activity); From 4e9ae8e8d5c862ac7a72f56d7bc0b7dab9c81044 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 19:35:23 +0900 Subject: [PATCH 32/58] Split code --- src/remote/activitypub/act/create.ts | 108 ------------------ src/remote/activitypub/act/create/image.ts | 19 +++ src/remote/activitypub/act/create/index.ts | 45 ++++++++ src/remote/activitypub/act/create/note.ts | 60 ++++++++++ .../act/{delete.ts => delete/index.ts} | 14 +-- src/remote/activitypub/act/delete/note.ts | 11 ++ 6 files changed, 137 insertions(+), 120 deletions(-) delete mode 100644 src/remote/activitypub/act/create.ts create mode 100644 src/remote/activitypub/act/create/image.ts create mode 100644 src/remote/activitypub/act/create/index.ts create mode 100644 src/remote/activitypub/act/create/note.ts rename src/remote/activitypub/act/{delete.ts => delete/index.ts} (50%) create mode 100644 src/remote/activitypub/act/delete/note.ts diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts deleted file mode 100644 index d1eeacbc3..000000000 --- a/src/remote/activitypub/act/create.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { JSDOM } from 'jsdom'; -import * as debug from 'debug'; - -import Resolver from '../resolver'; -import Post from '../../../models/post'; -import uploadFromUrl from '../../../services/drive/upload-from-url'; -import createPost from '../../../services/post/create'; -import { IRemoteUser, isRemoteUser } from '../../../models/user'; -import resolvePerson from '../resolve-person'; - -const log = debug('misskey:activitypub'); - -export default async (actor: IRemoteUser, activity): Promise => { - if ('actor' in activity && actor.account.uri !== activity.actor) { - throw new Error('invalid actor'); - } - - const uri = activity.id || activity; - - log(`Create: ${uri}`); - - // TODO: 同じURIをもつものが既に登録されていないかチェック - - const resolver = new Resolver(); - - let object; - - try { - object = await resolver.resolve(activity.object); - } catch (e) { - log(`Resolution failed: ${e}`); - throw e; - } - - switch (object.type) { - case 'Image': - createImage(resolver, actor, object); - break; - - case 'Note': - createNote(resolver, actor, object); - break; - - default: - console.warn(`Unknown type: ${object.type}`); - break; - } -}; - -async function createImage(resolver: Resolver, actor: IRemoteUser, image) { - if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { - log(`invalid image: ${JSON.stringify(image, null, 2)}`); - throw new Error('invalid image'); - } - - log(`Creating the Image: ${image.id}`); - - return await uploadFromUrl(image.url, actor); -} - -async function createNote(resolver: Resolver, actor: IRemoteUser, note) { - if ( - ('attributedTo' in note && actor.account.uri !== note.attributedTo) || - typeof note.id !== 'string' - ) { - log(`invalid note: ${JSON.stringify(note, null, 2)}`); - throw new Error('invalid note'); - } - - log(`Creating the Note: ${note.id}`); - - const media = []; - if ('attachment' in note && note.attachment != null) { - // TODO: attachmentは必ずしもImageではない - // TODO: ループの中でawaitはすべきでない - note.attachment.forEach(async media => { - const created = await createImage(resolver, note.actor, media); - media.push(created); - }); - } - - let reply = null; - if ('inReplyTo' in note && note.inReplyTo != null) { - const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); - if (inReplyToPost) { - reply = inReplyToPost; - } else { - const inReplyTo = await resolver.resolve(note.inReplyTo) as any; - const actor = await resolvePerson(inReplyTo.attributedTo); - if (isRemoteUser(actor)) { - reply = await createNote(resolver, actor, inReplyTo); - } - } - } - - const { window } = new JSDOM(note.content); - - return await createPost(actor, { - createdAt: new Date(note.published), - media, - reply, - repost: undefined, - text: window.document.body.textContent, - viaMobile: false, - geo: undefined, - uri: note.id - }); -} diff --git a/src/remote/activitypub/act/create/image.ts b/src/remote/activitypub/act/create/image.ts new file mode 100644 index 000000000..cd9e7b4e0 --- /dev/null +++ b/src/remote/activitypub/act/create/image.ts @@ -0,0 +1,19 @@ +import * as debug from 'debug'; + +import Resolver from '../../resolver'; +import uploadFromUrl from '../../../../services/drive/upload-from-url'; +import { IRemoteUser } from '../../../../models/user'; +import { IDriveFile } from '../../../../models/drive-file'; + +const log = debug('misskey:activitypub'); + +export default async function(resolver: Resolver, actor: IRemoteUser, image): Promise { + if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { + log(`invalid image: ${JSON.stringify(image, null, 2)}`); + throw new Error('invalid image'); + } + + log(`Creating the Image: ${image.id}`); + + return await uploadFromUrl(image.url, actor); +} diff --git a/src/remote/activitypub/act/create/index.ts b/src/remote/activitypub/act/create/index.ts new file mode 100644 index 000000000..d210aa4c5 --- /dev/null +++ b/src/remote/activitypub/act/create/index.ts @@ -0,0 +1,45 @@ +import * as debug from 'debug'; + +import Resolver from '../../resolver'; +import { IRemoteUser } from '../../../../models/user'; +import createNote from './note'; +import createImage from './image'; + +const log = debug('misskey:activitypub'); + +export default async (actor: IRemoteUser, activity): Promise => { + if ('actor' in activity && actor.account.uri !== activity.actor) { + throw new Error('invalid actor'); + } + + const uri = activity.id || activity; + + log(`Create: ${uri}`); + + // TODO: 同じURIをもつものが既に登録されていないかチェック + + const resolver = new Resolver(); + + let object; + + try { + object = await resolver.resolve(activity.object); + } catch (e) { + log(`Resolution failed: ${e}`); + throw e; + } + + switch (object.type) { + case 'Image': + createImage(resolver, actor, object); + break; + + case 'Note': + createNote(resolver, actor, object); + break; + + default: + console.warn(`Unknown type: ${object.type}`); + break; + } +}; diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts new file mode 100644 index 000000000..2ccd503ae --- /dev/null +++ b/src/remote/activitypub/act/create/note.ts @@ -0,0 +1,60 @@ +import { JSDOM } from 'jsdom'; +import * as debug from 'debug'; + +import Resolver from '../../resolver'; +import Post, { IPost } from '../../../../models/post'; +import createPost from '../../../../services/post/create'; +import { IRemoteUser, isRemoteUser } from '../../../../models/user'; +import resolvePerson from '../../resolve-person'; +import createImage from './image'; + +const log = debug('misskey:activitypub'); + +export default async function createNote(resolver: Resolver, actor: IRemoteUser, note): Promise { + if ( + ('attributedTo' in note && actor.account.uri !== note.attributedTo) || + typeof note.id !== 'string' + ) { + log(`invalid note: ${JSON.stringify(note, null, 2)}`); + throw new Error('invalid note'); + } + + log(`Creating the Note: ${note.id}`); + + const media = []; + if ('attachment' in note && note.attachment != null) { + // TODO: attachmentは必ずしもImageではない + // TODO: ループの中でawaitはすべきでない + note.attachment.forEach(async media => { + const created = await createImage(resolver, note.actor, media); + media.push(created); + }); + } + + let reply = null; + if ('inReplyTo' in note && note.inReplyTo != null) { + const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); + if (inReplyToPost) { + reply = inReplyToPost; + } else { + const inReplyTo = await resolver.resolve(note.inReplyTo) as any; + const actor = await resolvePerson(inReplyTo.attributedTo); + if (isRemoteUser(actor)) { + reply = await createNote(resolver, actor, inReplyTo); + } + } + } + + const { window } = new JSDOM(note.content); + + return await createPost(actor, { + createdAt: new Date(note.published), + media, + reply, + repost: undefined, + text: window.document.body.textContent, + viaMobile: false, + geo: undefined, + uri: note.id + }); +} diff --git a/src/remote/activitypub/act/delete.ts b/src/remote/activitypub/act/delete/index.ts similarity index 50% rename from src/remote/activitypub/act/delete.ts rename to src/remote/activitypub/act/delete/index.ts index 334ca47ed..764814bac 100644 --- a/src/remote/activitypub/act/delete.ts +++ b/src/remote/activitypub/act/delete/index.ts @@ -1,6 +1,5 @@ -import Resolver from '../resolver'; -import Post from '../../../models/post'; -import { createDb } from '../../../queue'; +import Resolver from '../../resolver'; +import deleteNote from './note'; export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { @@ -16,13 +15,4 @@ export default async (actor, activity): Promise => { deleteNote(object); break; } - - async function deleteNote(note) { - const post = await Post.findOneAndDelete({ uri: note.id }); - - createDb({ - type: 'deletePostDependents', - id: post._id - }).delay(65536).save(); - } }; diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts new file mode 100644 index 000000000..3b821f87c --- /dev/null +++ b/src/remote/activitypub/act/delete/note.ts @@ -0,0 +1,11 @@ +import Post from '../../../../models/post'; +import { createDb } from '../../../../queue'; + +export default async function(note) { + const post = await Post.findOneAndDelete({ uri: note.id }); + + createDb({ + type: 'deletePostDependents', + id: post._id + }).delay(65536).save(); +} From a01251477ee5ab4766810453dd540170e65c02b2 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 20:45:33 +0900 Subject: [PATCH 33/58] Revert "Remove silent flag" This reverts commit 9c15c94f801de7019f014446aa3b8ea42980a1da. --- src/remote/activitypub/act/create/note.ts | 2 +- src/services/post/create.ts | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index 2ccd503ae..d50042e16 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -10,7 +10,7 @@ import createImage from './image'; const log = debug('misskey:activitypub'); -export default async function createNote(resolver: Resolver, actor: IRemoteUser, note): Promise { +export default async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = false): Promise { if ( ('attributedTo' in note && actor.account.uri !== note.attributedTo) || typeof note.id !== 'string' diff --git a/src/services/post/create.ts b/src/services/post/create.ts index 0bede2772..745683b51 100644 --- a/src/services/post/create.ts +++ b/src/services/post/create.ts @@ -31,7 +31,7 @@ export default async (user: IUser, data: { visibility?: string; uri?: string; app?: IApp; -}) => new Promise(async (res, rej) => { +}, silent = false) => new Promise(async (res, rej) => { if (data.createdAt == null) data.createdAt = new Date(); if (data.visibility == null) data.visibility = 'public'; @@ -127,10 +127,7 @@ export default async (user: IUser, data: { _id: false }); - // この投稿が3分以内に作成されたものであるならストリームに配信 - const shouldDistribute = new Date().getTime() - post.createdAt.getTime() < 1000 * 60 * 3; - - if (shouldDistribute) { + if (!silent) { const note = await renderNote(user, post); const content = renderCreate(note); content['@context'] = context; From f640a8fd5b095534a3230bc22c3adb2094c904d3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 20:52:38 +0900 Subject: [PATCH 34/58] Resolve local Person ID --- src/remote/activitypub/resolve-person.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 907f19834..9cca50e41 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -1,11 +1,20 @@ import { JSDOM } from 'jsdom'; import { toUnicode } from 'punycode'; +import parseAcct from '../../acct/parse'; +import config from '../../config'; import User, { validateUsername, isValidName, isValidDescription } from '../../models/user'; import webFinger from '../webfinger'; import Resolver from './resolver'; import uploadFromUrl from '../../services/drive/upload-from-url'; export default async (value, verifier?: string) => { + const id = value.id || value; + const localPrefix = config.url + '/@'; + + if (id.startsWith(localPrefix)) { + return User.findOne(parseAcct(id.slice(localPrefix))); + } + const resolver = new Resolver(); const object = await resolver.resolve(value) as any; @@ -21,7 +30,7 @@ export default async (value, verifier?: string) => { throw new Error('invalid person'); } - const finger = await webFinger(object.id, verifier); + const finger = await webFinger(id, verifier); const host = toUnicode(finger.subject.replace(/^.*?@/, '')); const hostLower = host.replace(/[A-Z]+/, matched => matched.toLowerCase()); @@ -48,7 +57,7 @@ export default async (value, verifier?: string) => { publicKeyPem: object.publicKey.publicKeyPem }, inbox: object.inbox, - uri: object.id, + uri: id, }, }); From bc06c66407b1720047826572b6ea955ff751e3de Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 22:00:37 +0900 Subject: [PATCH 35/58] Add todo --- src/remote/activitypub/act/create/note.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index d50042e16..8b22f1b47 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -40,6 +40,7 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, const inReplyTo = await resolver.resolve(note.inReplyTo) as any; const actor = await resolvePerson(inReplyTo.attributedTo); if (isRemoteUser(actor)) { + // TODO: silentを常にtrueにしてはならない reply = await createNote(resolver, actor, inReplyTo); } } From 2b02655a3f420ddf08015e7c048d4eda12290d1d Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 22:15:31 +0900 Subject: [PATCH 36/58] Fix bug --- src/models/user.ts | 2 +- src/remote/activitypub/resolve-person.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/user.ts b/src/models/user.ts index f817c33aa..7c1ee498d 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -100,7 +100,7 @@ export function validatePassword(password: string): boolean { } export function isValidName(name: string): boolean { - return typeof name == 'string' && name.length < 30 && name.trim() != ''; + return name === null || (typeof name == 'string' && name.length < 30 && name.trim() != ''); } export function isValidDescription(description: string): boolean { diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 9cca50e41..39887ef77 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -24,7 +24,7 @@ export default async (value, verifier?: string) => { object.type !== 'Person' || typeof object.preferredUsername !== 'string' || !validateUsername(object.preferredUsername) || - (object.name != '' && !isValidName(object.name)) || + !isValidName(object.name == '' ? null : object.name) || !isValidDescription(object.summary) ) { throw new Error('invalid person'); From 3cf6eab11945e5b2577ef67117a51a210529e456 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 22:40:06 +0900 Subject: [PATCH 37/58] Log --- src/queue/processors/http/process-inbox.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index 4666e7f37..eb4b62d37 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -1,4 +1,5 @@ import * as kue from 'kue'; +import * as debug from 'debug'; import { verifySignature } from 'http-signature'; import parseAcct from '../../../acct/parse'; @@ -6,11 +7,20 @@ import User, { IRemoteUser } from '../../../models/user'; import act from '../../../remote/activitypub/act'; import resolvePerson from '../../../remote/activitypub/resolve-person'; +const log = debug('misskey:queue:inbox'); + // ユーザーのinboxにアクティビティが届いた時の処理 export default async (job: kue.Job, done): Promise => { const signature = job.data.signature; const activity = job.data.activity; + //#region Log + const info = Object.assign({}, activity); + delete info['@context']; + delete info['signature']; + log(info); + //#endregion + const keyIdLower = signature.keyId.toLowerCase(); let user; From 85d98f1e37ca35fb9b2785ed7794f17af6bb1f12 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Apr 2018 23:32:38 +0900 Subject: [PATCH 38/58] Add index --- src/models/post-watching.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/post-watching.ts b/src/models/post-watching.ts index b4ddcaafa..032b9d10f 100644 --- a/src/models/post-watching.ts +++ b/src/models/post-watching.ts @@ -2,6 +2,7 @@ import * as mongo from 'mongodb'; import db from '../db/mongodb'; const PostWatching = db.get('postWatching'); +PostWatching.createIndex(['userId', 'postId'], { unique: true }); export default PostWatching; export interface IPostWatching { From ef30390e768130598fd3e55736f2a33b189c5b24 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 00:07:30 +0900 Subject: [PATCH 39/58] Add todo --- src/remote/activitypub/act/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index c1d64b7c7..5be07c478 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -5,7 +5,7 @@ import undo from './undo'; import { IObject } from '../type'; import { IRemoteUser } from '../../../models/user'; -export default async (actor: IRemoteUser, activity: IObject): Promise => { +const self = async (actor: IRemoteUser, activity: IObject): Promise => { switch (activity.type) { case 'Create': await create(actor, activity); @@ -27,8 +27,15 @@ export default async (actor: IRemoteUser, activity: IObject): Promise => { await undo(actor, activity); break; + case 'Collection': + case 'OrderedCollection': + // TODO + break; + default: console.warn(`unknown activity type: ${activity.type}`); return null; } }; + +export default self; From ba1a81dab13b7f4750945c8ce4a29e3076267c6d Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:13:40 +0900 Subject: [PATCH 40/58] Bug fixes and some refactors --- src/remote/activitypub/act/create/index.ts | 2 -- src/remote/activitypub/act/create/note.ts | 40 ++++++++++++++++------ src/remote/activitypub/renderer/note.ts | 4 +-- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/remote/activitypub/act/create/index.ts b/src/remote/activitypub/act/create/index.ts index d210aa4c5..7ab4c2aba 100644 --- a/src/remote/activitypub/act/create/index.ts +++ b/src/remote/activitypub/act/create/index.ts @@ -16,8 +16,6 @@ export default async (actor: IRemoteUser, activity): Promise => { log(`Create: ${uri}`); - // TODO: 同じURIをもつものが既に登録されていないかチェック - const resolver = new Resolver(); let object; diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index 8b22f1b47..253478b6f 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -4,23 +4,31 @@ import * as debug from 'debug'; import Resolver from '../../resolver'; import Post, { IPost } from '../../../../models/post'; import createPost from '../../../../services/post/create'; -import { IRemoteUser, isRemoteUser } from '../../../../models/user'; +import { IRemoteUser } from '../../../../models/user'; import resolvePerson from '../../resolve-person'; import createImage from './image'; +import config from '../../../../config'; const log = debug('misskey:activitypub'); +/** + * 投稿作成アクティビティを捌きます + */ export default async function createNote(resolver: Resolver, actor: IRemoteUser, note, silent = false): Promise { - if ( - ('attributedTo' in note && actor.account.uri !== note.attributedTo) || - typeof note.id !== 'string' - ) { + if (typeof note.id !== 'string') { log(`invalid note: ${JSON.stringify(note, null, 2)}`); throw new Error('invalid note'); } + // 既に同じURIを持つものが登録されていないかチェックし、登録されていたらそれを返す + const exist = await Post.findOne({ uri: note.id }); + if (exist) { + return exist; + } + log(`Creating the Note: ${note.id}`); + //#region 添付メディア const media = []; if ('attachment' in note && note.attachment != null) { // TODO: attachmentは必ずしもImageではない @@ -30,21 +38,31 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, media.push(created); }); } + //#endregion + //#region リプライ let reply = null; if ('inReplyTo' in note && note.inReplyTo != null) { - const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); + // リプライ先の投稿がMisskeyに登録されているか調べる + const uri: string = note.inReplyTo.id || note.inReplyTo; + const inReplyToPost = uri.startsWith(config.url + '/') + ? await Post.findOne({ _id: uri.split('/').pop() }) + : await Post.findOne({ uri }); + if (inReplyToPost) { reply = inReplyToPost; } else { + // 無かったらフェッチ const inReplyTo = await resolver.resolve(note.inReplyTo) as any; - const actor = await resolvePerson(inReplyTo.attributedTo); - if (isRemoteUser(actor)) { - // TODO: silentを常にtrueにしてはならない - reply = await createNote(resolver, actor, inReplyTo); - } + + // リプライ先の投稿の投稿者をフェッチ + const actor = await resolvePerson(inReplyTo.attributedTo) as IRemoteUser; + + // TODO: silentを常にtrueにしてはならない + reply = await createNote(resolver, actor, inReplyTo); } } + //#endregion const { window } = new JSDOM(note.content); diff --git a/src/remote/activitypub/renderer/note.ts b/src/remote/activitypub/renderer/note.ts index e45b10215..b971a5395 100644 --- a/src/remote/activitypub/renderer/note.ts +++ b/src/remote/activitypub/renderer/note.ts @@ -19,11 +19,11 @@ export default async (user: IUser, post: IPost) => { if (inReplyToPost !== null) { const inReplyToUser = await User.findOne({ - _id: post.userId, + _id: inReplyToPost.userId, }); if (inReplyToUser !== null) { - inReplyTo = `${config.url}@${inReplyToUser.username}/${inReplyToPost._id}`; + inReplyTo = inReplyToPost.uri || `${config.url}/@${inReplyToUser.username}/${inReplyToPost._id}`; } } } else { From 66346495e56e07a5b167a8f94ed971f081c94e88 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:22:03 +0900 Subject: [PATCH 41/58] Fix bug --- src/queue/processors/http/deliver.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index da7e8bc36..f5d162fd0 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -6,9 +6,14 @@ export default async (job: kue.Job, done): Promise => { try { await request(job.data.user, job.data.to, job.data.content); done(); - } catch (e) { - console.warn(`deliver failed: ${e}`); - - done(e); + } catch (res) { + if (res.statusCode >= 300 && res.statusCode < 400) { + // HTTPステータスコード4xxはクライアントエラーであり、それはつまり + // 何回再送しても成功することはないということなのでエラーにはしないでおく + done(); + } else { + console.warn(`deliver failed: ${res.statusMessage}`); + done(new Error(res.statusMessage)); + } } }; From 866797a0058a0eac1e5c1d7f6cdeeab845a505ae Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:23:38 +0900 Subject: [PATCH 42/58] Fix bug --- src/queue/processors/http/deliver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index f5d162fd0..422e355b5 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -7,7 +7,7 @@ export default async (job: kue.Job, done): Promise => { await request(job.data.user, job.data.to, job.data.content); done(); } catch (res) { - if (res.statusCode >= 300 && res.statusCode < 400) { + if (res.statusCode >= 400 && res.statusCode < 500) { // HTTPステータスコード4xxはクライアントエラーであり、それはつまり // 何回再送しても成功することはないということなのでエラーにはしないでおく done(); From 765a10c8da6b64268b327b32bfd23d8f75bc9f80 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:25:58 +0900 Subject: [PATCH 43/58] Increase limit to avoid warning --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index f45bcaa6a..68b289793 100644 --- a/src/index.ts +++ b/src/index.ts @@ -35,7 +35,7 @@ if (process.env.NODE_ENV != 'production') { } // https://github.com/Automattic/kue/issues/822 -require('events').EventEmitter.prototype._maxListeners = 256; +require('events').EventEmitter.prototype._maxListeners = 512; // Start app main(); From de620c822aa6386f2c706f62e0cfd6b8d9449ab6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:44:29 +0900 Subject: [PATCH 44/58] Fix bug --- src/remote/activitypub/act/delete/index.ts | 23 +++++++++++++++++++--- src/remote/activitypub/act/delete/note.ts | 10 ++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/remote/activitypub/act/delete/index.ts b/src/remote/activitypub/act/delete/index.ts index 764814bac..42272433d 100644 --- a/src/remote/activitypub/act/delete/index.ts +++ b/src/remote/activitypub/act/delete/index.ts @@ -1,18 +1,35 @@ import Resolver from '../../resolver'; import deleteNote from './note'; +import Post from '../../../../models/post'; +/** + * 削除アクティビティを捌きます + */ export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { - throw new Error(); + throw new Error('invalid actor'); } const resolver = new Resolver(); - const object = await resolver.resolve(activity); + const object = await resolver.resolve(activity.object); + + const uri = (object as any).id; switch (object.type) { case 'Note': - deleteNote(object); + deleteNote(uri); + break; + + case 'Tombstone': + const post = await Post.findOne({ uri }); + if (post != null) { + deleteNote(uri); + } + break; + + default: + console.warn(`Unknown type: ${object.type}`); break; } }; diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts index 3b821f87c..75534250e 100644 --- a/src/remote/activitypub/act/delete/note.ts +++ b/src/remote/activitypub/act/delete/note.ts @@ -1,8 +1,14 @@ +import * as debug from 'debug'; + import Post from '../../../../models/post'; import { createDb } from '../../../../queue'; -export default async function(note) { - const post = await Post.findOneAndDelete({ uri: note.id }); +const log = debug('misskey:activitypub'); + +export default async function(uri: string) { + log(`Deleting the Note: ${uri}`); + + const post = await Post.findOneAndDelete({ uri }); createDb({ type: 'deletePostDependents', From a0c6e7af1c3a783cf82ba836b6a74037ecb40740 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:51:35 +0900 Subject: [PATCH 45/58] Fix bug --- src/remote/activitypub/act/delete/index.ts | 4 ++-- src/remote/activitypub/act/delete/note.ts | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/remote/activitypub/act/delete/index.ts b/src/remote/activitypub/act/delete/index.ts index 42272433d..8163ffc32 100644 --- a/src/remote/activitypub/act/delete/index.ts +++ b/src/remote/activitypub/act/delete/index.ts @@ -18,13 +18,13 @@ export default async (actor, activity): Promise => { switch (object.type) { case 'Note': - deleteNote(uri); + deleteNote(actor, uri); break; case 'Tombstone': const post = await Post.findOne({ uri }); if (post != null) { - deleteNote(uri); + deleteNote(actor, uri); } break; diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts index 75534250e..5306b705e 100644 --- a/src/remote/activitypub/act/delete/note.ts +++ b/src/remote/activitypub/act/delete/note.ts @@ -5,10 +5,20 @@ import { createDb } from '../../../../queue'; const log = debug('misskey:activitypub'); -export default async function(uri: string) { +export default async function(actor, uri: string) { log(`Deleting the Note: ${uri}`); - const post = await Post.findOneAndDelete({ uri }); + const post = await Post.findOne({ uri }); + + if (post == null) { + throw new Error('post not found'); + } + + if (post.userId !== actor._id) { + throw new Error('投稿を削除しようとしているユーザーは投稿の作成者ではありません'); + } + + Post.remove({ _id: post._id }); createDb({ type: 'deletePostDependents', From b98e67bca4a4381c70ebba6ee1cd34adc242a3ad Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:58:39 +0900 Subject: [PATCH 46/58] Fix bug --- src/server/web/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/web/index.ts b/src/server/web/index.ts index 1445d1aef..5b1b6409b 100644 --- a/src/server/web/index.ts +++ b/src/server/web/index.ts @@ -11,7 +11,7 @@ import * as bodyParser from 'body-parser'; import * as favicon from 'serve-favicon'; import * as compression from 'compression'; -const client = `${__dirname}/../../client/`; +const client = path.resolve(`${__dirname}/../../client/`); // Create server const app = express(); From a34710fea93c55f35f5352c9a6d2855c4c89721d Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:58:53 +0900 Subject: [PATCH 47/58] Refactor --- src/remote/activitypub/act/delete/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/remote/activitypub/act/delete/index.ts b/src/remote/activitypub/act/delete/index.ts index 8163ffc32..e34577b31 100644 --- a/src/remote/activitypub/act/delete/index.ts +++ b/src/remote/activitypub/act/delete/index.ts @@ -1,11 +1,12 @@ import Resolver from '../../resolver'; import deleteNote from './note'; import Post from '../../../../models/post'; +import { IRemoteUser } from '../../../../models/user'; /** * 削除アクティビティを捌きます */ -export default async (actor, activity): Promise => { +export default async (actor: IRemoteUser, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } From 8273a7e7489e046036d253b246b6315131decdc4 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:59:20 +0900 Subject: [PATCH 48/58] Fix bug --- src/remote/activitypub/act/delete/note.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts index 5306b705e..ff9a8ee5f 100644 --- a/src/remote/activitypub/act/delete/note.ts +++ b/src/remote/activitypub/act/delete/note.ts @@ -2,10 +2,11 @@ import * as debug from 'debug'; import Post from '../../../../models/post'; import { createDb } from '../../../../queue'; +import { IRemoteUser } from '../../../../models/user'; const log = debug('misskey:activitypub'); -export default async function(actor, uri: string) { +export default async function(actor: IRemoteUser, uri: string): Promise { log(`Deleting the Note: ${uri}`); const post = await Post.findOne({ uri }); @@ -14,7 +15,7 @@ export default async function(actor, uri: string) { throw new Error('post not found'); } - if (post.userId !== actor._id) { + if (!post.userId.equals(actor._id)) { throw new Error('投稿を削除しようとしているユーザーは投稿の作成者ではありません'); } From 494597236cea4a40bddd9655b6506464df053bfe Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 07:19:30 +0900 Subject: [PATCH 49/58] =?UTF-8?q?=E6=8A=95=E7=A8=BF=E3=81=AB=E9=96=A2?= =?UTF-8?q?=E3=81=97=E3=81=A6=E3=81=AF=E8=AB=96=E7=90=86=E5=89=8A=E9=99=A4?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 処理をシンプルにするため --- src/models/post.ts | 1 + src/queue/index.ts | 7 ------ .../processors/db/delete-post-dependents.ts | 22 ------------------- src/queue/processors/db/index.ts | 7 ------ src/remote/activitypub/act/delete/note.ts | 16 ++++++++------ 5 files changed, 10 insertions(+), 43 deletions(-) delete mode 100644 src/queue/processors/db/delete-post-dependents.ts delete mode 100644 src/queue/processors/db/index.ts diff --git a/src/models/post.ts b/src/models/post.ts index 68a638fa2..ac7890d2e 100644 --- a/src/models/post.ts +++ b/src/models/post.ts @@ -27,6 +27,7 @@ export type IPost = { _id: mongo.ObjectID; channelId: mongo.ObjectID; createdAt: Date; + deletedAt: Date; mediaIds: mongo.ObjectID[]; replyId: mongo.ObjectID; repostId: mongo.ObjectID; diff --git a/src/queue/index.ts b/src/queue/index.ts index 691223de2..4aa1dc032 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,7 +1,6 @@ import { createQueue } from 'kue'; import config from '../config'; -import db from './processors/db'; import http from './processors/http'; const queue = createQueue({ @@ -19,10 +18,6 @@ export function createHttp(data) { .backoff({ delay: 16384, type: 'exponential' }); } -export function createDb(data) { - return queue.create('db', data); -} - export function deliver(user, content, to) { return createHttp({ type: 'deliver', @@ -33,8 +28,6 @@ export function deliver(user, content, to) { } export default function() { - queue.process('db', db); - /* 256 is the default concurrency limit of Mozilla Firefox and Google Chromium. diff --git a/src/queue/processors/db/delete-post-dependents.ts b/src/queue/processors/db/delete-post-dependents.ts deleted file mode 100644 index 6de21eb05..000000000 --- a/src/queue/processors/db/delete-post-dependents.ts +++ /dev/null @@ -1,22 +0,0 @@ -import Favorite from '../../../models/favorite'; -import Notification from '../../../models/notification'; -import PollVote from '../../../models/poll-vote'; -import PostReaction from '../../../models/post-reaction'; -import PostWatching from '../../../models/post-watching'; -import Post from '../../../models/post'; - -export default async ({ data }) => Promise.all([ - Favorite.remove({ postId: data._id }), - Notification.remove({ postId: data._id }), - PollVote.remove({ postId: data._id }), - PostReaction.remove({ postId: data._id }), - PostWatching.remove({ postId: data._id }), - Post.find({ repostId: data._id }).then(reposts => Promise.all([ - Notification.remove({ - postId: { - $in: reposts.map(({ _id }) => _id) - } - }), - Post.remove({ repostId: data._id }) - ])) -]); diff --git a/src/queue/processors/db/index.ts b/src/queue/processors/db/index.ts deleted file mode 100644 index 75838c099..000000000 --- a/src/queue/processors/db/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -import deletePostDependents from './delete-post-dependents'; - -const handlers = { - deletePostDependents -}; - -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts index ff9a8ee5f..8e9447b48 100644 --- a/src/remote/activitypub/act/delete/note.ts +++ b/src/remote/activitypub/act/delete/note.ts @@ -1,7 +1,6 @@ import * as debug from 'debug'; import Post from '../../../../models/post'; -import { createDb } from '../../../../queue'; import { IRemoteUser } from '../../../../models/user'; const log = debug('misskey:activitypub'); @@ -19,10 +18,13 @@ export default async function(actor: IRemoteUser, uri: string): Promise { throw new Error('投稿を削除しようとしているユーザーは投稿の作成者ではありません'); } - Post.remove({ _id: post._id }); - - createDb({ - type: 'deletePostDependents', - id: post._id - }).delay(65536).save(); + Post.update({ _id: post._id }, { + $set: { + deletedAt: new Date(), + text: null, + textHtml: null, + mediaIds: [], + poll: null + } + }); } From 779a37c4ae0f9442c46e1dd847153c4d46f36088 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 07:27:18 +0900 Subject: [PATCH 50/58] Visibility support --- src/remote/activitypub/act/create/note.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index 253478b6f..88e3a875a 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -28,6 +28,11 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, log(`Creating the Note: ${note.id}`); + //#region Visibility + let visibility = 'public'; + if (note.cc.length == 0) visibility = 'private'; + //#endergion + //#region 添付メディア const media = []; if ('attachment' in note && note.attachment != null) { @@ -74,6 +79,7 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, text: window.document.body.textContent, viaMobile: false, geo: undefined, + visibility, uri: note.id }); } From e68dd11eccb0adfe957e598c5e385035d4e493da Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 12:02:25 +0900 Subject: [PATCH 51/58] Support unlisted visibility type --- src/remote/activitypub/act/create/note.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index 88e3a875a..df9f1d69e 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -30,6 +30,7 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, //#region Visibility let visibility = 'public'; + if (!note.to.includes('https://www.w3.org/ns/activitystreams#Public')) visibility = 'unlisted'; if (note.cc.length == 0) visibility = 'private'; //#endergion From ac5076c6782db4249b31b64318beec5dbce2f37a Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 12:05:15 +0900 Subject: [PATCH 52/58] Ignore post that not public --- src/remote/activitypub/act/create/note.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index df9f1d69e..c40facea4 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -32,6 +32,8 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, let visibility = 'public'; if (!note.to.includes('https://www.w3.org/ns/activitystreams#Public')) visibility = 'unlisted'; if (note.cc.length == 0) visibility = 'private'; + // TODO + if (visibility != 'public') throw new Error('unspported visibility'); //#endergion //#region 添付メディア From 62171dce226164cfe263c8a44153b72eb4b1c118 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 13:05:10 +0900 Subject: [PATCH 53/58] oops --- src/remote/resolve-user.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/remote/resolve-user.ts b/src/remote/resolve-user.ts index edb4c7860..9e1ae5195 100644 --- a/src/remote/resolve-user.ts +++ b/src/remote/resolve-user.ts @@ -1,7 +1,6 @@ import { toUnicode, toASCII } from 'punycode'; import User from '../models/user'; import resolvePerson from './activitypub/resolve-person'; -import Resolver from './activitypub/resolver'; import webFinger from './webfinger'; export default async (username, host, option) => { @@ -20,7 +19,7 @@ export default async (username, host, option) => { throw new Error('self link not found'); } - user = await resolvePerson(new Resolver(), self.href, acctLower); + user = await resolvePerson(self.href, acctLower); } return user; From 2259747072911ea1b054648dc73ab10caba221b3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 15:39:40 +0900 Subject: [PATCH 54/58] =?UTF-8?q?=E5=90=84=E7=A8=AE=E3=82=AB=E3=82=A6?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=82=92=E5=BE=A9=E6=B4=BB=E3=81=95=E3=81=9B?= =?UTF-8?q?=E3=81=9F=E3=82=8A=E3=81=AA=E3=81=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/remote/activitypub/resolve-person.ts | 39 +++++++++++++++--------- src/remote/activitypub/type.ts | 33 ++++++++++++++++++-- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 39887ef77..b3bac3cd3 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -6,6 +6,7 @@ import User, { validateUsername, isValidName, isValidDescription } from '../../m import webFinger from '../webfinger'; import Resolver from './resolver'; import uploadFromUrl from '../../services/drive/upload-from-url'; +import { isCollectionOrOrderedCollection } from './type'; export default async (value, verifier?: string) => { const id = value.id || value; @@ -30,7 +31,21 @@ export default async (value, verifier?: string) => { throw new Error('invalid person'); } - const finger = await webFinger(id, verifier); + const [followersCount = 0, followingCount = 0, postsCount = 0, finger] = await Promise.all([ + resolver.resolve(object.followers).then( + resolved => isCollectionOrOrderedCollection(resolved) ? resolved.totalItems : undefined, + () => undefined + ), + resolver.resolve(object.following).then( + resolved => isCollectionOrOrderedCollection(resolved) ? resolved.totalItems : undefined, + () => undefined + ), + resolver.resolve(object.outbox).then( + resolved => isCollectionOrOrderedCollection(resolved) ? resolved.totalItems : undefined, + () => undefined + ), + webFinger(id, verifier) + ]); const host = toUnicode(finger.subject.replace(/^.*?@/, '')); const hostLower = host.replace(/[A-Z]+/, matched => matched.toLowerCase()); @@ -42,10 +57,10 @@ export default async (value, verifier?: string) => { bannerId: null, createdAt: Date.parse(object.published) || null, description: summaryDOM.textContent, - followersCount: 0, - followingCount: 0, + followersCount, + followingCount, + postsCount, name: object.name, - postsCount: 0, driveCapacity: 1024 * 1024 * 8, // 8MiB username: object.preferredUsername, usernameLower: object.preferredUsername.toLowerCase(), @@ -61,18 +76,14 @@ export default async (value, verifier?: string) => { }, }); - const [avatarId, bannerId] = await Promise.all([ + const [avatarId, bannerId] = (await Promise.all([ object.icon, object.image - ].map(async img => { - if (img === undefined) { - return null; - } - - const file = await uploadFromUrl(img.url, user); - - return file._id; - })); + ].map(img => + img == null + ? Promise.resolve(null) + : uploadFromUrl(img.url, user) + ))).map(file => file != null ? file._id : null); User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); diff --git a/src/remote/activitypub/type.ts b/src/remote/activitypub/type.ts index 94e2c350a..cd7f40630 100644 --- a/src/remote/activitypub/type.ts +++ b/src/remote/activitypub/type.ts @@ -1,3 +1,32 @@ -export type IObject = { +export type Object = { [x: string]: any }; + +export type ActivityType = + 'Create'; + +export interface IObject { + '@context': string | object | any[]; type: string; -}; + id?: string; + summary?: string; +} + +export interface ICollection extends IObject { + type: 'Collection'; + totalItems: number; + items: IObject | string | IObject[] | string[]; +} + +export interface IOrderedCollection extends IObject { + type: 'OrderedCollection'; + totalItems: number; + orderedItems: IObject | string | IObject[] | string[]; +} + +export const isCollection = (object: IObject): object is ICollection => + object.type === 'Collection'; + +export const isOrderedCollection = (object: IObject): object is IOrderedCollection => + object.type === 'OrderedCollection'; + +export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => + isCollection(object) || isOrderedCollection(object); From c77013ab3ec748ccc441c497cd04231398194cc6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 15:45:03 +0900 Subject: [PATCH 55/58] oops --- src/queue/processors/http/index.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 0ea79305c..3dc259537 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -1,17 +1,20 @@ -import deliverPost from './deliver-post'; -import follow from './follow'; -import performActivityPub from './perform-activitypub'; +import deliver from './deliver'; import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; -import unfollow from './unfollow'; const handlers = { - deliverPost, - follow, - performActivityPub, - processInbox, - reportGitHubFailure, - unfollow + deliver, + processInbox, + reportGitHubFailure }; -export default (job, done) => handlers[job.data.type](job, done); +export default (job, done) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job, done); + } else { + console.error(`Unknown job: ${job.data.type}`); + done(); + } +}; From 81d19195cf2e913e9eaa2bef2ad4414e0c384be9 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 15:52:42 +0900 Subject: [PATCH 56/58] Add todo --- src/remote/activitypub/act/create/note.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index c40facea4..364fddfe0 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -40,6 +40,7 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, const media = []; if ('attachment' in note && note.attachment != null) { // TODO: attachmentは必ずしもImageではない + // TODO: attachmentは必ずしも配列ではない // TODO: ループの中でawaitはすべきでない note.attachment.forEach(async media => { const created = await createImage(resolver, note.actor, media); From 7dc06b3d4383321ef85fa9bf2a1bc1d16ecab8c2 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 15:54:11 +0900 Subject: [PATCH 57/58] Refactor --- src/remote/activitypub/act/undo/follow.ts | 25 +++++++++++++++++++ .../act/{undo.ts => undo/index.ts} | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/remote/activitypub/act/undo/follow.ts rename src/remote/activitypub/act/{undo.ts => undo/index.ts} (89%) diff --git a/src/remote/activitypub/act/undo/follow.ts b/src/remote/activitypub/act/undo/follow.ts new file mode 100644 index 000000000..b1c462a3b --- /dev/null +++ b/src/remote/activitypub/act/undo/follow.ts @@ -0,0 +1,25 @@ +import parseAcct from '../../../../acct/parse'; +import User from '../../../../models/user'; +import config from '../../../../config'; +import unfollow from '../../../../services/following/delete'; + +export default async (actor, activity): Promise => { + const prefix = config.url + '/@'; + const id = activity.object.id || activity.object; + + if (!id.startsWith(prefix)) { + return null; + } + + const { username, host } = parseAcct(id.slice(prefix.length)); + if (host !== null) { + throw new Error(); + } + + const followee = await User.findOne({ username, host }); + if (followee === null) { + throw new Error(); + } + + await unfollow(actor, followee, activity); +}; diff --git a/src/remote/activitypub/act/undo.ts b/src/remote/activitypub/act/undo/index.ts similarity index 89% rename from src/remote/activitypub/act/undo.ts rename to src/remote/activitypub/act/undo/index.ts index 9d9f6b035..ecd9944b4 100644 --- a/src/remote/activitypub/act/undo.ts +++ b/src/remote/activitypub/act/undo/index.ts @@ -1,4 +1,4 @@ -import unfollow from './unfollow'; +import unfollow from './follow'; export default async (actor, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { From 93f631e3586e0674dc177789bebca8ddd946c153 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 16:14:35 +0900 Subject: [PATCH 58/58] Refactor --- src/remote/activitypub/act/create/image.ts | 3 +-- src/remote/activitypub/act/create/index.ts | 5 ++-- src/remote/activitypub/act/create/note.ts | 2 +- src/remote/activitypub/act/follow.ts | 7 ++--- src/remote/activitypub/act/undo/follow.ts | 7 ++--- src/remote/activitypub/act/undo/index.ts | 30 +++++++++++++++++++--- src/remote/activitypub/act/unfollow.ts | 25 ------------------ src/remote/activitypub/type.ts | 22 +++++++++++++--- 8 files changed, 58 insertions(+), 43 deletions(-) delete mode 100644 src/remote/activitypub/act/unfollow.ts diff --git a/src/remote/activitypub/act/create/image.ts b/src/remote/activitypub/act/create/image.ts index cd9e7b4e0..30a75e737 100644 --- a/src/remote/activitypub/act/create/image.ts +++ b/src/remote/activitypub/act/create/image.ts @@ -1,13 +1,12 @@ import * as debug from 'debug'; -import Resolver from '../../resolver'; import uploadFromUrl from '../../../../services/drive/upload-from-url'; import { IRemoteUser } from '../../../../models/user'; import { IDriveFile } from '../../../../models/drive-file'; const log = debug('misskey:activitypub'); -export default async function(resolver: Resolver, actor: IRemoteUser, image): Promise { +export default async function(actor: IRemoteUser, image): Promise { if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { log(`invalid image: ${JSON.stringify(image, null, 2)}`); throw new Error('invalid image'); diff --git a/src/remote/activitypub/act/create/index.ts b/src/remote/activitypub/act/create/index.ts index 7ab4c2aba..dd0b11214 100644 --- a/src/remote/activitypub/act/create/index.ts +++ b/src/remote/activitypub/act/create/index.ts @@ -4,10 +4,11 @@ import Resolver from '../../resolver'; import { IRemoteUser } from '../../../../models/user'; import createNote from './note'; import createImage from './image'; +import { ICreate } from '../../type'; const log = debug('misskey:activitypub'); -export default async (actor: IRemoteUser, activity): Promise => { +export default async (actor: IRemoteUser, activity: ICreate): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } @@ -29,7 +30,7 @@ export default async (actor: IRemoteUser, activity): Promise => { switch (object.type) { case 'Image': - createImage(resolver, actor, object); + createImage(actor, object); break; case 'Note': diff --git a/src/remote/activitypub/act/create/note.ts b/src/remote/activitypub/act/create/note.ts index 364fddfe0..82a620703 100644 --- a/src/remote/activitypub/act/create/note.ts +++ b/src/remote/activitypub/act/create/note.ts @@ -43,7 +43,7 @@ export default async function createNote(resolver: Resolver, actor: IRemoteUser, // TODO: attachmentは必ずしも配列ではない // TODO: ループの中でawaitはすべきでない note.attachment.forEach(async media => { - const created = await createImage(resolver, note.actor, media); + const created = await createImage(note.actor, media); media.push(created); }); } diff --git a/src/remote/activitypub/act/follow.ts b/src/remote/activitypub/act/follow.ts index 4fc423d15..3dd029af5 100644 --- a/src/remote/activitypub/act/follow.ts +++ b/src/remote/activitypub/act/follow.ts @@ -1,11 +1,12 @@ import parseAcct from '../../../acct/parse'; -import User from '../../../models/user'; +import User, { IRemoteUser } from '../../../models/user'; import config from '../../../config'; import follow from '../../../services/following/create'; +import { IFollow } from '../type'; -export default async (actor, activity): Promise => { +export default async (actor: IRemoteUser, activity: IFollow): Promise => { const prefix = config.url + '/@'; - const id = activity.object.id || activity.object; + const id = typeof activity == 'string' ? activity : activity.id; if (!id.startsWith(prefix)) { return null; diff --git a/src/remote/activitypub/act/undo/follow.ts b/src/remote/activitypub/act/undo/follow.ts index b1c462a3b..fcf27c950 100644 --- a/src/remote/activitypub/act/undo/follow.ts +++ b/src/remote/activitypub/act/undo/follow.ts @@ -1,11 +1,12 @@ import parseAcct from '../../../../acct/parse'; -import User from '../../../../models/user'; +import User, { IRemoteUser } from '../../../../models/user'; import config from '../../../../config'; import unfollow from '../../../../services/following/delete'; +import { IFollow } from '../../type'; -export default async (actor, activity): Promise => { +export default async (actor: IRemoteUser, activity: IFollow): Promise => { const prefix = config.url + '/@'; - const id = activity.object.id || activity.object; + const id = typeof activity == 'string' ? activity : activity.id; if (!id.startsWith(prefix)) { return null; diff --git a/src/remote/activitypub/act/undo/index.ts b/src/remote/activitypub/act/undo/index.ts index ecd9944b4..3ede9fcfb 100644 --- a/src/remote/activitypub/act/undo/index.ts +++ b/src/remote/activitypub/act/undo/index.ts @@ -1,13 +1,35 @@ -import unfollow from './follow'; +import * as debug from 'debug'; -export default async (actor, activity): Promise => { +import { IRemoteUser } from '../../../../models/user'; +import { IUndo } from '../../type'; +import unfollow from './follow'; +import Resolver from '../../resolver'; + +const log = debug('misskey:activitypub'); + +export default async (actor: IRemoteUser, activity: IUndo): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } - switch (activity.object.type) { + const uri = activity.id || activity; + + log(`Undo: ${uri}`); + + const resolver = new Resolver(); + + let object; + + try { + object = await resolver.resolve(activity.object); + } catch (e) { + log(`Resolution failed: ${e}`); + throw e; + } + + switch (object.type) { case 'Follow': - unfollow(actor, activity.object); + unfollow(actor, object); break; } diff --git a/src/remote/activitypub/act/unfollow.ts b/src/remote/activitypub/act/unfollow.ts deleted file mode 100644 index 66c15e9a9..000000000 --- a/src/remote/activitypub/act/unfollow.ts +++ /dev/null @@ -1,25 +0,0 @@ -import parseAcct from '../../../acct/parse'; -import User from '../../../models/user'; -import config from '../../../config'; -import unfollow from '../../../services/following/delete'; - -export default async (actor, activity): Promise => { - const prefix = config.url + '/@'; - const id = activity.object.id || activity.object; - - if (!id.startsWith(prefix)) { - return null; - } - - const { username, host } = parseAcct(id.slice(prefix.length)); - if (host !== null) { - throw new Error(); - } - - const followee = await User.findOne({ username, host }); - if (followee === null) { - throw new Error(); - } - - await unfollow(actor, followee, activity); -}; diff --git a/src/remote/activitypub/type.ts b/src/remote/activitypub/type.ts index cd7f40630..9a4b3c75f 100644 --- a/src/remote/activitypub/type.ts +++ b/src/remote/activitypub/type.ts @@ -1,8 +1,5 @@ export type Object = { [x: string]: any }; -export type ActivityType = - 'Create'; - export interface IObject { '@context': string | object | any[]; type: string; @@ -10,6 +7,13 @@ export interface IObject { summary?: string; } +export interface IActivity extends IObject { + //type: 'Activity'; + actor: IObject | string; + object: IObject | string; + target?: IObject | string; +} + export interface ICollection extends IObject { type: 'Collection'; totalItems: number; @@ -30,3 +34,15 @@ export const isOrderedCollection = (object: IObject): object is IOrderedCollecti export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => isCollection(object) || isOrderedCollection(object); + +export interface ICreate extends IActivity { + type: 'Create'; +} + +export interface IUndo extends IActivity { + type: 'Undo'; +} + +export interface IFollow extends IActivity { + type: 'Follow'; +}