From 27d16c6a120d05e69eb069035a851c48c44548c5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 6 Feb 2019 20:56:48 +0900 Subject: [PATCH] Resolve #4151 --- locales/ja-JP.yml | 6 +- .../views/components/profile-editor.vue | 19 +++- src/queue/processors/export-blocking.ts | 90 +++++++++++++++++++ src/queue/processors/export-following.ts | 90 +++++++++++++++++++ src/queue/processors/export-mute.ts | 90 +++++++++++++++++++ src/queue/processors/index.ts | 6 ++ src/server/api/endpoints/i/export-blocking.ts | 18 ++++ .../api/endpoints/i/export-following.ts | 18 ++++ src/server/api/endpoints/i/export-mute.ts | 18 ++++ 9 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 src/queue/processors/export-blocking.ts create mode 100644 src/queue/processors/export-following.ts create mode 100644 src/queue/processors/export-mute.ts create mode 100644 src/server/api/endpoints/i/export-blocking.ts create mode 100644 src/server/api/endpoints/i/export-following.ts create mode 100644 src/server/api/endpoints/i/export-mute.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 77bcad3ff..41b891cf7 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -558,7 +558,11 @@ common/views/components/profile-editor.vue: email-verified: "メールアドレスが確認されました" email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。" export: "エクスポート" - export-notes: "すべての投稿のエクスポート" + export-targets: + all-notes: "すべての投稿データ" + following-list: "フォロー" + mute-list: "ミュート" + blocking-list: "ブロック" export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。" common/views/components/user-list-editor.vue: diff --git a/src/client/app/common/views/components/profile-editor.vue b/src/client/app/common/views/components/profile-editor.vue index a3b1c6d87..472124fa8 100644 --- a/src/client/app/common/views/components/profile-editor.vue +++ b/src/client/app/common/views/components/profile-editor.vue @@ -92,7 +92,14 @@
{{ $t('export') }}
- {{ $t('export-notes') }} + + {{ $t('export-target') }} + + + + + + {{ $t('export') }}
@@ -133,6 +140,7 @@ export default Vue.extend({ saving: false, avatarUploading: false, bannerUploading: false, + exportTarget: 'notes', faDownload }; }, @@ -264,8 +272,13 @@ export default Vue.extend({ }); }, - exportNotes() { - this.$root.api('i/export-notes', {}); + doExport() { + this.$root.api( + this.exportTarget == 'notes' ? 'i/export-notes' : + this.exportTarget == 'following' ? 'i/export-following' : + this.exportTarget == 'mute' ? 'i/export-mute' : + this.exportTarget == 'blocking' ? 'i/export-blocking' : + null, {}); this.$root.dialog({ type: 'info', diff --git a/src/queue/processors/export-blocking.ts b/src/queue/processors/export-blocking.ts new file mode 100644 index 000000000..b7a2d3122 --- /dev/null +++ b/src/queue/processors/export-blocking.ts @@ -0,0 +1,90 @@ + +import * as bq from 'bee-queue'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../logger'; +import addFile from '../../services/drive/add-file'; +import User from '../../models/user'; +import dateFormat = require('dateformat'); +import Blocking from '../../models/blocking'; +import config from '../../config'; + +const logger = queueLogger.createSubLogger('export-blocking'); + +export async function exportBlocking(job: bq.Job, done: any): Promise { + logger.info(`Exporting blocking of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let ended = false; + let cursor: any = null; + + while (!ended) { + const blockings = await Blocking.find({ + blockerId: user._id, + ...(cursor ? { _id: { $gt: cursor } } : {}) + }, { + limit: 100, + sort: { + _id: 1 + } + }); + + if (blockings.length === 0) { + ended = true; + job.reportProgress(100); + break; + } + + cursor = blockings[blockings.length - 1]._id; + + for (const block of blockings) { + const u = await User.findOne({ _id: block.blockeeId }, { fields: { username: true, host: true } }); + const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`; + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + exportedCount++; + } + + const total = await Blocking.count({ + blockerId: user._id, + }); + + job.reportProgress(exportedCount / total); + } + + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = dateFormat(new Date(), 'blocking-yyyy-mm-dd-HH-MM-ss') + '.json'; + const driveFile = await addFile(user, path, fileName); + + logger.succ(`Exported to: ${driveFile._id}`); + cleanup(); + done(); +} diff --git a/src/queue/processors/export-following.ts b/src/queue/processors/export-following.ts new file mode 100644 index 000000000..751592636 --- /dev/null +++ b/src/queue/processors/export-following.ts @@ -0,0 +1,90 @@ + +import * as bq from 'bee-queue'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../logger'; +import addFile from '../../services/drive/add-file'; +import User from '../../models/user'; +import dateFormat = require('dateformat'); +import Following from '../../models/following'; +import config from '../../config'; + +const logger = queueLogger.createSubLogger('export-following'); + +export async function exportFollowing(job: bq.Job, done: any): Promise { + logger.info(`Exporting following of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let ended = false; + let cursor: any = null; + + while (!ended) { + const followings = await Following.find({ + followerId: user._id, + ...(cursor ? { _id: { $gt: cursor } } : {}) + }, { + limit: 100, + sort: { + _id: 1 + } + }); + + if (followings.length === 0) { + ended = true; + job.reportProgress(100); + break; + } + + cursor = followings[followings.length - 1]._id; + + for (const following of followings) { + const u = await User.findOne({ _id: following.followeeId }, { fields: { username: true, host: true } }); + const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`; + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + exportedCount++; + } + + const total = await Following.count({ + followerId: user._id, + }); + + job.reportProgress(exportedCount / total); + } + + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = dateFormat(new Date(), 'following-yyyy-mm-dd-HH-MM-ss') + '.json'; + const driveFile = await addFile(user, path, fileName); + + logger.succ(`Exported to: ${driveFile._id}`); + cleanup(); + done(); +} diff --git a/src/queue/processors/export-mute.ts b/src/queue/processors/export-mute.ts new file mode 100644 index 000000000..e526aa06f --- /dev/null +++ b/src/queue/processors/export-mute.ts @@ -0,0 +1,90 @@ + +import * as bq from 'bee-queue'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as mongo from 'mongodb'; + +import { queueLogger } from '../logger'; +import addFile from '../../services/drive/add-file'; +import User from '../../models/user'; +import dateFormat = require('dateformat'); +import Mute from '../../models/mute'; +import config from '../../config'; + +const logger = queueLogger.createSubLogger('export-mute'); + +export async function exportMute(job: bq.Job, done: any): Promise { + logger.info(`Exporting mute of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + const stream = fs.createWriteStream(path, { flags: 'a' }); + + let exportedCount = 0; + let ended = false; + let cursor: any = null; + + while (!ended) { + const mutes = await Mute.find({ + muterId: user._id, + ...(cursor ? { _id: { $gt: cursor } } : {}) + }, { + limit: 100, + sort: { + _id: 1 + } + }); + + if (mutes.length === 0) { + ended = true; + job.reportProgress(100); + break; + } + + cursor = mutes[mutes.length - 1]._id; + + for (const mute of mutes) { + const u = await User.findOne({ _id: mute.muteeId }, { fields: { username: true, host: true } }); + const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`; + await new Promise((res, rej) => { + stream.write(content + '\n', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + exportedCount++; + } + + const total = await Mute.count({ + muterId: user._id, + }); + + job.reportProgress(exportedCount / total); + } + + stream.end(); + logger.succ(`Exported to: ${path}`); + + const fileName = dateFormat(new Date(), 'mute-yyyy-mm-dd-HH-MM-ss') + '.json'; + const driveFile = await addFile(user, path, fileName); + + logger.succ(`Exported to: ${driveFile._id}`); + cleanup(); + done(); +} diff --git a/src/queue/processors/index.ts b/src/queue/processors/index.ts index 3f08fe29f..686998320 100644 --- a/src/queue/processors/index.ts +++ b/src/queue/processors/index.ts @@ -1,12 +1,18 @@ import deliver from './http/deliver'; import processInbox from './http/process-inbox'; import { exportNotes } from './export-notes'; +import { exportFollowing } from './export-following'; +import { exportMute } from './export-mute'; +import { exportBlocking } from './export-blocking'; import { queueLogger } from '../logger'; const handlers: any = { deliver, processInbox, exportNotes, + exportFollowing, + exportMute, + exportBlocking, }; export default (job: any, done: any) => { diff --git a/src/server/api/endpoints/i/export-blocking.ts b/src/server/api/endpoints/i/export-blocking.ts new file mode 100644 index 000000000..12a08c6d7 --- /dev/null +++ b/src/server/api/endpoints/i/export-blocking.ts @@ -0,0 +1,18 @@ +import define from '../../define'; +import { createExportBlockingJob } from '../../../../queue'; +import ms = require('ms'); + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, +}; + +export default define(meta, (ps, user) => new Promise(async (res, rej) => { + createExportBlockingJob(user); + + res(); +})); diff --git a/src/server/api/endpoints/i/export-following.ts b/src/server/api/endpoints/i/export-following.ts new file mode 100644 index 000000000..e819898d9 --- /dev/null +++ b/src/server/api/endpoints/i/export-following.ts @@ -0,0 +1,18 @@ +import define from '../../define'; +import { createExportFollowingJob } from '../../../../queue'; +import ms = require('ms'); + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, +}; + +export default define(meta, (ps, user) => new Promise(async (res, rej) => { + createExportFollowingJob(user); + + res(); +})); diff --git a/src/server/api/endpoints/i/export-mute.ts b/src/server/api/endpoints/i/export-mute.ts new file mode 100644 index 000000000..c719a1f6a --- /dev/null +++ b/src/server/api/endpoints/i/export-mute.ts @@ -0,0 +1,18 @@ +import define from '../../define'; +import { createExportMuteJob } from '../../../../queue'; +import ms = require('ms'); + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, +}; + +export default define(meta, (ps, user) => new Promise(async (res, rej) => { + createExportMuteJob(user); + + res(); +}));