From c3d34bda37ca3b48214b094d54ee22d987a42574 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 11 Mar 2019 19:43:58 +0900 Subject: [PATCH] Resolve #4259 --- locales/ja-JP.yml | 3 + .../views/components/settings/profile.vue | 26 +++- src/queue/index.ts | 11 ++ src/queue/processors/db/import-user-lists.ts | 140 ++++++++++++++++++ src/queue/processors/db/index.ts | 4 +- .../api/endpoints/i/import-user-lists.ts | 64 ++++++++ 6 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 src/queue/processors/db/import-user-lists.ts create mode 100644 src/server/api/endpoints/i/import-user-lists.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 124e71f6b..813ea7209 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -701,6 +701,8 @@ common/views/components/profile-editor.vue: email-verified: "メールアドレスが確認されました" email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。" export: "エクスポート" + import: "インポート" + export-and-import: "エクスポートとインポート" export-targets: all-notes: "すべての投稿データ" following-list: "フォロー" @@ -708,6 +710,7 @@ common/views/components/profile-editor.vue: blocking-list: "ブロック" user-lists: "リスト" export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。" + import-requested: "インポートをリクエストしました。これには時間がかかる場合があります。" enter-password: "パスワードを入力してください" danger-zone: "危険な設定" delete-account: "アカウントを削除" diff --git a/src/client/app/common/views/components/settings/profile.vue b/src/client/app/common/views/components/settings/profile.vue index 16e7a3b25..c49b465ce 100644 --- a/src/client/app/common/views/components/settings/profile.vue +++ b/src/client/app/common/views/components/settings/profile.vue @@ -89,7 +89,7 @@
-
{{ $t('export') }}
+
{{ $t('export-and-import') }}
@@ -99,7 +99,10 @@ - {{ $t('export') }} + + {{ $t('export') }} + {{ $t('import') }} +
@@ -119,7 +122,7 @@ import { apiUrl, host } from '../../../../config'; import { toUnicode } from 'punycode'; import langmap from 'langmap'; import { unique } from '../../../../../../prelude/array'; -import { faDownload } from '@fortawesome/free-solid-svg-icons'; +import { faDownload, faUpload } from '@fortawesome/free-solid-svg-icons'; export default Vue.extend({ i18n: i18n('common/views/components/profile-editor.vue'), @@ -148,7 +151,7 @@ export default Vue.extend({ avatarUploading: false, bannerUploading: false, exportTarget: 'notes', - faDownload + faDownload, faUpload }; }, @@ -294,6 +297,21 @@ export default Vue.extend({ }); }, + doImport() { + this.$chooseDriveFile().then(file => { + this.$root.api( + this.exportTarget == 'user-lists' ? 'i/import-user-lists' : + null, { + fileId: file.id + }); + + this.$root.dialog({ + type: 'info', + text: this.$t('import-requested') + }); + }); + }, + async deleteAccount() { const { canceled: canceled, result: password } = await this.$root.dialog({ title: this.$t('enter-password'), diff --git a/src/queue/index.ts b/src/queue/index.ts index 00a4a48f1..09e0ad59c 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -9,6 +9,7 @@ import processDeliver from './processors/deliver'; import processInbox from './processors/inbox'; import processDb from './processors/db'; import { queueLogger } from './logger'; +import { IDriveFile } from '../models/drive-file'; function initializeQueue(name: string) { return new Queue(name, config.redis != null ? { @@ -145,6 +146,16 @@ export function createExportUserListsJob(user: ILocalUser) { }); } +export function createImportUserListsJob(user: ILocalUser, fileId: IDriveFile['_id']) { + return dbQueue.add('importUserLists', { + user: user, + fileId: fileId + }, { + removeOnComplete: true, + removeOnFail: true + }); +} + export default function() { if (!program.onlyServer) { deliverQueue.process(128, processDeliver); diff --git a/src/queue/processors/db/import-user-lists.ts b/src/queue/processors/db/import-user-lists.ts new file mode 100644 index 000000000..ee1468d5a --- /dev/null +++ b/src/queue/processors/db/import-user-lists.ts @@ -0,0 +1,140 @@ +import * as Bull from 'bull'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as util from 'util'; +import * as mongo from 'mongodb'; +import * as request from 'request'; + +import { queueLogger } from '../../logger'; +import User from '../../../models/user'; +import config from '../../../config'; +import UserList from '../../../models/user-list'; +import DriveFile from '../../../models/drive-file'; +import chalk from 'chalk'; +import { getOriginalUrl } from '../../../misc/get-drive-file-url'; +import parseAcct from '../../../misc/acct/parse'; +import resolveUser from '../../../remote/resolve-user'; + +const logger = queueLogger.createSubLogger('import-user-lists'); + +export async function importUserLists(job: Bull.Job, done: any): Promise { + logger.info(`Importing user lists of ${job.data.user._id} ...`); + + const user = await User.findOne({ + _id: new mongo.ObjectID(job.data.user._id.toString()) + }); + + const file = await DriveFile.findOne({ + _id: new mongo.ObjectID(job.data.fileId.toString()) + }); + + const url = getOriginalUrl(file); + + // Create temp file + const [path, cleanup] = await new Promise<[string, any]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp file is ${path}`); + + // write content at URL to temp file + await new Promise((res, rej) => { + logger.info(`Downloading ${chalk.cyan(url)} ...`); + + const writable = fs.createWriteStream(path); + + writable.on('finish', () => { + logger.succ(`Download finished: ${chalk.cyan(url)}`); + res(); + }); + + writable.on('error', error => { + logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, { + url: url, + e: error + }); + rej(error); + }); + + const requestUrl = new URL(url).pathname.match(/[^\u0021-\u00ff]/) ? encodeURI(url) : url; + + const req = request({ + url: requestUrl, + proxy: config.proxy, + timeout: 10 * 1000, + headers: { + 'User-Agent': config.userAgent + } + }); + + req.pipe(writable); + + req.on('response', response => { + if (response.statusCode !== 200) { + logger.error(`Got ${response.statusCode} (${url})`); + writable.close(); + rej(response.statusCode); + } + }); + + req.on('error', error => { + logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, { + url: url, + e: error + }); + writable.close(); + rej(error); + }); + }); + + logger.succ(`Downloaded to: ${path}`); + + const csv = await util.promisify(fs.readFile)(path, 'utf8'); + + for (const line of csv.trim().split('\n')) { + const listName = line.split(',')[0].trim(); + const { username, host } = parseAcct(line.split(',')[1].trim()); + + let list = await UserList.findOne({ + userId: user._id, + title: listName + }); + + if (list == null) { + list = await UserList.insert({ + createdAt: new Date(), + userId: user._id, + title: listName, + userIds: [] + }); + } + + let target = host === config.host ? await User.findOne({ + host: null, + usernameLower: username.toLowerCase() + }) : await User.findOne({ + host: host, + usernameLower: username.toLowerCase() + }); + + if (host == null && target == null) continue; + if (list.userIds.some(id => id.equals(target._id))) continue; + + if (target == null) { + target = await resolveUser(username, host); + } + + await UserList.update({ _id: list._id }, { + $push: { + userIds: target._id + } + }); + } + + logger.succ('Imported'); + cleanup(); + done(); +} diff --git a/src/queue/processors/db/index.ts b/src/queue/processors/db/index.ts index 8ac9c1a3d..4a97a1c88 100644 --- a/src/queue/processors/db/index.ts +++ b/src/queue/processors/db/index.ts @@ -6,6 +6,7 @@ import { exportFollowing } from './export-following'; import { exportMute } from './export-mute'; import { exportBlocking } from './export-blocking'; import { exportUserLists } from './export-user-lists'; +import { importUserLists } from './import-user-lists'; const jobs = { deleteNotes, @@ -14,7 +15,8 @@ const jobs = { exportFollowing, exportMute, exportBlocking, - exportUserLists + exportUserLists, + importUserLists } as any; export default function(dbQueue: Bull.Queue) { diff --git a/src/server/api/endpoints/i/import-user-lists.ts b/src/server/api/endpoints/i/import-user-lists.ts new file mode 100644 index 000000000..ed3085e5f --- /dev/null +++ b/src/server/api/endpoints/i/import-user-lists.ts @@ -0,0 +1,64 @@ +import $ from 'cafy'; +import ID, { transform } from '../../../../misc/cafy-id'; +import define from '../../define'; +import { createImportUserListsJob } from '../../../../queue'; +import ms = require('ms'); +import DriveFile from '../../../../models/drive-file'; +import { ApiError } from '../../error'; + +export const meta = { + secure: true, + requireCredential: true, + limit: { + duration: ms('1hour'), + max: 1, + }, + + params: { + fileId: { + validator: $.type(ID), + transform: transform, + } + }, + + errors: { + noSuchFile: { + message: 'No such file.', + code: 'NO_SUCH_FILE', + id: 'ea9cc34f-c415-4bc6-a6fe-28ac40357049' + }, + + unexpectedFileType: { + message: 'We need csv file.', + code: 'UNEXPECTED_FILE_TYPE', + id: 'a3c9edda-dd9b-4596-be6a-150ef813745c' + }, + + tooBigFile: { + message: 'That file is too big.', + code: 'TOO_BIG_FILE', + id: 'ae6e7a22-971b-4b52-b2be-fc0b9b121fe9' + }, + + emptyFile: { + message: 'That file is empty.', + code: 'EMPTY_FILE', + id: '99efe367-ce6e-4d44-93f8-5fae7b040356' + }, + } +}; + +export default define(meta, async (ps, user) => { + const file = await DriveFile.findOne({ + _id: ps.fileId + }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + //if (!file.contentType.endsWith('/csv')) throw new ApiError(meta.errors.unexpectedFileType); + if (file.length > 30000) throw new ApiError(meta.errors.tooBigFile); + if (file.length === 0) throw new ApiError(meta.errors.emptyFile); + + createImportUserListsJob(user, file._id); + + return; +});