自分の投稿情報をエクスポートできるように (#4144)

* wip

* 正しいJSONを生成するように

* データを整形
This commit is contained in:
syuilo 2019-02-05 19:50:14 +09:00 committed by GitHub
parent ba7e05837c
commit 5db5bbd1cd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 188 additions and 9 deletions

View file

@ -557,6 +557,9 @@ common/views/components/profile-editor.vue:
email-address: "メールアドレス" email-address: "メールアドレス"
email-verified: "メールアドレスが確認されました" email-verified: "メールアドレスが確認されました"
email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。" email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。"
export: "エクスポート"
export-notes: "すべての投稿のエクスポート"
export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。"
common/views/components/user-list-editor.vue: common/views/components/user-list-editor.vue:
users: "ユーザー" users: "ユーザー"

View file

@ -87,6 +87,14 @@
<ui-button @click="updateEmail()">{{ $t('save') }}</ui-button> <ui-button @click="updateEmail()">{{ $t('save') }}</ui-button>
</div> </div>
</section> </section>
<section>
<header>{{ $t('export') }}</header>
<div>
<ui-button @click="exportNotes()">{{ $t('export-notes') }}</ui-button>
</div>
</section>
</ui-card> </ui-card>
</template> </template>
@ -252,6 +260,15 @@ export default Vue.extend({
email: this.email == '' ? null : this.email email: this.email == '' ? null : this.email
}); });
}); });
},
exportNotes() {
this.$root.api('i/export-notes', {});
this.$root.dialog({
type: 'info',
text: this.$t('export-requested')
});
} }
} }
}); });

View file

@ -1,9 +1,9 @@
import * as Queue from 'bee-queue'; import * as Queue from 'bee-queue';
import config from '../config'; import config from '../config';
import http from './processors/http';
import { ILocalUser } from '../models/user'; import { ILocalUser } from '../models/user';
import Logger from '../misc/logger';
import { program } from '../argv'; import { program } from '../argv';
import handler from './processors';
const enableQueue = config.redis != null && !program.disableQueue; const enableQueue = config.redis != null && !program.disableQueue;
@ -36,7 +36,7 @@ export function createHttpJob(data: any) {
.backoff('exponential', 16384) // 16s .backoff('exponential', 16384) // 16s
.save(); .save();
} else { } else {
return http({ data }, () => {}); return handler({ data }, () => {});
} }
} }
@ -51,10 +51,18 @@ export function deliver(user: ILocalUser, content: any, to: any) {
}); });
} }
export const queueLogger = new Logger('queue'); export function createExportNotesJob(user: ILocalUser) {
if (!enableQueue) throw 'queue disabled';
return queue.createJob({
type: 'exportNotes',
user: user
})
.save();
}
export default function() { export default function() {
if (enableQueue) { if (enableQueue) {
queue.process(128, http); queue.process(128, handler);
} }
} }

3
src/queue/logger.ts Normal file
View file

@ -0,0 +1,3 @@
import Logger from '../misc/logger';
export const queueLogger = new Logger('queue', 'orange');

View file

@ -0,0 +1,128 @@
import * as bq from 'bee-queue';
import * as tmp from 'tmp';
import * as fs from 'fs';
import * as mongo from 'mongodb';
import { queueLogger } from '../logger';
import Note, { INote } from '../../models/note';
import addFile from '../../services/drive/add-file';
import User from '../../models/user';
import dateFormat = require('dateformat');
const logger = queueLogger.createSubLogger('export-notes');
export async function exportNotes(job: bq.Job, done: any): Promise<void> {
logger.info(`Exporting notes of ${job.data.user._id} ...`);
const user = await User.findOne({
_id: new mongo.ObjectID(job.data.user._id.toString())
});
// Create temp file
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp file is ${path}`);
const stream = fs.createWriteStream(path, { flags: 'a' });
await new Promise((res, rej) => {
stream.write('[', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
let exportedNotesCount = 0;
let ended = false;
let cursor: any = null;
while (!ended) {
const notes = await Note.find({
userId: user._id,
...(cursor ? { _id: { $gt: cursor } } : {})
}, {
limit: 100,
sort: {
_id: 1
}
});
if (notes.length === 0) {
ended = true;
job.reportProgress(100);
break;
}
cursor = notes[notes.length - 1]._id;
for (const note of notes) {
const content = JSON.stringify(serialize(note));
await new Promise((res, rej) => {
stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
exportedNotesCount++;
}
const total = await Note.count({
userId: user._id,
});
job.reportProgress(exportedNotesCount / total);
}
await new Promise((res, rej) => {
stream.write(']', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
stream.end();
logger.succ(`Exported to: ${path}`);
const fileName = dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.json';
const driveFile = await addFile(user, path, fileName);
logger.succ(`Exported to: ${driveFile._id}`);
cleanup();
done();
}
function serialize(note: INote): any {
return {
id: note._id,
text: note.text,
createdAt: note.createdAt,
fileIds: note.fileIds,
replyId: note.replyId,
renoteId: note.renoteId,
poll: note.poll,
cw: note.cw,
viaMobile: note.viaMobile,
visibility: note.visibility,
visibleUserIds: note.visibleUserIds,
appId: note.appId,
geo: note.geo,
localOnly: note.localOnly
};
}

View file

@ -1,7 +1,7 @@
import * as bq from 'bee-queue'; import * as bq from 'bee-queue';
import request from '../../../remote/activitypub/request'; import request from '../../../remote/activitypub/request';
import { queueLogger } from '../..'; import { queueLogger } from '../../logger';
export default async (job: bq.Job, done: any): Promise<void> => { export default async (job: bq.Job, done: any): Promise<void> => {
try { try {

View file

@ -1,10 +1,12 @@
import deliver from './deliver'; import deliver from './http/deliver';
import processInbox from './process-inbox'; import processInbox from './http/process-inbox';
import { queueLogger } from '../..'; import { exportNotes } from './export-notes';
import { queueLogger } from '../logger';
const handlers: any = { const handlers: any = {
deliver, deliver,
processInbox, processInbox,
exportNotes,
}; };
export default (job: any, done: any) => { export default (job: any, done: any) => {

View file

@ -0,0 +1,18 @@
import define from '../../define';
import { createExportNotesJob } from '../../../../queue';
import ms = require('ms');
export const meta = {
secure: true,
requireCredential: true,
limit: {
duration: ms('1day'),
max: 1,
},
};
export default define(meta, (ps, user) => new Promise(async (res, rej) => {
createExportNotesJob(user);
res();
}));