From 72fb23f4d51db1b999a10b7d8bb50293a81af163 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 27 May 2019 16:54:47 +0900 Subject: [PATCH] Improve drive management --- locales/ja-JP.yml | 3 ++ src/client/app/admin/views/drive.vue | 27 +++++++++++++++++ src/queue/index.ts | 30 +++++++++++++++++++ src/queue/processors/db/delete-drive-files.ts | 2 +- .../processors/object-storage/delete-file.ts | 22 ++++++++++++++ src/queue/processors/object-storage/index.ts | 12 ++++++++ .../admin/delete-all-files-of-a-user.ts | 4 +-- .../admin/drive/clean-remote-files.ts | 21 +++++++++++++ .../api/endpoints/admin/drive/cleanup.ts | 21 +++++++++++++ .../admin/federation/delete-all-files.ts | 4 +-- .../api/endpoints/drive/files/delete.ts | 4 +-- src/services/drive/add-file.ts | 4 +-- src/services/drive/delete-file.ts | 26 +++++----------- src/tools/clean-remote-files.ts | 26 ---------------- 14 files changed, 152 insertions(+), 54 deletions(-) create mode 100644 src/queue/processors/object-storage/delete-file.ts create mode 100644 src/queue/processors/object-storage/index.ts create mode 100644 src/server/api/endpoints/admin/drive/clean-remote-files.ts create mode 100644 src/server/api/endpoints/admin/drive/cleanup.ts delete mode 100644 src/tools/clean-remote-files.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 25c052bc4..493875af6 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1417,6 +1417,9 @@ admin/views/drive.vue: unmark-as-sensitive: "閲覧注意を解除" marked-as-sensitive: "閲覧注意に設定しました" unmarked-as-sensitive: "閲覧注意を解除しました" + clean-remote-files: "リモートファイルのキャッシュを削除" + clean-remote-files-are-you-sure: "すべてのリモートファイルのキャッシュを削除してもよろしいですか？" + clean-up: "クリーンアップ" admin/views/users.vue: operation: "操作" diff --git a/src/client/app/admin/views/drive.vue b/src/client/app/admin/views/drive.vue index 22b3b7eb9..1152db2b9 100644 --- a/src/client/app/admin/views/drive.vue +++ b/src/client/app/admin/views/drive.vue @@ -14,6 +14,10 @@ {{ $t('lookup') }} +
+ {{ $t('clean-up') }} + {{ $t('clean-remote-files') }} +
@@ -227,6 +231,29 @@ export default Vue.extend({ }); }); }, + + cleanRemoteFiles() { + this.$root.dialog({ + type: 'warning', + text: this.$t('clean-remote-files-are-you-sure'), + showCancelButton: true + }).then(({ canceled }) => { + if (canceled) return; + this.$root.api('admin/drive/clean-remote-files'); + this.$root.dialog({ + type: 'success', + splash: true + }); + }); + }, + + cleanUp() { + this.$root.api('admin/drive/cleanup'); + this.$root.dialog({ + type: 'success', + splash: true + }); + } } }); diff --git a/src/queue/index.ts b/src/queue/index.ts index a010004f1..70233d831 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -8,6 +8,7 @@ import { program } from '../argv'; import processDeliver from './processors/deliver'; import processInbox from './processors/inbox'; import processDb from './processors/db'; +import procesObjectStorage from './processors/object-storage'; import { queueLogger } from './logger'; import { DriveFile } from '../models/entities/drive-file'; @@ -34,9 +35,12 @@ function renderError(e: Error): any { export const deliverQueue = initializeQueue('deliver'); export const inboxQueue = initializeQueue('inbox'); export const dbQueue = initializeQueue('db'); +export const objectStorageQueue = initializeQueue('objectStorage'); const deliverLogger = queueLogger.createSubLogger('deliver'); const inboxLogger = queueLogger.createSubLogger('inbox'); +const dbLogger = queueLogger.createSubLogger('db'); +const objectStorageLogger = queueLogger.createSubLogger('objectStorage'); deliverQueue .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) @@ -54,6 +58,22 @@ inboxQueue .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => inboxLogger.warn(`stalled id=${job.id} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); +dbQueue + .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); + +objectStorageQueue + .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + export function deliver(user: ILocalUser, content: any, to: any) { if (content == null) return null; @@ -165,11 +185,21 @@ export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id }); } +export function createDeleteObjectStorageFileJob(key: string) { + return objectStorageQueue.add('deleteFile', { + key: key + }, { + removeOnComplete: true, + removeOnFail: true + }); +} + export default function() { if (!program.onlyServer) { deliverQueue.process(128, processDeliver); inboxQueue.process(128, processInbox); processDb(dbQueue); + procesObjectStorage(objectStorageQueue); } } diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts index 4e4eab86b..5ecee9139 100644 --- a/src/queue/processors/db/delete-drive-files.ts +++ b/src/queue/processors/db/delete-drive-files.ts @@ -1,7 +1,7 @@ import * as Bull from 'bull'; import { queueLogger } from '../../logger'; -import deleteFile from '../../../services/drive/delete-file'; +import { deleteFile } from '../../../services/drive/delete-file'; import { Users, DriveFiles } from '../../../models'; import { MoreThan } from 'typeorm'; diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts new file mode 100644 index 000000000..8e6b5f959 --- /dev/null +++ b/src/queue/processors/object-storage/delete-file.ts @@ -0,0 +1,22 @@ +import * as Bull from 'bull'; +import * as Minio from 'minio'; +import { fetchMeta } from '../../../misc/fetch-meta'; + +export default async (job: Bull.Job) => { + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + const key: string = job.data.key; + + await minio.removeObject(meta.objectStorageBucket!, key); + + return 'Success'; +}; diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts new file mode 100644 index 000000000..f2fc08f1a --- /dev/null +++ b/src/queue/processors/object-storage/index.ts @@ -0,0 +1,12 @@ +import * as Bull from 'bull'; +import deleteFile from './delete-file'; + +const jobs = { + deleteFile, +} as any; + +export default function(q: Bull.Queue) { + for (const [k, v] of Object.entries(jobs)) { + q.process(k, v as any); + } +} diff --git a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts index 84e9c363e..276d1f37b 100644 --- a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts +++ b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import define from '../../define'; -import del from '../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../services/drive/delete-file'; import { DriveFiles } from '../../../../models'; import { ID } from '../../../../misc/cafy-id'; @@ -27,6 +27,6 @@ export default define(meta, async (ps, me) => { }); for (const file of files) { - del(file); + deleteFile(file); } }); diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts new file mode 100644 index 000000000..a861052bb --- /dev/null +++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -0,0 +1,21 @@ +import { Not, IsNull } from 'typeorm'; +import define from '../../../define'; +import { deleteFile } from '../../../../../services/drive/delete-file'; +import { DriveFiles } from '../../../../../models'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireModerator: true, +}; + +export default define(meta, async (ps, me) => { + const files = await DriveFiles.find({ + userHost: Not(IsNull()) + }); + + for (const file of files) { + deleteFile(file, true); + } +}); diff --git a/src/server/api/endpoints/admin/drive/cleanup.ts b/src/server/api/endpoints/admin/drive/cleanup.ts new file mode 100644 index 000000000..272416dee --- /dev/null +++ b/src/server/api/endpoints/admin/drive/cleanup.ts @@ -0,0 +1,21 @@ +import { IsNull } from 'typeorm'; +import define from '../../../define'; +import { deleteFile } from '../../../../../services/drive/delete-file'; +import { DriveFiles } from '../../../../../models'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireModerator: true, +}; + +export default define(meta, async (ps, me) => { + const files = await DriveFiles.find({ + userId: IsNull() + }); + + for (const file of files) { + deleteFile(file); + } +}); diff --git a/src/server/api/endpoints/admin/federation/delete-all-files.ts b/src/server/api/endpoints/admin/federation/delete-all-files.ts index befb36226..24ad9ce80 100644 --- a/src/server/api/endpoints/admin/federation/delete-all-files.ts +++ b/src/server/api/endpoints/admin/federation/delete-all-files.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import define from '../../../define'; -import del from '../../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../../services/drive/delete-file'; import { DriveFiles } from '../../../../../models'; export const meta = { @@ -22,6 +22,6 @@ export default define(meta, async (ps, me) => { }); for (const file of files) { - del(file); + deleteFile(file); } }); diff --git a/src/server/api/endpoints/drive/files/delete.ts b/src/server/api/endpoints/drive/files/delete.ts index d8cc5ec0a..d6a20f352 100644 --- a/src/server/api/endpoints/drive/files/delete.ts +++ b/src/server/api/endpoints/drive/files/delete.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import { ID } from '../../../../../misc/cafy-id'; -import del from '../../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../../services/drive/delete-file'; import { publishDriveStream } from '../../../../../services/stream'; import define from '../../../define'; import { ApiError } from '../../../error'; @@ -57,7 +57,7 @@ export default define(meta, async (ps, user) => { } // Delete - await del(file); + await deleteFile(file); // Publish fileDeleted event publishDriveStream(user.id, 'fileDeleted', file.id); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index eb0b0abaa..d21c67d18 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -7,7 +7,7 @@ import * as uuid from 'uuid'; import * as sharp from 'sharp'; import { publishMainStream, publishDriveStream } from '../stream'; -import delFile from './delete-file'; +import { deleteFile } from './delete-file'; import { fetchMeta } from '../../misc/fetch-meta'; import { GenerateVideoThumbnail } from './generate-video-thumbnail'; import { driveLogger } from './logger'; @@ -233,7 +233,7 @@ async function deleteOldFile(user: IRemoteUser) { const oldFile = await q.getOne(); if (oldFile) { - delFile(oldFile, true); + deleteFile(oldFile, true); } } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index b429ca0db..258c0853f 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,11 +1,10 @@ -import * as Minio from 'minio'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; -import { fetchMeta } from '../../misc/fetch-meta'; +import { createDeleteObjectStorageFileJob } from '../../queue'; -export default async function(file: DriveFile, isExpired = false) { +export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { InternalStorage.del(file.accessKey!); @@ -17,25 +16,14 @@ export default async function(file: DriveFile, isExpired = false) { InternalStorage.del(file.webpublicAccessKey!); } } else if (!file.isLink) { - const meta = await fetchMeta(); - - const minio = new Minio.Client({ - endPoint: meta.objectStorageEndpoint!, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, - port: meta.objectStoragePort ? meta.objectStoragePort : undefined, - useSSL: meta.objectStorageUseSSL, - accessKey: meta.objectStorageAccessKey!, - secretKey: meta.objectStorageSecretKey!, - }); - - await minio.removeObject(meta.objectStorageBucket!, file.accessKey!); + createDeleteObjectStorageFileJob(file.accessKey!); if (file.thumbnailUrl) { - await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!); + createDeleteObjectStorageFileJob(file.thumbnailAccessKey!); } if (file.webpublicUrl) { - await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!); + createDeleteObjectStorageFileJob(file.webpublicAccessKey!); } } @@ -44,8 +32,8 @@ export default async function(file: DriveFile, isExpired = false) { DriveFiles.update(file.id, { isLink: true, url: file.uri, - thumbnailUrl: null, - webpublicUrl: null + thumbnailUrl: file.uri, + webpublicUrl: file.uri }); } else { DriveFiles.delete(file.id); diff --git a/src/tools/clean-remote-files.ts b/src/tools/clean-remote-files.ts deleted file mode 100644 index 633a6fc04..000000000 --- a/src/tools/clean-remote-files.ts +++ /dev/null @@ -1,26 +0,0 @@ -import * as promiseLimit from 'promise-limit'; -import del from '../services/drive/delete-file'; -import { DriveFiles } from '../models'; -import { Not, IsNull } from 'typeorm'; -import { DriveFile } from '../models/entities/drive-file'; -import { ensure } from '../prelude/ensure'; - -const limit = promiseLimit(16); - -DriveFiles.find({ - userHost: Not(IsNull()) -}).then(async files => { - console.log(`there is ${files.length} files`); - - await Promise.all(files.map(file => limit(() => job(file)))); - - console.log('ALL DONE'); -}); - -async function job(file: DriveFile): Promise { - file = await DriveFiles.findOne(file.id).then(ensure); - - await del(file, true); - - console.log('done', file.id); -}