diff --git a/src/queue/index.ts b/src/queue/index.ts index 70233d831..a7e9b9814 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) { }); } +export function createCleanRemoteFilesJob() { + return objectStorageQueue.add('cleanRemoteFiles', {}, { + removeOnComplete: true, + removeOnFail: true + }); +} + export default function() { if (!program.onlyServer) { deliverQueue.process(128, processDeliver); diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts index 5ecee9139..a2fd9050a 100644 --- a/src/queue/processors/db/delete-drive-files.ts +++ b/src/queue/processors/db/delete-drive-files.ts @@ -1,7 +1,7 @@ import * as Bull from 'bull'; import { queueLogger } from '../../logger'; -import { deleteFile } from '../../../services/drive/delete-file'; +import { deleteFileSync } from '../../../services/drive/delete-file'; import { Users, DriveFiles } from '../../../models'; import { MoreThan } from 'typeorm'; @@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise cursor = files[files.length - 1].id; for (const file of files) { - await deleteFile(file); + await deleteFileSync(file); deletedCount++; } diff --git a/src/queue/processors/object-storage/clean-remote-files.ts b/src/queue/processors/object-storage/clean-remote-files.ts new file mode 100644 index 000000000..7b34892e1 --- /dev/null +++ b/src/queue/processors/object-storage/clean-remote-files.ts @@ -0,0 +1,50 @@ +import * as Bull from 'bull'; + +import { queueLogger } from '../../logger'; +import { deleteFileSync } from '../../../services/drive/delete-file'; +import { DriveFiles } from '../../../models'; +import { MoreThan, Not, IsNull } from 'typeorm'; + +const logger = queueLogger.createSubLogger('clean-remote-files'); + +export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise { + logger.info(`Deleting cached remote files...`); + + let deletedCount = 0; + let cursor: any = null; + + while (true) { + const files = await DriveFiles.find({ + where: { + userHost: Not(IsNull()), + isLink: false, + ...(cursor ? { id: MoreThan(cursor) } : {}) + }, + take: 8, + order: { + id: 1 + } + }); + + if (files.length === 0) { + job.progress(100); + break; + } + + cursor = files[files.length - 1].id; + + await Promise.all(files.map(file => deleteFileSync(file, true))); + + deletedCount += 8; + + const total = await DriveFiles.count({ + userHost: Not(IsNull()), + isLink: false, + }); + + job.progress(deletedCount / total); + } + + logger.succ(`All cahced remote files has been deleted.`); + done(); +} diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts index 8e6b5f959..f899df7d2 100644 --- a/src/queue/processors/object-storage/delete-file.ts +++ b/src/queue/processors/object-storage/delete-file.ts @@ -1,22 +1,10 @@ import * as Bull from 'bull'; -import * as Minio from 'minio'; -import { fetchMeta } from '../../../misc/fetch-meta'; +import { deleteObjectStorageFile } from '../../../services/drive/delete-file'; export default async (job: Bull.Job) => { - const meta = await fetchMeta(); - - const minio = new Minio.Client({ - endPoint: meta.objectStorageEndpoint!, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, - port: meta.objectStoragePort ? meta.objectStoragePort : undefined, - useSSL: meta.objectStorageUseSSL, - accessKey: meta.objectStorageAccessKey!, - secretKey: meta.objectStorageSecretKey!, - }); - const key: string = job.data.key; - await minio.removeObject(meta.objectStorageBucket!, key); + await deleteObjectStorageFile(key); return 'Success'; }; diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts index 60f732ca6..33ef665b3 100644 --- a/src/queue/processors/object-storage/index.ts +++ b/src/queue/processors/object-storage/index.ts @@ -1,8 +1,10 @@ import * as Bull from 'bull'; import deleteFile from './delete-file'; +import cleanRemoteFiles from './clean-remote-files'; const jobs = { deleteFile, + cleanRemoteFiles, } as any; export default function(q: Bull.Queue) { diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts index 69cfe0db9..e837ae1bb 100644 --- a/src/server/api/endpoints/admin/drive/clean-remote-files.ts +++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -1,7 +1,5 @@ -import { Not, IsNull } from 'typeorm'; import define from '../../../define'; -import { deleteFile } from '../../../../../services/drive/delete-file'; -import { DriveFiles } from '../../../../../models'; +import { createCleanRemoteFilesJob } from '../../../../../queue'; export const meta = { tags: ['admin'], @@ -11,12 +9,5 @@ export const meta = { }; export default define(meta, async (ps, me) => { - const files = await DriveFiles.find({ - userHost: Not(IsNull()), - isLink: false, - }); - - for (const file of files) { - deleteFile(file, true); - } + createCleanRemoteFilesJob(); }); diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 258c0853f..5b44a0817 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,8 +1,10 @@ +import * as Minio from 'minio'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; import { createDeleteObjectStorageFileJob } from '../../queue'; +import { fetchMeta } from '../../misc/fetch-meta'; export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) { } } + postProcess(file, isExpired); +} + +export async function deleteFileSync(file: DriveFile, isExpired = false) { + if (file.storedInternal) { + InternalStorage.del(file.accessKey!); + + if (file.thumbnailUrl) { + InternalStorage.del(file.thumbnailAccessKey!); + } + + if (file.webpublicUrl) { + InternalStorage.del(file.webpublicAccessKey!); + } + } else if (!file.isLink) { + const promises = []; + + promises.push(deleteObjectStorageFile(file.accessKey!)); + + if (file.thumbnailUrl) { + promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!)); + } + + if (file.webpublicUrl) { + promises.push(deleteObjectStorageFile(file.webpublicAccessKey!)); + } + + await Promise.all(promises); + } + + postProcess(file, isExpired); +} + +function postProcess(file: DriveFile, isExpired = false) { // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { DriveFiles.update(file.id, { @@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) { Instances.decrement({ host: file.userHost }, 'driveFiles', 1); } } + +export async function deleteObjectStorageFile(key: string) { + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + await minio.removeObject(meta.objectStorageBucket!, key); +}