リモートファイルの削除が重い問題を修正

This commit is contained in:
syuilo 2019-07-01 21:12:14 +09:00
parent ed0070c470
commit 03e999875a
No known key found for this signature in database
GPG key ID: BDC4C49D06AB9D69
7 changed files with 116 additions and 27 deletions

View file

@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) {
}); });
} }
export function createCleanRemoteFilesJob() {
return objectStorageQueue.add('cleanRemoteFiles', {}, {
removeOnComplete: true,
removeOnFail: true
});
}
export default function() { export default function() {
if (!program.onlyServer) { if (!program.onlyServer) {
deliverQueue.process(128, processDeliver); deliverQueue.process(128, processDeliver);

View file

@ -1,7 +1,7 @@
import * as Bull from 'bull'; import * as Bull from 'bull';
import { queueLogger } from '../../logger'; import { queueLogger } from '../../logger';
import { deleteFile } from '../../../services/drive/delete-file'; import { deleteFileSync } from '../../../services/drive/delete-file';
import { Users, DriveFiles } from '../../../models'; import { Users, DriveFiles } from '../../../models';
import { MoreThan } from 'typeorm'; import { MoreThan } from 'typeorm';
@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise<void>
cursor = files[files.length - 1].id; cursor = files[files.length - 1].id;
for (const file of files) { for (const file of files) {
await deleteFile(file); await deleteFileSync(file);
deletedCount++; deletedCount++;
} }

View file

@ -0,0 +1,50 @@
import * as Bull from 'bull';
import { queueLogger } from '../../logger';
import { deleteFileSync } from '../../../services/drive/delete-file';
import { DriveFiles } from '../../../models';
import { MoreThan, Not, IsNull } from 'typeorm';
const logger = queueLogger.createSubLogger('clean-remote-files');
export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise<void> {
logger.info(`Deleting cached remote files...`);
let deletedCount = 0;
let cursor: any = null;
while (true) {
const files = await DriveFiles.find({
where: {
userHost: Not(IsNull()),
isLink: false,
...(cursor ? { id: MoreThan(cursor) } : {})
},
take: 8,
order: {
id: 1
}
});
if (files.length === 0) {
job.progress(100);
break;
}
cursor = files[files.length - 1].id;
await Promise.all(files.map(file => deleteFileSync(file, true)));
deletedCount += 8;
const total = await DriveFiles.count({
userHost: Not(IsNull()),
isLink: false,
});
job.progress(deletedCount / total);
}
logger.succ(`All cahced remote files has been deleted.`);
done();
}

View file

@ -1,22 +1,10 @@
import * as Bull from 'bull'; import * as Bull from 'bull';
import * as Minio from 'minio'; import { deleteObjectStorageFile } from '../../../services/drive/delete-file';
import { fetchMeta } from '../../../misc/fetch-meta';
export default async (job: Bull.Job) => { export default async (job: Bull.Job) => {
const meta = await fetchMeta();
const minio = new Minio.Client({
endPoint: meta.objectStorageEndpoint!,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
useSSL: meta.objectStorageUseSSL,
accessKey: meta.objectStorageAccessKey!,
secretKey: meta.objectStorageSecretKey!,
});
const key: string = job.data.key; const key: string = job.data.key;
await minio.removeObject(meta.objectStorageBucket!, key); await deleteObjectStorageFile(key);
return 'Success'; return 'Success';
}; };

View file

@ -1,8 +1,10 @@
import * as Bull from 'bull'; import * as Bull from 'bull';
import deleteFile from './delete-file'; import deleteFile from './delete-file';
import cleanRemoteFiles from './clean-remote-files';
const jobs = { const jobs = {
deleteFile, deleteFile,
cleanRemoteFiles,
} as any; } as any;
export default function(q: Bull.Queue) { export default function(q: Bull.Queue) {

View file

@ -1,7 +1,5 @@
import { Not, IsNull } from 'typeorm';
import define from '../../../define'; import define from '../../../define';
import { deleteFile } from '../../../../../services/drive/delete-file'; import { createCleanRemoteFilesJob } from '../../../../../queue';
import { DriveFiles } from '../../../../../models';
export const meta = { export const meta = {
tags: ['admin'], tags: ['admin'],
@ -11,12 +9,5 @@ export const meta = {
}; };
export default define(meta, async (ps, me) => { export default define(meta, async (ps, me) => {
const files = await DriveFiles.find({ createCleanRemoteFilesJob();
userHost: Not(IsNull()),
isLink: false,
});
for (const file of files) {
deleteFile(file, true);
}
}); });

View file

@ -1,8 +1,10 @@
import * as Minio from 'minio';
import { DriveFile } from '../../models/entities/drive-file'; import { DriveFile } from '../../models/entities/drive-file';
import { InternalStorage } from './internal-storage'; import { InternalStorage } from './internal-storage';
import { DriveFiles, Instances, Notes } from '../../models'; import { DriveFiles, Instances, Notes } from '../../models';
import { driveChart, perUserDriveChart, instanceChart } from '../chart'; import { driveChart, perUserDriveChart, instanceChart } from '../chart';
import { createDeleteObjectStorageFileJob } from '../../queue'; import { createDeleteObjectStorageFileJob } from '../../queue';
import { fetchMeta } from '../../misc/fetch-meta';
export async function deleteFile(file: DriveFile, isExpired = false) { export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) { if (file.storedInternal) {
@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
} }
} }
postProcess(file, isExpired);
}
export async function deleteFileSync(file: DriveFile, isExpired = false) {
if (file.storedInternal) {
InternalStorage.del(file.accessKey!);
if (file.thumbnailUrl) {
InternalStorage.del(file.thumbnailAccessKey!);
}
if (file.webpublicUrl) {
InternalStorage.del(file.webpublicAccessKey!);
}
} else if (!file.isLink) {
const promises = [];
promises.push(deleteObjectStorageFile(file.accessKey!));
if (file.thumbnailUrl) {
promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!));
}
if (file.webpublicUrl) {
promises.push(deleteObjectStorageFile(file.webpublicAccessKey!));
}
await Promise.all(promises);
}
postProcess(file, isExpired);
}
function postProcess(file: DriveFile, isExpired = false) {
// リモートファイル期限切れ削除後は直リンクにする // リモートファイル期限切れ削除後は直リンクにする
if (isExpired && file.userHost !== null && file.uri != null) { if (isExpired && file.userHost !== null && file.uri != null) {
DriveFiles.update(file.id, { DriveFiles.update(file.id, {
@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) {
Instances.decrement({ host: file.userHost }, 'driveFiles', 1); Instances.decrement({ host: file.userHost }, 'driveFiles', 1);
} }
} }
export async function deleteObjectStorageFile(key: string) {
const meta = await fetchMeta();
const minio = new Minio.Client({
endPoint: meta.objectStorageEndpoint!,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
useSSL: meta.objectStorageUseSSL,
accessKey: meta.objectStorageAccessKey!,
secretKey: meta.objectStorageSecretKey!,
});
await minio.removeObject(meta.objectStorageBucket!, key);
}