puniko/FoundKey
1
0
Fork 0
forked from FoundKeyGang/FoundKey
Code Issues Pull requests Projects Releases Packages Wiki Activity

並列に処理するように

Browse source
This commit is contained in:
syuilo 2018-10-11 21:14:20 +09:00
parent 07560a4fdd
commit 3a80b59986
No known key found for this signature in database
GPG key ID: BDC4C49D06AB9D69
2 changed files with 63 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -169,6 +169,7 @@
"parse5": "5.1.0", "parse5": "5.1.0",
"portscanner": "2.2.0", "portscanner": "2.2.0",
"progress-bar-webpack-plugin": "1.11.0", "progress-bar-webpack-plugin": "1.11.0",
"promise-limit": "2.7.0",
"promise-sequential": "1.1.1", "promise-sequential": "1.1.1",
"pug": "2.0.3", "pug": "2.0.3",
"punycode": "2.1.1", "punycode": "2.1.1",

16
src/tools/move-drive-files.ts
View file

@ -1,10 +1,12 @@
import * as Minio from 'minio'; import * as Minio from 'minio';
import * as uuid from 'uuid'; import * as uuid from 'uuid';
const sequential = require('promise-sequential'); import * as promiseLimit from 'promise-limit';
import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file'; import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
import config from '../config'; import config from '../config';
const limit = promiseLimit(16);
DriveFile.find({ DriveFile.find({
$or: [{ $or: [{
withoutChunks: { $exists: false } withoutChunks: { $exists: false }
@ -19,7 +21,12 @@ DriveFile.find({
}).then(async files => { }).then(async files => {
console.log(`there is ${files.length} files`); console.log(`there is ${files.length} files`);
await sequential(files.map(file => async () => { await Promise.all(files.map(file => limit(() => job(file))));
console.log('ALL DONE');
});
async function job(file: IDriveFile): Promise<any> {
file = await DriveFile.findOne({ _id: file._id }); file = await DriveFile.findOne({ _id: file._id });
const minio = new Minio.Client(config.drive.config); const minio = new Minio.Client(config.drive.config);
@ -73,5 +80,4 @@ DriveFile.find({
//#endregion //#endregion
console.log('done', file._id); console.log('done', file._id);
})); }
});