From 3a80b599860037f0b0f3221ef003413c548f66db Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Thu, 11 Oct 2018 21:14:20 +0900
Subject: [PATCH] =?UTF-8?q?=E4=B8=A6=E5=88=97=E3=81=AB=E5=87=A6=E7=90=86?=
 =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 package.json                  |   1 +
 src/tools/move-drive-files.ts | 118 ++++++++++++++++++----------------
 2 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/package.json b/package.json
index 19e618402..b6d4fccbc 100644
--- a/package.json
+++ b/package.json
@@ -169,6 +169,7 @@
 		"parse5": "5.1.0",
 		"portscanner": "2.2.0",
 		"progress-bar-webpack-plugin": "1.11.0",
+		"promise-limit": "2.7.0",
 		"promise-sequential": "1.1.1",
 		"pug": "2.0.3",
 		"punycode": "2.1.1",
diff --git a/src/tools/move-drive-files.ts b/src/tools/move-drive-files.ts
index 0c201ff8d..782ea664a 100644
--- a/src/tools/move-drive-files.ts
+++ b/src/tools/move-drive-files.ts
@@ -1,10 +1,12 @@
 import * as Minio from 'minio';
 import * as uuid from 'uuid';
-const sequential = require('promise-sequential');
-import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file';
+import * as promiseLimit from 'promise-limit';
+import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
 import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
 import config from '../config';
 
+const limit = promiseLimit(16);
+
 DriveFile.find({
 	$or: [{
 		withoutChunks: { $exists: false }
@@ -19,59 +21,63 @@ DriveFile.find({
 }).then(async files => {
 	console.log(`there is ${files.length} files`);
 
-	await sequential(files.map(file => async () => {
-		file = await DriveFile.findOne({ _id: file._id });
+	await Promise.all(files.map(file => limit(() => job(file))));
 
-		const minio = new Minio.Client(config.drive.config);
-
-		const name = file.filename;
-		const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const key = `${keyDir}/${name}`;
-		const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
-
-		const baseUrl = config.drive.baseUrl
-			|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
-
-		const bucket = await getDriveFileBucket();
-		const readable = bucket.openDownloadStream(file._id);
-
-		await minio.putObject(config.drive.bucket, key, readable, file.length, {
-			'Content-Type': file.contentType,
-			'Cache-Control': 'max-age=31536000, immutable'
-		});
-
-		await DriveFile.findOneAndUpdate({ _id: file._id }, {
-			$set: {
-				'metadata.withoutChunks': true,
-				'metadata.storage': 'minio',
-				'metadata.storageProps': {
-					key: key,
-					thumbnailKey: thumbnailKey
-				},
-				'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
-			}
-		});
-
-		// チャンクをすべて削除
-		await DriveFileChunk.remove({
-			files_id: file._id
-		});
-
-		//#region サムネイルもあれば削除
-		const thumbnail = await DriveFileThumbnail.findOne({
-			'metadata.originalId': file._id
-		});
-
-		if (thumbnail) {
-			await DriveFileThumbnailChunk.remove({
-				files_id: thumbnail._id
-			});
-
-			await DriveFileThumbnail.remove({ _id: thumbnail._id });
-		}
-		//#endregion
-
-		console.log('done', file._id);
-	}));
+	console.log('ALL DONE');
 });
+
+async function job(file: IDriveFile): Promise<any> {
+	file = await DriveFile.findOne({ _id: file._id });
+
+	const minio = new Minio.Client(config.drive.config);
+
+	const name = file.filename;
+	const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const key = `${keyDir}/${name}`;
+	const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
+
+	const baseUrl = config.drive.baseUrl
+		|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
+
+	const bucket = await getDriveFileBucket();
+	const readable = bucket.openDownloadStream(file._id);
+
+	await minio.putObject(config.drive.bucket, key, readable, file.length, {
+		'Content-Type': file.contentType,
+		'Cache-Control': 'max-age=31536000, immutable'
+	});
+
+	await DriveFile.findOneAndUpdate({ _id: file._id }, {
+		$set: {
+			'metadata.withoutChunks': true,
+			'metadata.storage': 'minio',
+			'metadata.storageProps': {
+				key: key,
+				thumbnailKey: thumbnailKey
+			},
+			'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
+		}
+	});
+
+	// チャンクをすべて削除
+	await DriveFileChunk.remove({
+		files_id: file._id
+	});
+
+	//#region サムネイルもあれば削除
+	const thumbnail = await DriveFileThumbnail.findOne({
+		'metadata.originalId': file._id
+	});
+
+	if (thumbnail) {
+		await DriveFileThumbnailChunk.remove({
+			files_id: thumbnail._id
+		});
+
+		await DriveFileThumbnail.remove({ _id: thumbnail._id });
+	}
+	//#endregion
+
+	console.log('done', file._id);
+}