This commit is contained in:
syuilo 2018-07-24 01:58:11 +09:00
parent 337ecafa56
commit 5458b10774
4 changed files with 63 additions and 31 deletions

View file

@ -53,6 +53,22 @@ remoteDriveCapacityMb: 8
# Users cannot see remote images when they turn off "Show media from a remote server" setting. # Users cannot see remote images when they turn off "Show media from a remote server" setting.
preventCache: false preventCache: false
drive:
storage: 'db'
# OR
# storage: 'object-storage'
# service: 'minio'
# bucket:
# prefix:
# config:
# endPoint:
# port:
# secure:
# accessKey:
# secretKey:
# #
# Below settings are optional # Below settings are optional
# #

View file

@ -57,6 +57,7 @@
"@types/koa-views": "2.0.3", "@types/koa-views": "2.0.3",
"@types/koa__cors": "2.2.2", "@types/koa__cors": "2.2.2",
"@types/kue": "0.11.9", "@types/kue": "0.11.9",
"@types/minio": "6.0.2",
"@types/mkdirp": "0.5.2", "@types/mkdirp": "0.5.2",
"@types/mocha": "5.2.3", "@types/mocha": "5.2.3",
"@types/mongodb": "3.1.2", "@types/mongodb": "3.1.2",
@ -147,6 +148,7 @@
"kue": "0.11.6", "kue": "0.11.6",
"loader-utils": "1.1.0", "loader-utils": "1.1.0",
"mecab-async": "0.1.2", "mecab-async": "0.1.2",
"minio": "6.0.0",
"mkdirp": "0.5.1", "mkdirp": "0.5.1",
"mocha": "5.2.0", "mocha": "5.2.0",
"moji": "0.5.1", "moji": "0.5.1",

View file

@ -49,6 +49,14 @@ export type Source = {
remoteDriveCapacityMb: number; remoteDriveCapacityMb: number;
preventCacheRemoteFiles: boolean; preventCacheRemoteFiles: boolean;
drive?: {
storage: string;
bucket: string;
prefix: string;
service?: string;
config?: any;
};
/** /**
* ID * ID
*/ */

View file

@ -8,14 +8,14 @@ import * as _gm from 'gm';
import * as debug from 'debug'; import * as debug from 'debug';
import fileType = require('file-type'); import fileType = require('file-type');
const prominence = require('prominence'); const prominence = require('prominence');
import * as Minio from 'minio';
import * as uuid from 'uuid';
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file'; import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
import DriveFolder from '../../models/drive-folder'; import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file'; import { pack } from '../../models/drive-file';
import event, { publishDriveStream } from '../../stream'; import event, { publishDriveStream } from '../../stream';
import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import genThumbnail from '../../drive/gen-thumbnail';
import delFile from './delete-file'; import delFile from './delete-file';
import config from '../../config'; import config from '../../config';
@ -25,28 +25,43 @@ const gm = _gm.subClass({
const log = debug('misskey:drive:add-file'); const log = debug('misskey:drive:add-file');
const writeChunks = (name: string, readable: stream.Readable, type: string, metadata: any) => async function save(readable: stream.Readable, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
getDriveFileBucket() if (config.drive && config.drive.storage == 'object-storage') {
.then(bucket => new Promise((resolve, reject) => { if (config.drive.service == 'minio') {
const minio = new Minio.Client(config.drive.config);
const id = uuid.v4();
const obj = `${config.drive.prefix}/${id}`;
await minio.putObject(config.drive.bucket, obj, readable);
Object.assign(metadata, {
obj: id,
url: `${ config.drive.config.secure ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? ':' + config.drive.config.port : '' }/${ config.drive.bucket }/${ obj }`
});
const file = await DriveFile.insert({
length: size,
uploadDate: new Date(),
md5: hash,
filename: name,
metadata: metadata,
contentType: type
});
return file;
}
} else {
// Get MongoDB GridFS bucket
const bucket = await getDriveFileBucket();
return new Promise<IDriveFile>((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); const writeStream = bucket.openUploadStream(name, { contentType: type, metadata });
writeStream.once('finish', resolve); writeStream.once('finish', resolve);
writeStream.on('error', reject); writeStream.on('error', reject);
readable.pipe(writeStream); readable.pipe(writeStream);
})); });
}
const writeThumbnailChunks = (name: string, readable: stream.Readable, originalId: mongodb.ObjectID) => }
getDriveFileThumbnailBucket()
.then(bucket => new Promise((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
contentType: 'image/jpeg',
metadata: {
originalId
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
readable.pipe(writeStream);
}));
async function deleteOldFile(user: IRemoteUser) { async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({ const oldFile = await DriveFile.findOne({
@ -283,7 +298,7 @@ export default async function(
metadata: metadata, metadata: metadata,
contentType: mime contentType: mime
}) })
: await (writeChunks(detectedName, fs.createReadStream(path), mime, metadata) as Promise<IDriveFile>); : await (save(fs.createReadStream(path), detectedName, mime, hash, size, metadata));
log(`drive file has been created ${driveFile._id}`); log(`drive file has been created ${driveFile._id}`);
@ -293,16 +308,7 @@ export default async function(
publishDriveStream(user._id, 'file_created', packedFile); publishDriveStream(user._id, 'file_created', packedFile);
}); });
if (!metaOnly) { // TODO: サムネイル生成
try {
const thumb = await genThumbnail(driveFile);
if (thumb) {
await writeThumbnailChunks(detectedName, thumb, driveFile._id);
}
} catch (e) {
// noop
}
}
return driveFile; return driveFile;
} }