Image for web publish (#3402)

* Image for Web

* Add comment

* Make main to original
This commit is contained in:
MeiMei 2018-11-26 04:25:48 +09:00 committed by syuilo
parent 0863e5d379
commit bcb04924ff
14 changed files with 283 additions and 43 deletions

View file

@ -6,15 +6,24 @@ export default function(file: IDriveFile, thumbnail = false): string {
if (file.metadata.withoutChunks) { if (file.metadata.withoutChunks) {
if (thumbnail) { if (thumbnail) {
return file.metadata.thumbnailUrl || file.metadata.url; return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || file.metadata.url;
} else { } else {
return file.metadata.url; return file.metadata.webpublicUrl || file.metadata.url;
} }
} else { } else {
if (thumbnail) { if (thumbnail) {
return `${config.drive_url}/${file._id}?thumbnail`; return `${config.drive_url}/${file._id}?thumbnail`;
} else { } else {
return `${config.drive_url}/${file._id}`; return `${config.drive_url}/${file._id}?web`;
} }
} }
} }
export function getOriginalUrl(file: IDriveFile) {
if (file.metadata && file.metadata.url) {
return file.metadata.url;
}
const accessKey = file.metadata ? file.metadata.accessKey : null;
return `${config.drive_url}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
}

View file

@ -0,0 +1,29 @@
import * as mongo from 'mongodb';
import monkDb, { nativeDbConn } from '../db/mongodb';
const DriveFileWebpublic = monkDb.get<IDriveFileWebpublic>('driveFileWebpublics.files');
DriveFileWebpublic.createIndex('metadata.originalId', { sparse: true, unique: true });
export default DriveFileWebpublic;
export const DriveFileWebpublicChunk = monkDb.get('driveFileWebpublics.chunks');
export const getDriveFileWebpublicBucket = async (): Promise<mongo.GridFSBucket> => {
const db = await nativeDbConn();
const bucket = new mongo.GridFSBucket(db, {
bucketName: 'driveFileWebpublics'
});
return bucket;
};
export type IMetadata = {
originalId: mongo.ObjectID;
};
export type IDriveFileWebpublic = {
_id: mongo.ObjectID;
uploadDate: Date;
md5: string;
filename: string;
contentType: string;
metadata: IMetadata;
};

View file

@ -3,7 +3,7 @@ const deepcopy = require('deepcopy');
import { pack as packFolder } from './drive-folder'; import { pack as packFolder } from './drive-folder';
import monkDb, { nativeDbConn } from '../db/mongodb'; import monkDb, { nativeDbConn } from '../db/mongodb';
import isObjectId from '../misc/is-objectid'; import isObjectId from '../misc/is-objectid';
import getDriveFileUrl from '../misc/get-drive-file-url'; import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url';
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files'); const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
DriveFile.createIndex('md5'); DriveFile.createIndex('md5');
@ -28,21 +28,48 @@ export type IMetadata = {
_user: any; _user: any;
folderId: mongo.ObjectID; folderId: mongo.ObjectID;
comment: string; comment: string;
/**
* URL
*/
uri?: string; uri?: string;
/**
* URL for web() or original
* * or
*/
url?: string; url?: string;
/**
* URL for thumbnail (thumbnailがなければなし)
* * or
*/
thumbnailUrl?: string; thumbnailUrl?: string;
/**
* URL for original (web用が生成されてない場合はurlがoriginalを指す)
* * or
*/
webpublicUrl?: string;
accessKey?: string;
src?: string; src?: string;
deletedAt?: Date; deletedAt?: Date;
/** /**
* MongoDB内に保存されているのか否か * MongoDB内に保存されていないか否か
* or * or
* false * true
*/ */
withoutChunks?: boolean; withoutChunks?: boolean;
storage?: string; storage?: string;
storageProps?: any;
/***
* ObjectStorage
*/
storageProps?: IStorageProps;
isSensitive?: boolean; isSensitive?: boolean;
/** /**
@ -56,6 +83,25 @@ export type IMetadata = {
isRemote?: boolean; isRemote?: boolean;
}; };
export type IStorageProps = {
/**
* ObjectStorage key for original
*/
key: string;
/***
* ObjectStorage key for thumbnail (thumbnailがなければなし)
*/
thumbnailKey?: string;
/***
* ObjectStorage key for webpublic (webpublicがなければなし)
*/
webpublicKey?: string;
id?: string;
};
export type IDriveFile = { export type IDriveFile = {
_id: mongo.ObjectID; _id: mongo.ObjectID;
uploadDate: Date; uploadDate: Date;
@ -83,7 +129,8 @@ export function validateFileName(name: string): boolean {
export const packMany = ( export const packMany = (
files: any[], files: any[],
options?: { options?: {
detail: boolean detail?: boolean
self?: boolean,
} }
) => { ) => {
return Promise.all(files.map(f => pack(f, options))); return Promise.all(files.map(f => pack(f, options)));
@ -95,11 +142,13 @@ export const packMany = (
export const pack = ( export const pack = (
file: any, file: any,
options?: { options?: {
detail: boolean detail?: boolean,
self?: boolean,
} }
) => new Promise<any>(async (resolve, reject) => { ) => new Promise<any>(async (resolve, reject) => {
const opts = Object.assign({ const opts = Object.assign({
detail: false detail: false,
self: false
}, options); }, options);
let _file: any; let _file: any;
@ -165,5 +214,9 @@ export const pack = (
delete _target.isRemote; delete _target.isRemote;
delete _target._user; delete _target._user;
if (opts.self) {
_target.url = getOriginalUrl(_file);
}
resolve(_target); resolve(_target);
}); });

View file

@ -77,5 +77,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort sort: sort
}); });
res(await packMany(files)); res(await packMany(files, { detail: false, self: true }));
})); }));

View file

@ -32,6 +32,6 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
if (file === null) { if (file === null) {
res({ file: null }); res({ file: null });
} else { } else {
res({ file: await pack(file) }); res({ file: await pack(file, { self: true }) });
} }
})); }));

View file

@ -74,7 +74,7 @@ export default define(meta, (ps, user, app, file, cleanup) => new Promise(async
cleanup(); cleanup();
res(pack(driveFile)); res(pack(driveFile, { self: true }));
} catch (e) { } catch (e) {
console.error(e); console.error(e);

View file

@ -31,5 +31,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
'metadata.folderId': ps.folderId 'metadata.folderId': ps.folderId
}); });
res(await Promise.all(files.map(file => pack(file)))); res(await Promise.all(files.map(file => pack(file, { self: true }))));
})); }));

View file

@ -41,7 +41,8 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
// Serialize // Serialize
const _file = await pack(file, { const _file = await pack(file, {
detail: true detail: true,
self: true
}); });
res(_file); res(_file);

View file

@ -111,7 +111,7 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
}); });
// Serialize // Serialize
const fileObj = await pack(file); const fileObj = await pack(file, { self: true });
// Response // Response
res(fileObj); res(fileObj);

View file

@ -50,5 +50,5 @@ export const meta = {
}; };
export default define(meta, (ps, user) => new Promise(async (res, rej) => { export default define(meta, (ps, user) => new Promise(async (res, rej) => {
res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force))); res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true }));
})); }));

View file

@ -65,5 +65,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
sort: sort sort: sort
}); });
res(await packMany(files)); res(await packMany(files, { self: true }));
})); }));

View file

@ -3,6 +3,7 @@ import * as send from 'koa-send';
import * as mongodb from 'mongodb'; import * as mongodb from 'mongodb';
import DriveFile, { getDriveFileBucket } from '../../models/drive-file'; import DriveFile, { getDriveFileBucket } from '../../models/drive-file';
import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
const assets = `${__dirname}/../../server/file/assets/`; const assets = `${__dirname}/../../server/file/assets/`;
@ -41,6 +42,11 @@ export default async function(ctx: Koa.Context) {
} }
const sendRaw = async () => { const sendRaw = async () => {
if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) {
ctx.status = 403;
return;
}
const bucket = await getDriveFileBucket(); const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(fileId); const readable = bucket.openDownloadStream(fileId);
readable.on('error', commonReadableHandlerGenerator(ctx)); readable.on('error', commonReadableHandlerGenerator(ctx));
@ -60,6 +66,19 @@ export default async function(ctx: Koa.Context) {
} else { } else {
await sendRaw(); await sendRaw();
} }
} else if ('web' in ctx.query) {
const web = await DriveFileWebpublic.findOne({
'metadata.originalId': fileId
});
if (web != null) {
ctx.set('Content-Type', file.contentType);
const bucket = await getDriveFileWebpublicBucket();
ctx.body = bucket.openDownloadStream(web._id);
} else {
await sendRaw();
}
} else { } else {
if ('download' in ctx.query) { if ('download' in ctx.query) {
ctx.set('Content-Disposition', 'attachment'); ctx.set('Content-Disposition', 'attachment');

View file

@ -16,6 +16,7 @@ import { publishMainStream, publishDriveStream } from '../../stream';
import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import delFile from './delete-file'; import delFile from './delete-file';
import config from '../../config'; import config from '../../config';
import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import driveChart from '../../chart/drive'; import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive'; import perUserDriveChart from '../../chart/per-user-drive';
@ -23,7 +24,71 @@ import fetchMeta from '../../misc/fetch-meta';
const log = debug('misskey:drive:add-file'); const log = debug('misskey:drive:add-file');
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> { /***
* Save file
* @param path Path for original
* @param name Name for original
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param metadata
*/
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> {
// #region webpublic
let webpublic: Buffer;
let webpublicExt = 'jpg';
let webpublicType = 'image/jpeg';
if (!metadata.uri) { // from local instance
log(`creating web image`);
if (['image/jpeg'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.jpeg({
quality: 85,
progressive: true
})
.toBuffer();
} else if (['image/webp'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.webp({
quality: 85
})
.toBuffer();
webpublicExt = 'webp';
webpublicType = 'image/webp';
} else if (['image/png'].includes(type)) {
webpublic = await sharp(path)
.resize(2048, 2048, {
fit: 'inside',
withoutEnlargement: true
})
.rotate()
.png()
.toBuffer();
webpublicExt = 'png';
webpublicType = 'image/png';
} else {
log(`web image not created (not an image)`);
}
} else {
log(`web image not created (from remote)`);
}
// #endregion webpublic
// #region thumbnail
let thumbnail: Buffer; let thumbnail: Buffer;
let thumbnailExt = 'jpg'; let thumbnailExt = 'jpg';
let thumbnailType = 'image/jpeg'; let thumbnailType = 'image/jpeg';
@ -53,10 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size
thumbnailExt = 'png'; thumbnailExt = 'png';
thumbnailType = 'image/png'; thumbnailType = 'image/png';
} }
// #endregion thumbnail
if (config.drive && config.drive.storage == 'minio') { if (config.drive && config.drive.storage == 'minio') {
const minio = new Minio.Client(config.drive.config);
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') { if (ext === '') {
@ -66,33 +130,41 @@ async function save(path: string, name: string, type: string, hash: string, size
} }
const key = `${config.drive.prefix}/${uuid.v4()}${ext}`; const key = `${config.drive.prefix}/${uuid.v4()}${ext}`;
const webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${webpublicExt}`;
const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`; const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`;
log(`uploading original: ${key}`);
const uploads = [
upload(key, fs.createReadStream(path), type)
];
if (webpublic) {
log(`uploading webpublic: ${webpublicKey}`);
uploads.push(upload(webpublicKey, webpublic, webpublicType));
}
if (thumbnail) {
log(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(upload(thumbnailKey, thumbnail, thumbnailType));
}
await Promise.all(uploads);
const baseUrl = config.drive.baseUrl const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
await minio.putObject(config.drive.bucket, key, fs.createReadStream(path), size, {
'Content-Type': type,
'Cache-Control': 'max-age=31536000, immutable'
});
if (thumbnail) {
await minio.putObject(config.drive.bucket, thumbnailKey, thumbnail, size, {
'Content-Type': thumbnailType,
'Cache-Control': 'max-age=31536000, immutable'
});
}
Object.assign(metadata, { Object.assign(metadata, {
withoutChunks: true, withoutChunks: true,
storage: 'minio', storage: 'minio',
storageProps: { storageProps: {
key: key, key: key,
thumbnailKey: thumbnailKey webpublicKey: webpublic ? webpublicKey : null,
thumbnailKey: thumbnail ? thumbnailKey : null,
}, },
url: `${ baseUrl }/${ key }`, url: `${ baseUrl }/${ key }`,
webpublicUrl: webpublic ? `${ baseUrl }/${ webpublicKey }` : null,
thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null
}); } as IMetadata);
const file = await DriveFile.insert({ const file = await DriveFile.insert({
length: size, length: size,
@ -105,29 +177,55 @@ async function save(path: string, name: string, type: string, hash: string, size
return file; return file;
} else { } else {
// Get MongoDB GridFS bucket // #region store original
const bucket = await getDriveFileBucket(); const originalDst = await getDriveFileBucket();
const file = await new Promise<IDriveFile>((resolve, reject) => { // web用(Exif削除済み)がある場合はオリジナルにアクセス制限
const writeStream = bucket.openUploadStream(name, { if (webpublic) metadata.accessKey = uuid.v4();
const originalFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = originalDst.openUploadStream(name, {
contentType: type, contentType: type,
metadata metadata
}); });
writeStream.once('finish', resolve); writeStream.once('finish', resolve);
writeStream.on('error', reject); writeStream.on('error', reject);
fs.createReadStream(path).pipe(writeStream); fs.createReadStream(path).pipe(writeStream);
}); });
log(`original stored to ${originalFile._id}`);
// #endregion store original
// #region store webpublic
if (webpublic) {
const webDst = await getDriveFileWebpublicBucket();
const webFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = webDst.openUploadStream(name, {
contentType: webpublicType,
metadata: {
originalId: originalFile._id
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
writeStream.end(webpublic);
});
log(`web stored ${webFile._id}`);
}
// #endregion store webpublic
if (thumbnail) { if (thumbnail) {
const thumbnailBucket = await getDriveFileThumbnailBucket(); const thumbnailBucket = await getDriveFileThumbnailBucket();
await new Promise<IDriveFile>((resolve, reject) => { const tuhmFile = await new Promise<IDriveFile>((resolve, reject) => {
const writeStream = thumbnailBucket.openUploadStream(name, { const writeStream = thumbnailBucket.openUploadStream(name, {
contentType: thumbnailType, contentType: thumbnailType,
metadata: { metadata: {
originalId: file._id originalId: originalFile._id
} }
}); });
@ -135,12 +233,23 @@ async function save(path: string, name: string, type: string, hash: string, size
writeStream.on('error', reject); writeStream.on('error', reject);
writeStream.end(thumbnail); writeStream.end(thumbnail);
}); });
log(`thumbnail stored ${tuhmFile._id}`);
} }
return file; return originalFile;
} }
} }
async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) {
const minio = new Minio.Client(config.drive.config);
await minio.putObject(config.drive.bucket, key, stream, null, {
'Content-Type': type,
'Cache-Control': 'max-age=31536000, immutable'
});
}
async function deleteOldFile(user: IRemoteUser) { async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({ const oldFile = await DriveFile.findOne({
_id: { _id: {

View file

@ -4,6 +4,7 @@ import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-
import config from '../../config'; import config from '../../config';
import driveChart from '../../chart/drive'; import driveChart from '../../chart/drive';
import perUserDriveChart from '../../chart/per-user-drive'; import perUserDriveChart from '../../chart/per-user-drive';
import DriveFileWebpublic, { DriveFileWebpublicChunk } from '../../models/drive-file-webpublic';
export default async function(file: IDriveFile, isExpired = false) { export default async function(file: IDriveFile, isExpired = false) {
if (file.metadata.storage == 'minio') { if (file.metadata.storage == 'minio') {
@ -20,6 +21,11 @@ export default async function(file: IDriveFile, isExpired = false) {
const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`; const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`;
await minio.removeObject(config.drive.bucket, thumbnailObj); await minio.removeObject(config.drive.bucket, thumbnailObj);
} }
if (file.metadata.webpublicUrl) {
const webpublicObj = file.metadata.storageProps.webpublicKey ? file.metadata.storageProps.webpublicKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-original`;
await minio.removeObject(config.drive.bucket, webpublicObj);
}
} }
// チャンクをすべて削除 // チャンクをすべて削除
@ -48,6 +54,20 @@ export default async function(file: IDriveFile, isExpired = false) {
} }
//#endregion //#endregion
//#region Web公開用もあれば削除
const webpublic = await DriveFileWebpublic.findOne({
'metadata.originalId': file._id
});
if (webpublic) {
await DriveFileWebpublicChunk.remove({
files_id: webpublic._id
});
await DriveFileWebpublic.remove({ _id: webpublic._id });
}
//#endregion
// 統計を更新 // 統計を更新
driveChart.update(file, false); driveChart.update(file, false);
perUserDriveChart.update(file, false); perUserDriveChart.update(file, false);