forked from FoundKeyGang/FoundKey
Image for web publish (#3402)
* Image for Web * Add comment * Make main to original
This commit is contained in:
MeiMei
syuilo
parent
0863e5d379
commit
bcb04924ff
14 changed files with 283 additions and 43 deletions
|
|
@ -6,15 +6,24 @@ export default function(file: IDriveFile, thumbnail = false): string {
|
|||
|
||||
if (file.metadata.withoutChunks) {
|
||||
if (thumbnail) {
|
||||
return file.metadata.thumbnailUrl || file.metadata.url;
|
||||
return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || file.metadata.url;
|
||||
} else {
|
||||
return file.metadata.url;
|
||||
return file.metadata.webpublicUrl || file.metadata.url;
|
||||
}
|
||||
} else {
|
||||
if (thumbnail) {
|
||||
return `${config.drive_url}/${file._id}?thumbnail`;
|
||||
} else {
|
||||
return `${config.drive_url}/${file._id}`;
|
||||
return `${config.drive_url}/${file._id}?web`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function getOriginalUrl(file: IDriveFile) {
|
||||
if (file.metadata && file.metadata.url) {
|
||||
return file.metadata.url;
|
||||
}
|
||||
|
||||
const accessKey = file.metadata ? file.metadata.accessKey : null;
|
||||
return `${config.drive_url}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
|
||||
}
|
||||
|
|
|
|||
29
src/models/drive-file-webpublic.ts
Normal file
29
src/models/drive-file-webpublic.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import * as mongo from 'mongodb';
|
||||
import monkDb, { nativeDbConn } from '../db/mongodb';
|
||||
|
||||
const DriveFileWebpublic = monkDb.get<IDriveFileWebpublic>('driveFileWebpublics.files');
|
||||
DriveFileWebpublic.createIndex('metadata.originalId', { sparse: true, unique: true });
|
||||
export default DriveFileWebpublic;
|
||||
|
||||
export const DriveFileWebpublicChunk = monkDb.get('driveFileWebpublics.chunks');
|
||||
|
||||
export const getDriveFileWebpublicBucket = async (): Promise<mongo.GridFSBucket> => {
|
||||
const db = await nativeDbConn();
|
||||
const bucket = new mongo.GridFSBucket(db, {
|
||||
bucketName: 'driveFileWebpublics'
|
||||
});
|
||||
return bucket;
|
||||
};
|
||||
|
||||
export type IMetadata = {
|
||||
originalId: mongo.ObjectID;
|
||||
};
|
||||
|
||||
export type IDriveFileWebpublic = {
|
||||
_id: mongo.ObjectID;
|
||||
uploadDate: Date;
|
||||
md5: string;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
metadata: IMetadata;
|
||||
};
|
||||
|
|
@ -3,7 +3,7 @@ const deepcopy = require('deepcopy');
|
|||
import { pack as packFolder } from './drive-folder';
|
||||
import monkDb, { nativeDbConn } from '../db/mongodb';
|
||||
import isObjectId from '../misc/is-objectid';
|
||||
import getDriveFileUrl from '../misc/get-drive-file-url';
|
||||
import getDriveFileUrl, { getOriginalUrl } from '../misc/get-drive-file-url';
|
||||
|
||||
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
|
||||
DriveFile.createIndex('md5');
|
||||
|
|
@ -28,21 +28,48 @@ export type IMetadata = {
|
|||
_user: any;
|
||||
folderId: mongo.ObjectID;
|
||||
comment: string;
|
||||
|
||||
/**
|
||||
* リモートインスタンスから取得した場合の元URL
|
||||
*/
|
||||
uri?: string;
|
||||
|
||||
/**
|
||||
* URL for web(生成されている場合) or original
|
||||
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
|
||||
*/
|
||||
url?: string;
|
||||
|
||||
/**
|
||||
* URL for thumbnail (thumbnailがなければなし)
|
||||
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
|
||||
*/
|
||||
thumbnailUrl?: string;
|
||||
|
||||
/**
|
||||
* URL for original (web用が生成されてない場合はurlがoriginalを指す)
|
||||
* * オブジェクトストレージを利用している or リモートサーバーへの直リンクである 場合のみ
|
||||
*/
|
||||
webpublicUrl?: string;
|
||||
|
||||
accessKey?: string;
|
||||
|
||||
src?: string;
|
||||
deletedAt?: Date;
|
||||
|
||||
/**
|
||||
* このファイルの中身データがMongoDB内に保存されているのか否か
|
||||
* このファイルの中身データがMongoDB内に保存されていないか否か
|
||||
* オブジェクトストレージを利用している or リモートサーバーへの直リンクである
|
||||
* な場合は false になります
|
||||
* な場合は true になります
|
||||
*/
|
||||
withoutChunks?: boolean;
|
||||
|
||||
storage?: string;
|
||||
storageProps?: any;
|
||||
|
||||
/***
|
||||
* ObjectStorage の格納先の情報
|
||||
*/
|
||||
storageProps?: IStorageProps;
|
||||
isSensitive?: boolean;
|
||||
|
||||
/**
|
||||
|
|
@ -56,6 +83,25 @@ export type IMetadata = {
|
|||
isRemote?: boolean;
|
||||
};
|
||||
|
||||
export type IStorageProps = {
|
||||
/**
|
||||
* ObjectStorage key for original
|
||||
*/
|
||||
key: string;
|
||||
|
||||
/***
|
||||
* ObjectStorage key for thumbnail (thumbnailがなければなし)
|
||||
*/
|
||||
thumbnailKey?: string;
|
||||
|
||||
/***
|
||||
* ObjectStorage key for webpublic (webpublicがなければなし)
|
||||
*/
|
||||
webpublicKey?: string;
|
||||
|
||||
id?: string;
|
||||
};
|
||||
|
||||
export type IDriveFile = {
|
||||
_id: mongo.ObjectID;
|
||||
uploadDate: Date;
|
||||
|
|
@ -83,7 +129,8 @@ export function validateFileName(name: string): boolean {
|
|||
export const packMany = (
|
||||
files: any[],
|
||||
options?: {
|
||||
detail: boolean
|
||||
detail?: boolean
|
||||
self?: boolean,
|
||||
}
|
||||
) => {
|
||||
return Promise.all(files.map(f => pack(f, options)));
|
||||
|
|
@ -95,11 +142,13 @@ export const packMany = (
|
|||
export const pack = (
|
||||
file: any,
|
||||
options?: {
|
||||
detail: boolean
|
||||
detail?: boolean,
|
||||
self?: boolean,
|
||||
}
|
||||
) => new Promise<any>(async (resolve, reject) => {
|
||||
const opts = Object.assign({
|
||||
detail: false
|
||||
detail: false,
|
||||
self: false
|
||||
}, options);
|
||||
|
||||
let _file: any;
|
||||
|
|
@ -165,5 +214,9 @@ export const pack = (
|
|||
delete _target.isRemote;
|
||||
delete _target._user;
|
||||
|
||||
if (opts.self) {
|
||||
_target.url = getOriginalUrl(_file);
|
||||
}
|
||||
|
||||
resolve(_target);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -77,5 +77,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
sort: sort
|
||||
});
|
||||
|
||||
res(await packMany(files));
|
||||
res(await packMany(files, { detail: false, self: true }));
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -32,6 +32,6 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
if (file === null) {
|
||||
res({ file: null });
|
||||
} else {
|
||||
res({ file: await pack(file) });
|
||||
res({ file: await pack(file, { self: true }) });
|
||||
}
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ export default define(meta, (ps, user, app, file, cleanup) => new Promise(async
|
|||
|
||||
cleanup();
|
||||
|
||||
res(pack(driveFile));
|
||||
res(pack(driveFile, { self: true }));
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
|
||||
|
|
|
|||
|
|
@ -31,5 +31,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
'metadata.folderId': ps.folderId
|
||||
});
|
||||
|
||||
res(await Promise.all(files.map(file => pack(file))));
|
||||
res(await Promise.all(files.map(file => pack(file, { self: true }))));
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -41,7 +41,8 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
|
||||
// Serialize
|
||||
const _file = await pack(file, {
|
||||
detail: true
|
||||
detail: true,
|
||||
self: true
|
||||
});
|
||||
|
||||
res(_file);
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
});
|
||||
|
||||
// Serialize
|
||||
const fileObj = await pack(file);
|
||||
const fileObj = await pack(file, { self: true });
|
||||
|
||||
// Response
|
||||
res(fileObj);
|
||||
|
|
|
|||
|
|
@ -50,5 +50,5 @@ export const meta = {
|
|||
};
|
||||
|
||||
export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
||||
res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force)));
|
||||
res(pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true }));
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -65,5 +65,5 @@ export default define(meta, (ps, user) => new Promise(async (res, rej) => {
|
|||
sort: sort
|
||||
});
|
||||
|
||||
res(await packMany(files));
|
||||
res(await packMany(files, { self: true }));
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import * as send from 'koa-send';
|
|||
import * as mongodb from 'mongodb';
|
||||
import DriveFile, { getDriveFileBucket } from '../../models/drive-file';
|
||||
import DriveFileThumbnail, { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
|
||||
import DriveFileWebpublic, { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
|
||||
|
||||
const assets = `${__dirname}/../../server/file/assets/`;
|
||||
|
||||
|
|
@ -41,6 +42,11 @@ export default async function(ctx: Koa.Context) {
|
|||
}
|
||||
|
||||
const sendRaw = async () => {
|
||||
if (file.metadata && file.metadata.accessKey && file.metadata.accessKey != ctx.query['original']) {
|
||||
ctx.status = 403;
|
||||
return;
|
||||
}
|
||||
|
||||
const bucket = await getDriveFileBucket();
|
||||
const readable = bucket.openDownloadStream(fileId);
|
||||
readable.on('error', commonReadableHandlerGenerator(ctx));
|
||||
|
|
@ -60,6 +66,19 @@ export default async function(ctx: Koa.Context) {
|
|||
} else {
|
||||
await sendRaw();
|
||||
}
|
||||
} else if ('web' in ctx.query) {
|
||||
const web = await DriveFileWebpublic.findOne({
|
||||
'metadata.originalId': fileId
|
||||
});
|
||||
|
||||
if (web != null) {
|
||||
ctx.set('Content-Type', file.contentType);
|
||||
|
||||
const bucket = await getDriveFileWebpublicBucket();
|
||||
ctx.body = bucket.openDownloadStream(web._id);
|
||||
} else {
|
||||
await sendRaw();
|
||||
}
|
||||
} else {
|
||||
if ('download' in ctx.query) {
|
||||
ctx.set('Content-Disposition', 'attachment');
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import { publishMainStream, publishDriveStream } from '../../stream';
|
|||
import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
|
||||
import delFile from './delete-file';
|
||||
import config from '../../config';
|
||||
import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
|
||||
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
|
||||
import driveChart from '../../chart/drive';
|
||||
import perUserDriveChart from '../../chart/per-user-drive';
|
||||
|
|
@ -23,7 +24,71 @@ import fetchMeta from '../../misc/fetch-meta';
|
|||
|
||||
const log = debug('misskey:drive:add-file');
|
||||
|
||||
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: any): Promise<IDriveFile> {
|
||||
/***
|
||||
* Save file
|
||||
* @param path Path for original
|
||||
* @param name Name for original
|
||||
* @param type Content-Type for original
|
||||
* @param hash Hash for original
|
||||
* @param size Size for original
|
||||
* @param metadata
|
||||
*/
|
||||
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> {
|
||||
// #region webpublic
|
||||
let webpublic: Buffer;
|
||||
let webpublicExt = 'jpg';
|
||||
let webpublicType = 'image/jpeg';
|
||||
|
||||
if (!metadata.uri) { // from local instance
|
||||
log(`creating web image`);
|
||||
|
||||
if (['image/jpeg'].includes(type)) {
|
||||
webpublic = await sharp(path)
|
||||
.resize(2048, 2048, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true
|
||||
})
|
||||
.rotate()
|
||||
.jpeg({
|
||||
quality: 85,
|
||||
progressive: true
|
||||
})
|
||||
.toBuffer();
|
||||
} else if (['image/webp'].includes(type)) {
|
||||
webpublic = await sharp(path)
|
||||
.resize(2048, 2048, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true
|
||||
})
|
||||
.rotate()
|
||||
.webp({
|
||||
quality: 85
|
||||
})
|
||||
.toBuffer();
|
||||
|
||||
webpublicExt = 'webp';
|
||||
webpublicType = 'image/webp';
|
||||
} else if (['image/png'].includes(type)) {
|
||||
webpublic = await sharp(path)
|
||||
.resize(2048, 2048, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true
|
||||
})
|
||||
.rotate()
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
webpublicExt = 'png';
|
||||
webpublicType = 'image/png';
|
||||
} else {
|
||||
log(`web image not created (not an image)`);
|
||||
}
|
||||
} else {
|
||||
log(`web image not created (from remote)`);
|
||||
}
|
||||
// #endregion webpublic
|
||||
|
||||
// #region thumbnail
|
||||
let thumbnail: Buffer;
|
||||
let thumbnailExt = 'jpg';
|
||||
let thumbnailType = 'image/jpeg';
|
||||
|
|
@ -53,10 +118,9 @@ async function save(path: string, name: string, type: string, hash: string, size
|
|||
thumbnailExt = 'png';
|
||||
thumbnailType = 'image/png';
|
||||
}
|
||||
// #endregion thumbnail
|
||||
|
||||
if (config.drive && config.drive.storage == 'minio') {
|
||||
const minio = new Minio.Client(config.drive.config);
|
||||
|
||||
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
|
||||
|
||||
if (ext === '') {
|
||||
|
|
@ -66,33 +130,41 @@ async function save(path: string, name: string, type: string, hash: string, size
|
|||
}
|
||||
|
||||
const key = `${config.drive.prefix}/${uuid.v4()}${ext}`;
|
||||
const webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${webpublicExt}`;
|
||||
const thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${thumbnailExt}`;
|
||||
|
||||
log(`uploading original: ${key}`);
|
||||
const uploads = [
|
||||
upload(key, fs.createReadStream(path), type)
|
||||
];
|
||||
|
||||
if (webpublic) {
|
||||
log(`uploading webpublic: ${webpublicKey}`);
|
||||
uploads.push(upload(webpublicKey, webpublic, webpublicType));
|
||||
}
|
||||
|
||||
if (thumbnail) {
|
||||
log(`uploading thumbnail: ${thumbnailKey}`);
|
||||
uploads.push(upload(thumbnailKey, thumbnail, thumbnailType));
|
||||
}
|
||||
|
||||
await Promise.all(uploads);
|
||||
|
||||
const baseUrl = config.drive.baseUrl
|
||||
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
|
||||
|
||||
await minio.putObject(config.drive.bucket, key, fs.createReadStream(path), size, {
|
||||
'Content-Type': type,
|
||||
'Cache-Control': 'max-age=31536000, immutable'
|
||||
});
|
||||
|
||||
if (thumbnail) {
|
||||
await minio.putObject(config.drive.bucket, thumbnailKey, thumbnail, size, {
|
||||
'Content-Type': thumbnailType,
|
||||
'Cache-Control': 'max-age=31536000, immutable'
|
||||
});
|
||||
}
|
||||
|
||||
Object.assign(metadata, {
|
||||
withoutChunks: true,
|
||||
storage: 'minio',
|
||||
storageProps: {
|
||||
key: key,
|
||||
thumbnailKey: thumbnailKey
|
||||
webpublicKey: webpublic ? webpublicKey : null,
|
||||
thumbnailKey: thumbnail ? thumbnailKey : null,
|
||||
},
|
||||
url: `${ baseUrl }/${ key }`,
|
||||
webpublicUrl: webpublic ? `${ baseUrl }/${ webpublicKey }` : null,
|
||||
thumbnailUrl: thumbnail ? `${ baseUrl }/${ thumbnailKey }` : null
|
||||
});
|
||||
} as IMetadata);
|
||||
|
||||
const file = await DriveFile.insert({
|
||||
length: size,
|
||||
|
|
@ -105,29 +177,55 @@ async function save(path: string, name: string, type: string, hash: string, size
|
|||
|
||||
return file;
|
||||
} else {
|
||||
// Get MongoDB GridFS bucket
|
||||
const bucket = await getDriveFileBucket();
|
||||
// #region store original
|
||||
const originalDst = await getDriveFileBucket();
|
||||
|
||||
const file = await new Promise<IDriveFile>((resolve, reject) => {
|
||||
const writeStream = bucket.openUploadStream(name, {
|
||||
// web用(Exif削除済み)がある場合はオリジナルにアクセス制限
|
||||
if (webpublic) metadata.accessKey = uuid.v4();
|
||||
|
||||
const originalFile = await new Promise<IDriveFile>((resolve, reject) => {
|
||||
const writeStream = originalDst.openUploadStream(name, {
|
||||
contentType: type,
|
||||
metadata
|
||||
});
|
||||
|
||||
writeStream.once('finish', resolve);
|
||||
writeStream.on('error', reject);
|
||||
|
||||
fs.createReadStream(path).pipe(writeStream);
|
||||
});
|
||||
|
||||
log(`original stored to ${originalFile._id}`);
|
||||
// #endregion store original
|
||||
|
||||
// #region store webpublic
|
||||
if (webpublic) {
|
||||
const webDst = await getDriveFileWebpublicBucket();
|
||||
|
||||
const webFile = await new Promise<IDriveFile>((resolve, reject) => {
|
||||
const writeStream = webDst.openUploadStream(name, {
|
||||
contentType: webpublicType,
|
||||
metadata: {
|
||||
originalId: originalFile._id
|
||||
}
|
||||
});
|
||||
|
||||
writeStream.once('finish', resolve);
|
||||
writeStream.on('error', reject);
|
||||
writeStream.end(webpublic);
|
||||
});
|
||||
|
||||
log(`web stored ${webFile._id}`);
|
||||
}
|
||||
// #endregion store webpublic
|
||||
|
||||
if (thumbnail) {
|
||||
const thumbnailBucket = await getDriveFileThumbnailBucket();
|
||||
|
||||
await new Promise<IDriveFile>((resolve, reject) => {
|
||||
const tuhmFile = await new Promise<IDriveFile>((resolve, reject) => {
|
||||
const writeStream = thumbnailBucket.openUploadStream(name, {
|
||||
contentType: thumbnailType,
|
||||
metadata: {
|
||||
originalId: file._id
|
||||
originalId: originalFile._id
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -135,12 +233,23 @@ async function save(path: string, name: string, type: string, hash: string, size
|
|||
writeStream.on('error', reject);
|
||||
writeStream.end(thumbnail);
|
||||
});
|
||||
|
||||
log(`thumbnail stored ${tuhmFile._id}`);
|
||||
}
|
||||
|
||||
return file;
|
||||
return originalFile;
|
||||
}
|
||||
}
|
||||
|
||||
async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) {
|
||||
const minio = new Minio.Client(config.drive.config);
|
||||
|
||||
await minio.putObject(config.drive.bucket, key, stream, null, {
|
||||
'Content-Type': type,
|
||||
'Cache-Control': 'max-age=31536000, immutable'
|
||||
});
|
||||
}
|
||||
|
||||
async function deleteOldFile(user: IRemoteUser) {
|
||||
const oldFile = await DriveFile.findOne({
|
||||
_id: {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-
|
|||
import config from '../../config';
|
||||
import driveChart from '../../chart/drive';
|
||||
import perUserDriveChart from '../../chart/per-user-drive';
|
||||
import DriveFileWebpublic, { DriveFileWebpublicChunk } from '../../models/drive-file-webpublic';
|
||||
|
||||
export default async function(file: IDriveFile, isExpired = false) {
|
||||
if (file.metadata.storage == 'minio') {
|
||||
|
|
@ -20,6 +21,11 @@ export default async function(file: IDriveFile, isExpired = false) {
|
|||
const thumbnailObj = file.metadata.storageProps.thumbnailKey ? file.metadata.storageProps.thumbnailKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-thumbnail`;
|
||||
await minio.removeObject(config.drive.bucket, thumbnailObj);
|
||||
}
|
||||
|
||||
if (file.metadata.webpublicUrl) {
|
||||
const webpublicObj = file.metadata.storageProps.webpublicKey ? file.metadata.storageProps.webpublicKey : `${config.drive.prefix}/${file.metadata.storageProps.id}-original`;
|
||||
await minio.removeObject(config.drive.bucket, webpublicObj);
|
||||
}
|
||||
}
|
||||
|
||||
// チャンクをすべて削除
|
||||
|
|
@ -48,6 +54,20 @@ export default async function(file: IDriveFile, isExpired = false) {
|
|||
}
|
||||
//#endregion
|
||||
|
||||
//#region Web公開用もあれば削除
|
||||
const webpublic = await DriveFileWebpublic.findOne({
|
||||
'metadata.originalId': file._id
|
||||
});
|
||||
|
||||
if (webpublic) {
|
||||
await DriveFileWebpublicChunk.remove({
|
||||
files_id: webpublic._id
|
||||
});
|
||||
|
||||
await DriveFileWebpublic.remove({ _id: webpublic._id });
|
||||
}
|
||||
//#endregion
|
||||
|
||||
// 統計を更新
|
||||
driveChart.update(file, false);
|
||||
perUserDriveChart.update(file, false);
|
||||
|
|
|
|||
Loading…
Reference in a new issue