Merge branch 'master' into greenkeeper/cafy-3.1.1

This commit is contained in:
こぴなたみぽ 2017-11-06 19:14:44 +09:00 committed by GitHub
commit 779011e5b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 360 additions and 233 deletions

View file

@ -18,22 +18,22 @@
"clean": "gulp clean", "clean": "gulp clean",
"cleanall": "gulp cleanall", "cleanall": "gulp cleanall",
"lint": "gulp lint", "lint": "gulp lint",
"test": "gulp test" "test": "gulp test"
}, },
"devDependencies": { "devDependencies": {
"@types/bcryptjs": "2.4.0", "@types/bcryptjs": "2.4.1",
"@types/body-parser": "1.16.5", "@types/body-parser": "1.16.7",
"@types/chai": "4.0.4", "@types/chai": "4.0.4",
"@types/chai-http": "3.0.3", "@types/chai-http": "3.0.3",
"@types/chalk": "0.4.31", "@types/chalk": "2.2.0",
"@types/compression": "0.0.34", "@types/compression": "0.0.34",
"@types/cors": "2.8.1", "@types/cors": "2.8.1",
"@types/debug": "0.0.30", "@types/debug": "0.0.30",
"@types/deep-equal": "1.0.1", "@types/deep-equal": "1.0.1",
"@types/elasticsearch": "5.0.14", "@types/elasticsearch": "5.0.17",
"@types/event-stream": "3.3.32", "@types/event-stream": "3.3.32",
"@types/express": "4.0.37", "@types/express": "4.0.37",
"@types/gm": "1.17.32", "@types/gm": "1.17.33",
"@types/gulp": "4.0.3", "@types/gulp": "4.0.3",
"@types/gulp-htmlmin": "1.3.30", "@types/gulp-htmlmin": "1.3.30",
"@types/gulp-mocha": "0.0.30", "@types/gulp-mocha": "0.0.30",
@ -41,30 +41,32 @@
"@types/gulp-replace": "0.0.30", "@types/gulp-replace": "0.0.30",
"@types/gulp-tslint": "3.6.31", "@types/gulp-tslint": "3.6.31",
"@types/gulp-typescript": "2.13.0", "@types/gulp-typescript": "2.13.0",
"@types/gulp-uglify": "0.0.30", "@types/gulp-uglify": "3.0.3",
"@types/gulp-util": "3.0.31", "@types/gulp-util": "3.0.33",
"@types/inquirer": "0.0.34", "@types/inquirer": "0.0.34",
"@types/is-root": "1.0.0", "@types/is-root": "1.0.0",
"@types/is-url": "1.2.28", "@types/is-url": "1.2.28",
"@types/js-yaml": "3.9.0", "@types/js-yaml": "3.9.1",
"@types/mocha": "2.2.43", "@types/mocha": "2.2.44",
"@types/mongodb": "2.2.13", "@types/mongodb": "2.2.13",
"@types/monk": "1.0.6", "@types/monk": "1.0.6",
"@types/morgan": "1.7.33", "@types/morgan": "1.7.35",
"@types/ms": "0.7.30", "@types/ms": "0.7.30",
"@types/multer": "1.3.2", "@types/multer": "1.3.5",
"@types/node": "8.0.33", "@types/node": "8.0.47",
"@types/ratelimiter": "2.1.28", "@types/ratelimiter": "2.1.28",
"@types/redis": "2.6.0", "@types/redis": "2.8.1",
"@types/request": "2.0.4", "@types/request": "2.0.7",
"@types/rimraf": "2.0.0", "@types/rimraf": "2.0.2",
"@types/riot": "3.6.0", "@types/riot": "3.6.1",
"@types/serve-favicon": "2.2.28", "@types/serve-favicon": "2.2.29",
"@types/uuid": "3.4.2", "@types/uuid": "3.4.3",
"@types/webpack": "3.0.14",
"@types/uuid": "3.4.3",
"@types/webpack": "3.0.13", "@types/webpack": "3.0.13",
"@types/webpack-stream": "3.2.7", "@types/webpack-stream": "3.2.8",
"@types/websocket": "0.0.34", "@types/websocket": "0.0.34",
"awesome-typescript-loader": "3.2.3", "awesome-typescript-loader": "3.3.0",
"chai": "4.1.2", "chai": "4.1.2",
"chai-http": "3.0.0", "chai-http": "3.0.0",
"css-loader": "0.28.7", "css-loader": "0.28.7",
@ -78,30 +80,31 @@
"gulp-rename": "1.2.2", "gulp-rename": "1.2.2",
"gulp-replace": "0.6.1", "gulp-replace": "0.6.1",
"gulp-tslint": "8.1.2", "gulp-tslint": "8.1.2",
"gulp-typescript": "3.2.2", "gulp-typescript": "3.2.3",
"gulp-uglify": "3.0.0", "gulp-uglify": "3.0.0",
"gulp-util": "3.0.8", "gulp-util": "3.0.8",
"mocha": "3.5.3", "mocha": "4.0.1",
"riot-tag-loader": "1.0.0", "riot-tag-loader": "1.0.0",
"string-replace-webpack-plugin": "0.1.3", "string-replace-webpack-plugin": "0.1.3",
"style-loader": "0.19.0", "style-loader": "0.19.0",
"stylus": "0.54.5", "stylus": "0.54.5",
"stylus-loader": "3.0.1", "stylus-loader": "3.0.1",
"swagger-jsdoc": "1.9.7", "swagger-jsdoc": "1.9.7",
"tslint": "5.7.0", "tslint": "5.8.0",
"uglify-es": "3.0.27", "uglify-es": "3.0.27",
"uglify-js": "git+https://github.com/mishoo/UglifyJS2.git#harmony", "uglify-js": "git+https://github.com/mishoo/UglifyJS2.git#harmony",
"uglifyjs-webpack-plugin": "1.0.0-beta.2", "uglifyjs-webpack-plugin": "1.0.1",
"webpack": "3.8.1" "webpack": "3.8.1"
}, },
"dependencies": { "dependencies": {
"@prezzemolo/rap": "0.1.2",
"accesses": "2.5.0", "accesses": "2.5.0",
"animejs": "2.2.0", "animejs": "2.2.0",
"autwh": "0.0.1", "autwh": "0.0.1",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"body-parser": "1.18.2", "body-parser": "1.18.2",
"cafy": "3.1.1", "cafy": "3.1.1",
"chalk": "2.1.0", "chalk": "2.3.0",
"compression": "1.7.1", "compression": "1.7.1",
"cors": "2.8.4", "cors": "2.8.4",
"cropperjs": "1.1.3", "cropperjs": "1.1.3",
@ -114,7 +117,7 @@
"elasticsearch": "13.3.1", "elasticsearch": "13.3.1",
"escape-regexp": "0.0.1", "escape-regexp": "0.0.1",
"express": "4.15.4", "express": "4.15.4",
"file-type": "6.2.0", "file-type": "7.2.0",
"fuckadblock": "3.2.1", "fuckadblock": "3.2.1",
"gm": "1.23.0", "gm": "1.23.0",
"inquirer": "3.3.0", "inquirer": "3.3.0",
@ -140,7 +143,7 @@
"redis": "2.8.0", "redis": "2.8.0",
"request": "2.83.0", "request": "2.83.0",
"rimraf": "2.6.2", "rimraf": "2.6.2",
"riot": "3.7.3", "riot": "3.7.4",
"rndstr": "1.0.0", "rndstr": "1.0.0",
"s-age": "1.1.0", "s-age": "1.1.0",
"serve-favicon": "2.4.5", "serve-favicon": "2.4.5",
@ -149,7 +152,7 @@
"tcp-port-used": "0.1.2", "tcp-port-used": "0.1.2",
"textarea-caret": "3.0.2", "textarea-caret": "3.0.2",
"ts-node": "3.3.0", "ts-node": "3.3.0",
"typescript": "2.5.3", "typescript": "2.6.1",
"uuid": "3.1.0", "uuid": "3.1.0",
"vhost": "3.0.2", "vhost": "3.0.2",
"websocket": "1.0.25", "websocket": "1.0.25",

View file

@ -4,14 +4,27 @@ import * as gm from 'gm';
import * as debug from 'debug'; import * as debug from 'debug';
import fileType = require('file-type'); import fileType = require('file-type');
import prominence = require('prominence'); import prominence = require('prominence');
import DriveFile from '../models/drive-file'; import DriveFile, { getGridFSBucket } from '../models/drive-file';
import DriveFolder from '../models/drive-folder'; import DriveFolder from '../models/drive-folder';
import serialize from '../serializers/drive-file'; import serialize from '../serializers/drive-file';
import event from '../event'; import event from '../event';
import config from '../../conf'; import config from '../../conf';
import { Duplex } from 'stream';
const log = debug('misskey:register-drive-file'); const log = debug('misskey:register-drive-file');
const addToGridFS = (name, binary, metadata): Promise<any> => new Promise(async (resolve, reject) => {
const dataStream = new Duplex();
dataStream.push(binary);
dataStream.push(null);
const bucket = await getGridFSBucket();
const writeStream = bucket.openUploadStream(name, { metadata });
writeStream.once('finish', (doc) => { resolve(doc); });
writeStream.on('error', reject);
dataStream.pipe(writeStream);
});
/** /**
* Add file to drive * Add file to drive
* *
@ -58,7 +71,7 @@ export default (
// Generate hash // Generate hash
const hash = crypto const hash = crypto
.createHash('sha256') .createHash('md5')
.update(data) .update(data)
.digest('hex') as string; .digest('hex') as string;
@ -67,8 +80,8 @@ export default (
if (!force) { if (!force) {
// Check if there is a file with the same hash // Check if there is a file with the same hash
const much = await DriveFile.findOne({ const much = await DriveFile.findOne({
user_id: user._id, md5: hash,
hash: hash 'metadata.user_id': user._id
}); });
if (much !== null) { if (much !== null) {
@ -82,13 +95,13 @@ export default (
// Calculate drive usage // Calculate drive usage
const usage = ((await DriveFile const usage = ((await DriveFile
.aggregate([ .aggregate([
{ $match: { user_id: user._id } }, { $match: { 'metadata.user_id': user._id } },
{ $project: { { $project: {
datasize: true length: true
}}, }},
{ $group: { { $group: {
_id: null, _id: null,
usage: { $sum: '$datasize' } usage: { $sum: '$length' }
}} }}
]))[0] || { ]))[0] || {
usage: 0 usage: 0
@ -131,21 +144,15 @@ export default (
} }
// Create DriveFile document // Create DriveFile document
const file = await DriveFile.insert({ const file = await addToGridFS(`${user._id}/${name}`, data, {
created_at: new Date(),
user_id: user._id, user_id: user._id,
folder_id: folder !== null ? folder._id : null, folder_id: folder !== null ? folder._id : null,
data: data,
datasize: size,
type: mime, type: mime,
name: name, name: name,
comment: comment, comment: comment,
hash: hash,
properties: properties properties: properties
}); });
delete file.data;
log(`drive file has been created ${file._id}`); log(`drive file has been created ${file._id}`);
resolve(file); resolve(file);

View file

@ -14,16 +14,16 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Calculate drive usage // Calculate drive usage
const usage = ((await DriveFile const usage = ((await DriveFile
.aggregate([ .aggregate([
{ $match: { user_id: user._id } }, { $match: { 'metadata.user_id': user._id } },
{ {
$project: { $project: {
datasize: true length: true
} }
}, },
{ {
$group: { $group: {
_id: null, _id: null,
usage: { $sum: '$datasize' } usage: { $sum: '$length' }
} }
} }
]))[0] || { ]))[0] || {

View file

@ -13,35 +13,35 @@ import serialize from '../../serializers/drive-file';
* @param {any} app * @param {any} app
* @return {Promise<any>} * @return {Promise<any>}
*/ */
module.exports = (params, user, app) => new Promise(async (res, rej) => { module.exports = async (params, user, app) => {
// Get 'limit' parameter // Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param'); if (limitErr) throw 'invalid limit param';
// Get 'since_id' parameter // Get 'since_id' parameter
const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
if (sinceIdErr) return rej('invalid since_id param'); if (sinceIdErr) throw 'invalid since_id param';
// Get 'max_id' parameter // Get 'max_id' parameter
const [maxId, maxIdErr] = $(params.max_id).optional.id().$; const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
if (maxIdErr) return rej('invalid max_id param'); if (maxIdErr) throw 'invalid max_id param';
// Check if both of since_id and max_id is specified // Check if both of since_id and max_id is specified
if (sinceId && maxId) { if (sinceId && maxId) {
return rej('cannot set since_id and max_id'); throw 'cannot set since_id and max_id';
} }
// Get 'folder_id' parameter // Get 'folder_id' parameter
const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
if (folderIdErr) return rej('invalid folder_id param'); if (folderIdErr) throw 'invalid folder_id param';
// Construct query // Construct query
const sort = { const sort = {
_id: -1 _id: -1
}; };
const query = { const query = {
user_id: user._id, 'metadata.user_id': user._id,
folder_id: folderId 'metadata.folder_id': folderId
} as any; } as any;
if (sinceId) { if (sinceId) {
sort._id = 1; sort._id = 1;
@ -57,14 +57,11 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
// Issue query // Issue query
const files = await DriveFile const files = await DriveFile
.find(query, { .find(query, {
fields: {
data: false
},
limit: limit, limit: limit,
sort: sort sort: sort
}); });
// Serialize // Serialize
res(await Promise.all(files.map(async file => const _files = await Promise.all(files.map(file => serialize(file)));
await serialize(file)))); return _files;
}); };

View file

@ -24,13 +24,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Issue query // Issue query
const files = await DriveFile const files = await DriveFile
.find({ .find({
name: name, 'metadata.name': name,
user_id: user._id, 'metadata.user_id': user._id,
folder_id: folderId 'metadata.folder_id': folderId
}, {
fields: {
data: false
}
}); });
// Serialize // Serialize

View file

@ -12,28 +12,26 @@ import serialize from '../../../serializers/drive-file';
* @param {any} user * @param {any} user
* @return {Promise<any>} * @return {Promise<any>}
*/ */
module.exports = (params, user) => new Promise(async (res, rej) => { module.exports = async (params, user) => {
// Get 'file_id' parameter // Get 'file_id' parameter
const [fileId, fileIdErr] = $(params.file_id).id().$; const [fileId, fileIdErr] = $(params.file_id).id().$;
if (fileIdErr) return rej('invalid file_id param'); if (fileIdErr) throw 'invalid file_id param';
// Fetch file // Fetch file
const file = await DriveFile const file = await DriveFile
.findOne({ .findOne({
_id: fileId, _id: fileId,
user_id: user._id 'metadata.user_id': user._id
}, {
fields: {
data: false
}
}); });
if (file === null) { if (file === null) {
return rej('file-not-found'); throw 'file-not-found';
} }
// Serialize // Serialize
res(await serialize(file, { const _file = await serialize(file, {
detail: true detail: true
})); });
});
return _file;
};

View file

@ -24,11 +24,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
const file = await DriveFile const file = await DriveFile
.findOne({ .findOne({
_id: fileId, _id: fileId,
user_id: user._id 'metadata.user_id': user._id
}, {
fields: {
data: false
}
}); });
if (file === null) { if (file === null) {
@ -38,7 +34,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
// Get 'name' parameter // Get 'name' parameter
const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$;
if (nameErr) return rej('invalid name param'); if (nameErr) return rej('invalid name param');
if (name) file.name = name; if (name) file.metadata.name = name;
// Get 'folder_id' parameter // Get 'folder_id' parameter
const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$; const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$;
@ -46,7 +42,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (folderId !== undefined) { if (folderId !== undefined) {
if (folderId === null) { if (folderId === null) {
file.folder_id = null; file.metadata.folder_id = null;
} else { } else {
// Fetch folder // Fetch folder
const folder = await DriveFolder const folder = await DriveFolder
@ -59,14 +55,14 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
return rej('folder-not-found'); return rej('folder-not-found');
} }
file.folder_id = folder._id; file.metadata.folder_id = folder._id;
} }
} }
DriveFile.update(file._id, { await DriveFile.update(file._id, {
$set: { $set: {
name: file.name, 'metadata.name': file.metadata.name,
folder_id: file.folder_id 'metadata.folder_id': file.metadata.folder_id
} }
}); });

View file

@ -30,6 +30,5 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
}); });
// Serialize // Serialize
res(await Promise.all(folders.map(async folder => res(await Promise.all(folders.map(folder => serialize(folder))));
await serialize(folder))));
}); });

View file

@ -4,7 +4,7 @@
import $ from 'cafy'; import $ from 'cafy';
import DriveFolder from '../../../models/drive-folder'; import DriveFolder from '../../../models/drive-folder';
import { isValidFolderName } from '../../../models/drive-folder'; import { isValidFolderName } from '../../../models/drive-folder';
import serialize from '../../../serializers/drive-file'; import serialize from '../../../serializers/drive-folder';
import event from '../../../event'; import event from '../../../event';
/** /**

View file

@ -54,9 +54,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => {
if (fileId !== undefined) { if (fileId !== undefined) {
file = await DriveFile.findOne({ file = await DriveFile.findOne({
_id: fileId, _id: fileId,
user_id: user._id 'metadata.user_id': user._id
}, {
data: false
}); });
if (file === null) { if (file === null) {

View file

@ -44,9 +44,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => {
// SELECT _id // SELECT _id
const entity = await DriveFile.findOne({ const entity = await DriveFile.findOne({
_id: mediaId, _id: mediaId,
user_id: user._id 'metadata.user_id': user._id
}, {
_id: true
}); });
if (entity === null) { if (entity === null) {

View file

@ -2,6 +2,7 @@
* Module dependencies * Module dependencies
*/ */
import $ from 'cafy'; import $ from 'cafy';
import rap from '@prezzemolo/rap';
import Post from '../../models/post'; import Post from '../../models/post';
import ChannelWatching from '../../models/channel-watching'; import ChannelWatching from '../../models/channel-watching';
import getFriends from '../../common/get-friends'; import getFriends from '../../common/get-friends';
@ -15,32 +16,33 @@ import serialize from '../../serializers/post';
* @param {any} app * @param {any} app
* @return {Promise<any>} * @return {Promise<any>}
*/ */
module.exports = (params, user, app) => new Promise(async (res, rej) => { module.exports = async (params, user, app) => {
// Get 'limit' parameter // Get 'limit' parameter
const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$;
if (limitErr) return rej('invalid limit param'); if (limitErr) throw 'invalid limit param';
// Get 'since_id' parameter // Get 'since_id' parameter
const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$;
if (sinceIdErr) return rej('invalid since_id param'); if (sinceIdErr) throw 'invalid since_id param';
// Get 'max_id' parameter // Get 'max_id' parameter
const [maxId, maxIdErr] = $(params.max_id).optional.id().$; const [maxId, maxIdErr] = $(params.max_id).optional.id().$;
if (maxIdErr) return rej('invalid max_id param'); if (maxIdErr) throw 'invalid max_id param';
// Check if both of since_id and max_id is specified // Check if both of since_id and max_id is specified
if (sinceId && maxId) { if (sinceId && maxId) {
return rej('cannot set since_id and max_id'); throw 'cannot set since_id and max_id';
} }
// ID list of the user itself and other users who the user follows const { followingIds, watchChannelIds } = await rap({
const followingIds = await getFriends(user._id); // ID list of the user itself and other users who the user follows
followingIds: getFriends(user._id),
// Watchしているチャンネルを取得 // Watchしているチャンネルを取得
const watches = await ChannelWatching.find({ watchChannelIds: ChannelWatching.find({
user_id: user._id, user_id: user._id,
// 削除されたドキュメントは除く // 削除されたドキュメントは除く
deleted_at: { $exists: false } deleted_at: { $exists: false }
}).then(watches => watches.map(w => w.channel_id))
}); });
//#region Construct query //#region Construct query
@ -65,7 +67,7 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
}, { }, {
// Watchしているチャンネルへの投稿 // Watchしているチャンネルへの投稿
channel_id: { channel_id: {
$in: watches.map(w => w.channel_id) $in: watchChannelIds
} }
}] }]
} as any; } as any;
@ -90,7 +92,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => {
}); });
// Serialize // Serialize
res(await Promise.all(timeline.map(async post => const _timeline = await Promise.all(timeline.map(post => serialize(post, user)));
await serialize(post, user) return _timeline;
))); };
});

View file

@ -1,11 +1,22 @@
import db from '../../db/mongodb'; import * as mongodb from 'mongodb';
import monkDb, { nativeDbConn } from '../../db/mongodb';
const collection = db.get('drive_files'); const collection = monkDb.get('drive_files.files');
(collection as any).createIndex('hash'); // fuck type definition (collection as any).createIndex('hash'); // fuck type definition
export default collection as any; // fuck type definition export default collection as any; // fuck type definition
const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
const db = await nativeDbConn();
const bucket = new mongodb.GridFSBucket(db, {
bucketName: 'drive_files'
});
return bucket;
};
export { getGridFSBucket };
export function validateFileName(name: string): boolean { export function validateFileName(name: string): boolean {
return ( return (
(name.trim().length > 0) && (name.trim().length > 0) &&

View file

@ -31,44 +31,40 @@ export default (
if (mongo.ObjectID.prototype.isPrototypeOf(file)) { if (mongo.ObjectID.prototype.isPrototypeOf(file)) {
_file = await DriveFile.findOne({ _file = await DriveFile.findOne({
_id: file _id: file
}, { });
fields: {
data: false
}
});
} else if (typeof file === 'string') { } else if (typeof file === 'string') {
_file = await DriveFile.findOne({ _file = await DriveFile.findOne({
_id: new mongo.ObjectID(file) _id: new mongo.ObjectID(file)
}, { });
fields: {
data: false
}
});
} else { } else {
_file = deepcopy(file); _file = deepcopy(file);
} }
// Rename _id to id if (!_file) return reject('invalid file arg.');
_file.id = _file._id;
delete _file._id;
delete _file.data; // rendered target
let _target: any = {};
_file.url = `${config.drive_url}/${_file.id}/${encodeURIComponent(_file.name)}`; _target.id = _file._id;
_target.created_at = _file.uploadDate;
if (opts.detail && _file.folder_id) { _target = Object.assign(_target, _file.metadata);
_target.url = `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`;
if (opts.detail && _target.folder_id) {
// Populate folder // Populate folder
_file.folder = await serializeDriveFolder(_file.folder_id, { _target.folder = await serializeDriveFolder(_target.folder_id, {
detail: true detail: true
}); });
} }
if (opts.detail && _file.tags) { if (opts.detail && _target.tags) {
// Populate tags // Populate tags
_file.tags = await _file.tags.map(async (tag: any) => _target.tags = await _target.tags.map(async (tag: any) =>
await serializeDriveTag(tag) await serializeDriveTag(tag)
); );
} }
resolve(_file); resolve(_target);
}); });

View file

@ -44,7 +44,7 @@ const self = (
}); });
const childFilesCount = await DriveFile.count({ const childFilesCount = await DriveFile.count({
folder_id: _folder.id 'metadata.folder_id': _folder.id
}); });
_folder.folders_count = childFoldersCount; _folder.folders_count = childFoldersCount;

View file

@ -12,6 +12,7 @@ import serializeChannel from './channel';
import serializeUser from './user'; import serializeUser from './user';
import serializeDriveFile from './drive-file'; import serializeDriveFile from './drive-file';
import parse from '../common/text'; import parse from '../common/text';
import rap from '@prezzemolo/rap';
/** /**
* Serialize a post * Serialize a post
@ -21,13 +22,13 @@ import parse from '../common/text';
* @param options? serialize options * @param options? serialize options
* @return response * @return response
*/ */
const self = ( const self = async (
post: string | mongo.ObjectID | IPost, post: string | mongo.ObjectID | IPost,
me?: string | mongo.ObjectID | IUser, me?: string | mongo.ObjectID | IUser,
options?: { options?: {
detail: boolean detail: boolean
} }
) => new Promise<any>(async (resolve, reject) => { ) => {
const opts = options || { const opts = options || {
detail: true, detail: true,
}; };
@ -56,6 +57,8 @@ const self = (
_post = deepcopy(post); _post = deepcopy(post);
} }
if (!_post) throw 'invalid post arg.';
const id = _post._id; const id = _post._id;
// Rename _id to id // Rename _id to id
@ -70,105 +73,120 @@ const self = (
} }
// Populate user // Populate user
_post.user = await serializeUser(_post.user_id, meId); _post.user = serializeUser(_post.user_id, meId);
// Populate app // Populate app
if (_post.app_id) { if (_post.app_id) {
_post.app = await serializeApp(_post.app_id); _post.app = serializeApp(_post.app_id);
} }
// Populate channel // Populate channel
if (_post.channel_id) { if (_post.channel_id) {
_post.channel = await serializeChannel(_post.channel_id); _post.channel = serializeChannel(_post.channel_id);
} }
// Populate media // Populate media
if (_post.media_ids) { if (_post.media_ids) {
_post.media = await Promise.all(_post.media_ids.map(async fileId => _post.media = Promise.all(_post.media_ids.map(fileId =>
await serializeDriveFile(fileId) serializeDriveFile(fileId)
)); ));
} }
// When requested a detailed post data // When requested a detailed post data
if (opts.detail) { if (opts.detail) {
// Get previous post info // Get previous post info
const prev = await Post.findOne({ _post.prev = (async () => {
user_id: _post.user_id, const prev = await Post.findOne({
_id: { user_id: _post.user_id,
$lt: id _id: {
} $lt: id
}, { }
fields: { }, {
_id: true fields: {
}, _id: true
sort: { },
_id: -1 sort: {
} _id: -1
}); }
_post.prev = prev ? prev._id : null; });
return prev ? prev._id : null;
})();
// Get next post info // Get next post info
const next = await Post.findOne({ _post.next = (async () => {
user_id: _post.user_id, const next = await Post.findOne({
_id: { user_id: _post.user_id,
$gt: id _id: {
} $gt: id
}, { }
fields: { }, {
_id: true fields: {
}, _id: true
sort: { },
_id: 1 sort: {
} _id: 1
}); }
_post.next = next ? next._id : null; });
return next ? next._id : null;
})();
if (_post.reply_id) { if (_post.reply_id) {
// Populate reply to post // Populate reply to post
_post.reply = await self(_post.reply_id, meId, { _post.reply = self(_post.reply_id, meId, {
detail: false detail: false
}); });
} }
if (_post.repost_id) { if (_post.repost_id) {
// Populate repost // Populate repost
_post.repost = await self(_post.repost_id, meId, { _post.repost = self(_post.repost_id, meId, {
detail: _post.text == null detail: _post.text == null
}); });
} }
// Poll // Poll
if (meId && _post.poll) { if (meId && _post.poll) {
const vote = await Vote _post.poll = (async (poll) => {
.findOne({ const vote = await Vote
user_id: meId, .findOne({
post_id: id user_id: meId,
}); post_id: id
});
if (vote != null) { if (vote != null) {
const myChoice = _post.poll.choices const myChoice = poll.choices
.filter(c => c.id == vote.choice)[0]; .filter(c => c.id == vote.choice)[0];
myChoice.is_voted = true; myChoice.is_voted = true;
} }
return poll;
})(_post.poll);
} }
// Fetch my reaction // Fetch my reaction
if (meId) { if (meId) {
const reaction = await Reaction _post.my_reaction = (async () => {
.findOne({ const reaction = await Reaction
user_id: meId, .findOne({
post_id: id, user_id: meId,
deleted_at: { $exists: false } post_id: id,
}); deleted_at: { $exists: false }
});
if (reaction) { if (reaction) {
_post.my_reaction = reaction.reaction; return reaction.reaction;
} }
return null;
})();
} }
} }
resolve(_post); // resolve promises in _post object
}); _post = await rap(_post);
return _post;
};
export default self; export default self;

View file

@ -8,6 +8,7 @@ import serializePost from './post';
import Following from '../models/following'; import Following from '../models/following';
import getFriends from '../common/get-friends'; import getFriends from '../common/get-friends';
import config from '../../conf'; import config from '../../conf';
import rap from '@prezzemolo/rap';
/** /**
* Serialize a user * Serialize a user
@ -55,6 +56,8 @@ export default (
_user = deepcopy(user); _user = deepcopy(user);
} }
if (!_user) return reject('invalid user arg.');
// Me // Me
const meId: mongo.ObjectID = me const meId: mongo.ObjectID = me
? mongo.ObjectID.prototype.isPrototypeOf(me) ? mongo.ObjectID.prototype.isPrototypeOf(me)
@ -104,26 +107,30 @@ export default (
if (meId && !meId.equals(_user.id)) { if (meId && !meId.equals(_user.id)) {
// If the user is following // If the user is following
const follow = await Following.findOne({ _user.is_following = (async () => {
follower_id: meId, const follow = await Following.findOne({
followee_id: _user.id, follower_id: meId,
deleted_at: { $exists: false } followee_id: _user.id,
}); deleted_at: { $exists: false }
_user.is_following = follow !== null; });
return follow !== null;
})();
// If the user is followed // If the user is followed
const follow2 = await Following.findOne({ _user.is_followed = (async () => {
follower_id: _user.id, const follow2 = await Following.findOne({
followee_id: meId, follower_id: _user.id,
deleted_at: { $exists: false } followee_id: meId,
}); deleted_at: { $exists: false }
_user.is_followed = follow2 !== null; });
return follow2 !== null;
})();
} }
if (opts.detail) { if (opts.detail) {
if (_user.pinned_post_id) { if (_user.pinned_post_id) {
// Populate pinned post // Populate pinned post
_user.pinned_post = await serializePost(_user.pinned_post_id, meId, { _user.pinned_post = serializePost(_user.pinned_post_id, meId, {
detail: true detail: true
}); });
} }
@ -132,23 +139,24 @@ export default (
const myFollowingIds = await getFriends(meId); const myFollowingIds = await getFriends(meId);
// Get following you know count // Get following you know count
const followingYouKnowCount = await Following.count({ _user.following_you_know_count = Following.count({
followee_id: { $in: myFollowingIds }, followee_id: { $in: myFollowingIds },
follower_id: _user.id, follower_id: _user.id,
deleted_at: { $exists: false } deleted_at: { $exists: false }
}); });
_user.following_you_know_count = followingYouKnowCount;
// Get followers you know count // Get followers you know count
const followersYouKnowCount = await Following.count({ _user.followers_you_know_count = Following.count({
followee_id: _user.id, followee_id: _user.id,
follower_id: { $in: myFollowingIds }, follower_id: { $in: myFollowingIds },
deleted_at: { $exists: false } deleted_at: { $exists: false }
}); });
_user.followers_you_know_count = followersYouKnowCount;
} }
} }
// resolve promises in _user object
_user = await rap(_user);
resolve(_user); resolve(_user);
}); });
/* /*

View file

@ -1,11 +1,38 @@
import * as mongo from 'monk';
import config from '../conf'; import config from '../conf';
const uri = config.mongodb.user && config.mongodb.pass const uri = config.mongodb.user && config.mongodb.pass
? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}` ? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`
: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`; : `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
/**
* monk
*/
import * as mongo from 'monk';
const db = mongo(uri); const db = mongo(uri);
export default db; export default db;
/**
* MongoDB native module (officialy)
*/
import * as mongodb from 'mongodb';
let mdb: mongodb.Db;
const nativeDbConn = async (): Promise<mongodb.Db> => {
if (mdb) return mdb;
const db = await ((): Promise<mongodb.Db> => new Promise((resolve, reject) => {
mongodb.MongoClient.connect(uri, (e, db) => {
if (e) return reject(e);
resolve(db);
});
}))();
mdb = db;
return db;
};
export { nativeDbConn };

View file

@ -9,7 +9,7 @@ import * as cors from 'cors';
import * as mongodb from 'mongodb'; import * as mongodb from 'mongodb';
import * as gm from 'gm'; import * as gm from 'gm';
import File from '../api/models/drive-file'; import DriveFile, { getGridFSBucket } from '../api/models/drive-file';
/** /**
* Init app * Init app
@ -97,17 +97,28 @@ app.get('/:id', async (req, res) => {
return; return;
} }
const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); const fileId = new mongodb.ObjectID(req.params.id);
const file = await DriveFile.findOne({ _id: fileId });
if (file == null) { if (file == null) {
res.status(404).sendFile(`${__dirname} / assets / dummy.png`); res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
return;
} else if (file.data == null) {
res.sendStatus(400);
return; return;
} }
send(file.data.buffer, file.type, req, res); const bucket = await getGridFSBucket();
const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
const chunks = [];
const readableStream = bucket.openDownloadStream(id);
readableStream.on('data', chunk => {
chunks.push(chunk);
});
readableStream.on('end', () => {
resolve(Buffer.concat(chunks));
});
}))(fileId);
send(buffer, file.metadata.type, req, res);
}); });
app.get('/:id/:name', async (req, res) => { app.get('/:id/:name', async (req, res) => {
@ -117,17 +128,28 @@ app.get('/:id/:name', async (req, res) => {
return; return;
} }
const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); const fileId = new mongodb.ObjectID(req.params.id);
const file = await DriveFile.findOne({ _id: fileId });
if (file == null) { if (file == null) {
res.status(404).sendFile(`${__dirname}/assets/dummy.png`); res.status(404).sendFile(`${__dirname}/assets/dummy.png`);
return; return;
} else if (file.data == null) {
res.sendStatus(400);
return;
} }
send(file.data.buffer, file.type, req, res); const bucket = await getGridFSBucket();
const buffer = await ((id): Promise<Buffer> => new Promise((resolve, reject) => {
const chunks = [];
const readableStream = bucket.openDownloadStream(id);
readableStream.on('data', chunk => {
chunks.push(chunk);
});
readableStream.on('end', () => {
resolve(Buffer.concat(chunks));
});
}))(fileId);
send(buffer, file.metadata.type, req, res);
}); });
module.exports = app; module.exports = app;

View file

@ -1152,9 +1152,12 @@ async function insertHimawari(opts) {
} }
async function insertDriveFile(opts) { async function insertDriveFile(opts) {
return await db.get('drive_files').insert(Object.assign({ return await db.get('drive_files.files').insert({
name: 'strawberry-pasta.png' length: opts.datasize,
}, opts)); metadata: Object.assign({
name: 'strawberry-pasta.png'
}, opts)
});
} }
async function insertDriveFolder(opts) { async function insertDriveFolder(opts) {

View file

@ -0,0 +1,49 @@
// for Node.js interpret
const { default: db } = require('../../built/db/mongodb')
const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file')
const { Duplex } = require('stream')
const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => {
const writeStream = bucket.openUploadStreamWithId(...rest)
const dataStream = new Duplex()
dataStream.push(buffer)
dataStream.push(null)
writeStream.once('finish', resolve)
writeStream.on('error', reject)
dataStream.pipe(writeStream)
})
const migrateToGridFS = async (doc) => {
const id = doc._id
const buffer = doc.data.buffer
const created_at = doc.created_at
delete doc._id
delete doc.created_at
delete doc.datasize
delete doc.hash
delete doc.data
const bucket = await getGridFSBucket()
const added = await writeToGridFS(bucket, buffer, id, `${id}/${doc.name}`, { metadata: doc })
const result = await DriveFile.update(id, {
$set: {
uploadDate: created_at
}
})
return added && result.ok === 1
}
const main = async () => {
const docs = await db.get('drive_files').find()
const all = await Promise.all(docs.map(migrateToGridFS))
return all
}
main().then(console.dir).catch(console.error)