Merge pull request #763 from syuilo/#757

#757

.gitignore

@@ -3,6 +3,7 @@
 /node_modules
 /built
 /uploads
+/data
 npm-debug.log
 *.pem
 run.bat

locales/en.yml

@@ -22,6 +22,14 @@ common:
     confused: "Confused"
     pudding: "Pudding"
 
+  post_categories:
+    music: "Music"
+    game: "Video Game"
+    anime: "Anime"
+    it: "IT"
+    gadgets: "Gadgets"
+    photography: "Photography"
+
   input-message-here: "Enter message here"
   send: "Send"
   delete: "Delete"
@@ -80,6 +88,9 @@ common:
     mk-post-menu:
       pin: "Pin"
       pinned: "Pinned"
+      select: "Select category"
+      categorize: "Accept"
+      categorized: "Category reported. Thank you!"
 
     mk-reaction-picker:
       choose-reaction: "Pick your reaction"
@@ -375,6 +386,7 @@ mobile:
       twitter-integration: "Twitter integration"
       signin-history: "Sign in history"
       api: "API"
+      link: "MisskeyLink"
       settings: "Settings"
       signout: "Sign out"
 

locales/ja.yml

@@ -22,6 +22,14 @@ common:
     confused: "こまこまのこまり"
     pudding: "Pudding"
 
+  post_categories:
+    music: "音楽"
+    game: "ゲーム"
+    anime: "アニメ"
+    it: "IT"
+    gadgets: "ガジェット"
+    photography: "写真"
+
   input-message-here: "ここにメッセージを入力"
   send: "送信"
   delete: "削除"
@@ -80,6 +88,9 @@ common:
     mk-post-menu:
       pin: "ピン留め"
       pinned: "ピン留めしました"
+      select: "カテゴリを選択"
+      categorize: "決定"
+      categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。"
 
     mk-reaction-picker:
       choose-reaction: "リアクションを選択"
@@ -375,6 +386,7 @@ mobile:
       twitter-integration: "Twitter連携"
       signin-history: "ログイン履歴"
       api: "API"
+      link: "Misskeyリンク"
       settings: "設定"
       signout: "サインアウト"
 

package.json

@@ -64,6 +64,7 @@
     "@types/webpack": "3.0.10",
     "@types/webpack-stream": "3.2.7",
     "@types/websocket": "0.0.34",
+    "@types/msgpack-lite": "^0.1.5",
     "chai": "4.1.2",
     "chai-http": "3.0.0",
     "css-loader": "0.28.7",
@@ -120,10 +121,12 @@
     "is-root": "1.0.0",
     "is-url": "1.2.2",
     "js-yaml": "3.9.1",
+    "mecab-async": "^0.1.0",
     "mongodb": "2.2.31",
     "monk": "6.0.3",
     "morgan": "1.8.2",
     "ms": "2.0.0",
+    "msgpack-lite": "^0.1.26",
     "multer": "1.3.0",
     "nprogress": "0.2.0",
     "os-utils": "0.0.14",

src/api/endpoints.ts

@@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [
 		name: 'posts/trend',
 		withCredential: true
 	},
+	{
+		name: 'posts/categorize',
+		withCredential: true
+	},
 	{
 		name: 'posts/reactions',
 		withCredential: true

src/api/endpoints/posts/categorize.ts

@@ -0,0 +1,52 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import Post from '../../models/post';
+
+/**
+ * Categorize a post
+ *
+ * @param {any} params
+ * @param {any} user
+ * @return {Promise<any>}
+ */
+module.exports = (params, user) => new Promise(async (res, rej) => {
+	if (!user.is_pro) {
+		return rej('This endpoint is available only from a Pro account');
+	}
+
+	// Get 'post_id' parameter
+	const [postId, postIdErr] = $(params.post_id).id().$;
+	if (postIdErr) return rej('invalid post_id param');
+
+	// Get categorizee
+	const post = await Post.findOne({
+		_id: postId
+	});
+
+	if (post === null) {
+		return rej('post not found');
+	}
+
+	if (post.is_category_verified) {
+		return rej('This post already has the verified category');
+	}
+
+	// Get 'category' parameter
+	const [category, categoryErr] = $(params.category).string().or([
+		'music', 'game', 'anime', 'it', 'gadgets', 'photography'
+	]).$;
+	if (categoryErr) return rej('invalid category param');
+
+	// Set category
+	Post.update({ _id: post._id }, {
+		$set: {
+			category: category,
+			is_category_verified: true
+		}
+	});
+
+	// Send response
+	res();
+});

src/config.ts

@@ -68,6 +68,9 @@ type Source = {
 		hook_secret: string;
 		username: string;
 	};
+	categorizer?: {
+		mecab_command?: string;
+	};
 };
 
 /**

src/tools/ai/naive-bayes.js

@@ -0,0 +1,302 @@
+// Original source code: https://github.com/ttezel/bayes/blob/master/lib/naive_bayes.js (commit: 2c20d3066e4fc786400aaedcf3e42987e52abe3c)
+// CUSTOMIZED BY SYUILO
+
+/*
+		Expose our naive-bayes generator function
+*/
+module.exports = function (options) {
+	return new Naivebayes(options)
+}
+
+// keys we use to serialize a classifier's state
+var STATE_KEYS = module.exports.STATE_KEYS = [
+	'categories', 'docCount', 'totalDocuments', 'vocabulary', 'vocabularySize',
+	'wordCount', 'wordFrequencyCount', 'options'
+];
+
+/**
+ * Initializes a NaiveBayes instance from a JSON state representation.
+ * Use this with classifier.toJson().
+ *
+ * @param  {String} jsonStr   state representation obtained by classifier.toJson()
+ * @return {NaiveBayes}       Classifier
+ */
+module.exports.fromJson = function (jsonStr) {
+	var parsed;
+	try {
+		parsed = JSON.parse(jsonStr)
+	} catch (e) {
+		throw new Error('Naivebayes.fromJson expects a valid JSON string.')
+	}
+	// init a new classifier
+	var classifier = new Naivebayes(parsed.options)
+
+	// override the classifier's state
+	STATE_KEYS.forEach(function (k) {
+		if (!parsed[k]) {
+			throw new Error('Naivebayes.fromJson: JSON string is missing an expected property: `'+k+'`.')
+		}
+		classifier[k] = parsed[k]
+	})
+
+	return classifier
+}
+
+/**
+ * Given an input string, tokenize it into an array of word tokens.
+ * This is the default tokenization function used if user does not provide one in `options`.
+ *
+ * @param  {String} text
+ * @return {Array}
+ */
+var defaultTokenizer = function (text) {
+	//remove punctuation from text - remove anything that isn't a word char or a space
+	var rgxPunctuation = /[^(a-zA-ZA-Яa-я0-9_)+\s]/g
+
+	var sanitized = text.replace(rgxPunctuation, ' ')
+
+	return sanitized.split(/\s+/)
+}
+
+/**
+ * Naive-Bayes Classifier
+ *
+ * This is a naive-bayes classifier that uses Laplace Smoothing.
+ *
+ * Takes an (optional) options object containing:
+ *   - `tokenizer`  => custom tokenization function
+ *
+ */
+function Naivebayes (options) {
+	// set options object
+	this.options = {}
+	if (typeof options !== 'undefined') {
+		if (!options || typeof options !== 'object' || Array.isArray(options)) {
+			throw TypeError('NaiveBayes got invalid `options`: `' + options + '`. Pass in an object.')
+		}
+		this.options = options
+	}
+
+	this.tokenizer = this.options.tokenizer || defaultTokenizer
+
+	//initialize our vocabulary and its size
+	this.vocabulary = {}
+	this.vocabularySize = 0
+
+	//number of documents we have learned from
+	this.totalDocuments = 0
+
+	//document frequency table for each of our categories
+	//=> for each category, how often were documents mapped to it
+	this.docCount = {}
+
+	//for each category, how many words total were mapped to it
+	this.wordCount = {}
+
+	//word frequency table for each category
+	//=> for each category, how frequent was a given word mapped to it
+	this.wordFrequencyCount = {}
+
+	//hashmap of our category names
+	this.categories = {}
+}
+
+/**
+ * Initialize each of our data structure entries for this new category
+ *
+ * @param  {String} categoryName
+ */
+Naivebayes.prototype.initializeCategory = function (categoryName) {
+	if (!this.categories[categoryName]) {
+		this.docCount[categoryName] = 0
+		this.wordCount[categoryName] = 0
+		this.wordFrequencyCount[categoryName] = {}
+		this.categories[categoryName] = true
+	}
+	return this
+}
+
+/**
+ * train our naive-bayes classifier by telling it what `category`
+ * the `text` corresponds to.
+ *
+ * @param  {String} text
+ * @param  {String} class
+ */
+Naivebayes.prototype.learn = function (text, category) {
+	var self = this
+
+	//initialize category data structures if we've never seen this category
+	self.initializeCategory(category)
+
+	//update our count of how many documents mapped to this category
+	self.docCount[category]++
+
+	//update the total number of documents we have learned from
+	self.totalDocuments++
+
+	//normalize the text into a word array
+	var tokens = self.tokenizer(text)
+
+	//get a frequency count for each token in the text
+	var frequencyTable = self.frequencyTable(tokens)
+
+	/*
+			Update our vocabulary and our word frequency count for this category
+	*/
+
+	Object
+	.keys(frequencyTable)
+	.forEach(function (token) {
+		//add this word to our vocabulary if not already existing
+		if (!self.vocabulary[token]) {
+			self.vocabulary[token] = true
+			self.vocabularySize++
+		}
+
+		var frequencyInText = frequencyTable[token]
+
+		//update the frequency information for this word in this category
+		if (!self.wordFrequencyCount[category][token])
+			self.wordFrequencyCount[category][token] = frequencyInText
+		else
+			self.wordFrequencyCount[category][token] += frequencyInText
+
+		//update the count of all words we have seen mapped to this category
+		self.wordCount[category] += frequencyInText
+	})
+
+	return self
+}
+
+/**
+ * Determine what category `text` belongs to.
+ *
+ * @param  {String} text
+ * @return {String} category
+ */
+Naivebayes.prototype.categorize = function (text) {
+	var self = this
+		, maxProbability = -Infinity
+		, chosenCategory = null
+
+	var tokens = self.tokenizer(text)
+	var frequencyTable = self.frequencyTable(tokens)
+
+	//iterate thru our categories to find the one with max probability for this text
+	Object
+	.keys(self.categories)
+	.forEach(function (category) {
+
+		//start by calculating the overall probability of this category
+		//=>  out of all documents we've ever looked at, how many were
+		//    mapped to this category
+		var categoryProbability = self.docCount[category] / self.totalDocuments
+
+		//take the log to avoid underflow
+		var logProbability = Math.log(categoryProbability)
+
+		//now determine P( w | c ) for each word `w` in the text
+		Object
+		.keys(frequencyTable)
+		.forEach(function (token) {
+			var frequencyInText = frequencyTable[token]
+			var tokenProbability = self.tokenProbability(token, category)
+
+			// console.log('token: %s category: `%s` tokenProbability: %d', token, category, tokenProbability)
+
+			//determine the log of the P( w | c ) for this word
+			logProbability += frequencyInText * Math.log(tokenProbability)
+		})
+
+		if (logProbability > maxProbability) {
+			maxProbability = logProbability
+			chosenCategory = category
+		}
+	})
+
+	return chosenCategory
+}
+
+/**
+ * Calculate probability that a `token` belongs to a `category`
+ *
+ * @param  {String} token
+ * @param  {String} category
+ * @return {Number} probability
+ */
+Naivebayes.prototype.tokenProbability = function (token, category) {
+	//how many times this word has occurred in documents mapped to this category
+	var wordFrequencyCount = this.wordFrequencyCount[category][token] || 0
+
+	//what is the count of all words that have ever been mapped to this category
+	var wordCount = this.wordCount[category]
+
+	//use laplace Add-1 Smoothing equation
+	return ( wordFrequencyCount + 1 ) / ( wordCount + this.vocabularySize )
+}
+
+/**
+ * Build a frequency hashmap where
+ * - the keys are the entries in `tokens`
+ * - the values are the frequency of each entry in `tokens`
+ *
+ * @param  {Array} tokens  Normalized word array
+ * @return {Object}
+ */
+Naivebayes.prototype.frequencyTable = function (tokens) {
+	var frequencyTable = Object.create(null)
+
+	tokens.forEach(function (token) {
+		if (!frequencyTable[token])
+			frequencyTable[token] = 1
+		else
+			frequencyTable[token]++
+	})
+
+	return frequencyTable
+}
+
+/**
+ * Dump the classifier's state as a JSON string.
+ * @return {String} Representation of the classifier.
+ */
+Naivebayes.prototype.toJson = function () {
+	var state = {}
+	var self = this
+	STATE_KEYS.forEach(function (k) {
+		state[k] = self[k]
+	})
+
+	var jsonStr = JSON.stringify(state)
+
+	return jsonStr
+}
+
+// (original method)
+Naivebayes.prototype.export = function () {
+	var state = {}
+	var self = this
+	STATE_KEYS.forEach(function (k) {
+		state[k] = self[k]
+	})
+
+	return state
+}
+
+module.exports.import = function (data) {
+	var parsed = data
+
+	// init a new classifier
+	var classifier = new Naivebayes()
+
+	// override the classifier's state
+	STATE_KEYS.forEach(function (k) {
+		if (!parsed[k]) {
+			throw new Error('Naivebayes.import: data is missing an expected property: `'+k+'`.')
+		}
+		classifier[k] = parsed[k]
+	})
+
+	return classifier
+}

src/tools/ai/predict-all-post-category.ts

@@ -0,0 +1,57 @@
+const bayes = require('./naive-bayes.js');
+const MeCab = require('mecab-async');
+
+import Post from '../../api/models/post';
+import config from '../../conf';
+
+const classifier = bayes({
+	tokenizer: this.tokenizer
+});
+
+const mecab = new MeCab();
+if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
+
+// 訓練データ取得
+Post.find({
+	is_category_verified: true
+}, {
+	fields: {
+		_id: false,
+		text: true,
+		category: true
+	}
+}).then(verifiedPosts => {
+	// 学習
+	verifiedPosts.forEach(post => {
+		classifier.learn(post.text, post.category);
+	});
+
+	// 全ての(人間によって証明されていない)投稿を取得
+	Post.find({
+		text: {
+			$exists: true
+		},
+		is_category_verified: {
+			$ne: true
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		fields: {
+			_id: true,
+			text: true
+		}
+	}).then(posts => {
+		posts.forEach(post => {
+			console.log(`predicting... ${post._id}`);
+			const category = classifier.categorize(post.text);
+
+			Post.update({ _id: post._id }, {
+				$set: {
+					category: category
+				}
+			});
+		});
+	});
+});

src/tools/ai/predict-user-interst.ts

@@ -0,0 +1,45 @@
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+
+export async function predictOne(id) {
+	console.log(`predict interest of ${id} ...`);
+
+	// TODO: repostなども含める
+	const recentPosts = await Post.find({
+		user_id: id,
+		category: {
+			$exists: true
+		}
+	}, {
+		sort: {
+			_id: -1
+		},
+		limit: 1000,
+		fields: {
+			_id: false,
+			category: true
+		}
+	});
+
+	const categories = {};
+
+	recentPosts.forEach(post => {
+		if (categories[post.category]) {
+			categories[post.category]++;
+		} else {
+			categories[post.category] = 1;
+		}
+	});
+}
+
+export async function predictAll() {
+	const allUsers = await User.find({}, {
+		fields: {
+			_id: true
+		}
+	});
+
+	allUsers.forEach(user => {
+		predictOne(user._id);
+	});
+}

src/web/app/common/tags/post-menu.tag

@@ -2,6 +2,18 @@
 	<div class="backdrop" ref="backdrop" onclick={ close }></div>
 	<div class="popover { compact: opts.compact }" ref="popover">
 		<button if={ post.user_id === I.id } onclick={ pin }>%i18n:common.tags.mk-post-menu.pin%</button>
+		<div if={ I.is_pro && !post.is_category_verified }>
+			<select ref="categorySelect">
+				<option value="">%i18n:common.tags.mk-post-menu.select%</option>
+				<option value="music">%i18n:common.post_categories.music%</option>
+				<option value="game">%i18n:common.post_categories.game%</option>
+				<option value="anime">%i18n:common.post_categories.anime%</option>
+				<option value="it">%i18n:common.post_categories.it%</option>
+				<option value="gadgets">%i18n:common.post_categories.gadgets%</option>
+				<option value="photography">%i18n:common.post_categories.photography%</option>
+			</select>
+			<button onclick={ categorize }>%i18n:common.tags.mk-post-menu.categorize%</button>
+		</div>
 	</div>
 	<style>
 		$border-color = rgba(27, 31, 35, 0.15)
@@ -111,6 +123,17 @@
 			});
 		};
 
+		this.categorize = () => {
+			const category = this.refs.categorySelect.options[this.refs.categorySelect.selectedIndex].value;
+			this.api('posts/categorize', {
+				post_id: this.post.id,
+				category: category
+			}).then(() => {
+				if (this.opts.cb) this.opts.cb('categorized', '%i18n:common.tags.mk-post-menu.categorized%');
+				this.unmount();
+			});
+		};
+
 		this.close = () => {
 			this.refs.backdrop.style.pointerEvents = 'none';
 			anime({