From dc3c80e3cef48bbbb1de2c5d8c6fb462b25de6e0 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 11 Jun 2018 06:48:25 +0900 Subject: [PATCH 1/4] wip --- src/daemons/hashtags-stats-child.ts | 60 +++++++++++++++++++++++++ src/daemons/hashtags-stats.ts | 20 +++++++++ src/{ => daemons}/notes-stats-child.ts | 10 +++-- src/{ => daemons}/notes-stats.ts | 0 src/{ => daemons}/server-stats.ts | 10 ++++- src/index.ts | 6 ++- src/server/api/stream/hashtags-stats.ts | 35 +++++++++++++++ src/server/api/streaming.ts | 6 +++ 8 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 src/daemons/hashtags-stats-child.ts create mode 100644 src/daemons/hashtags-stats.ts rename src/{ => daemons}/notes-stats-child.ts (75%) rename src/{ => daemons}/notes-stats.ts (100%) rename src/{ => daemons}/server-stats.ts (88%) create mode 100644 src/server/api/stream/hashtags-stats.ts diff --git a/src/daemons/hashtags-stats-child.ts b/src/daemons/hashtags-stats-child.ts new file mode 100644 index 000000000..3f7f4d6e9 --- /dev/null +++ b/src/daemons/hashtags-stats-child.ts @@ -0,0 +1,60 @@ +import Note from '../models/note'; + +// 10分 +const interval = 1000 * 60 * 10; + +async function tick() { + const res = await Note.aggregate([{ + $match: { + createdAt: { + $gt: new Date(Date.now() - interval) + }, + tags: { + $exists: true, + $ne: [] + } + } + }, { + $unwind: '$tags' + }, { + $group: { + _id: '$tags', + count: { + $sum: 1 + } + } + }, { + $group: { + _id: null, + tags: { + $push: { + tag: '$_id', + count: '$count' + } + } + } + }, { + $project: { + _id: false, + tags: true + } + }]) as { + tags: Array<{ + tag: string; + count: number; + }> + }; + + const stats = res.tags + .sort((a, b) => a.count - b.count) + .map(tag => [tag.tag, tag.count]) + .slice(0, 10); + + console.log(stats); + + process.send(stats); +} + +tick(); + +setInterval(tick, interval); diff --git a/src/daemons/hashtags-stats.ts b/src/daemons/hashtags-stats.ts new file mode 100644 index 000000000..5ed028ac3 --- /dev/null +++ b/src/daemons/hashtags-stats.ts @@ -0,0 +1,20 @@ +import * as childProcess from 'child_process'; +import Xev from 'xev'; + +const ev = new Xev(); + +export default function() { + const log = []; + + const p = childProcess.fork(__dirname + '/hashtags-stats-child.js'); + + p.on('message', stats => { + ev.emit('hashtagsStats', stats); + log.push(stats); + if (log.length > 30) log.shift(); + }); + + ev.on('requestHashTagsStatsLog', id => { + ev.emit('hashtagsStatsLog:' + id, log); + }); +} diff --git a/src/notes-stats-child.ts b/src/daemons/notes-stats-child.ts similarity index 75% rename from src/notes-stats-child.ts rename to src/daemons/notes-stats-child.ts index 5f85a2a3c..7f54a36bf 100644 --- a/src/notes-stats-child.ts +++ b/src/daemons/notes-stats-child.ts @@ -1,8 +1,8 @@ -import Note from './models/note'; +import Note from '../models/note'; const interval = 5000; -setInterval(async () => { +async function tick() { const [all, local] = await Promise.all([Note.count({ createdAt: { $gte: new Date(Date.now() - interval) @@ -19,4 +19,8 @@ setInterval(async () => { }; process.send(stats); -}, interval); +} + +tick(); + +setInterval(tick, interval); diff --git a/src/notes-stats.ts b/src/daemons/notes-stats.ts similarity index 100% rename from src/notes-stats.ts rename to src/daemons/notes-stats.ts diff --git a/src/server-stats.ts b/src/daemons/server-stats.ts similarity index 88% rename from src/server-stats.ts rename to src/daemons/server-stats.ts index 7b0d4a857..140340250 100644 --- a/src/server-stats.ts +++ b/src/daemons/server-stats.ts @@ -5,6 +5,8 @@ import Xev from 'xev'; const ev = new Xev(); +const interval = 1000; + /** * Report server stats regularly */ @@ -15,7 +17,7 @@ export default function() { ev.emit('serverStatsLog:' + id, log); }); - setInterval(() => { + async function tick() { osUtils.cpuUsage(cpuUsage => { const disk = diskusage.checkSync(os.platform() == 'win32' ? 'c:' : '/'); const stats = { @@ -32,5 +34,9 @@ export default function() { log.push(stats); if (log.length > 50) log.shift(); }); - }, 1000); + } + + tick(); + + setInterval(tick, interval); } diff --git a/src/index.ts b/src/index.ts index 4a98b7564..27c5dd027 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,8 +17,9 @@ import ProgressBar from './utils/cli/progressbar'; import EnvironmentInfo from './utils/environmentInfo'; import MachineInfo from './utils/machineInfo'; import DependencyInfo from './utils/dependencyInfo'; -import serverStats from './server-stats'; -import notesStats from './notes-stats'; +import serverStats from './daemons/server-stats'; +import notesStats from './daemons/notes-stats'; +import hashtagsStats from './daemons/hashtags-stats'; import loadConfig from './config/load'; import { Config } from './config/types'; @@ -52,6 +53,7 @@ function main() { ev.mount(); serverStats(); notesStats(); + hashtagsStats(); } else { workerMain(opt); } diff --git a/src/server/api/stream/hashtags-stats.ts b/src/server/api/stream/hashtags-stats.ts new file mode 100644 index 000000000..47183467f --- /dev/null +++ b/src/server/api/stream/hashtags-stats.ts @@ -0,0 +1,35 @@ +import * as websocket from 'websocket'; +import Xev from 'xev'; + +const ev = new Xev(); + +export default function(request: websocket.request, connection: websocket.connection): void { + const onStats = stats => { + connection.send(JSON.stringify({ + type: 'stats', + body: stats + })); + }; + + connection.on('message', async data => { + const msg = JSON.parse(data.utf8Data); + + switch (msg.type) { + case 'requestLog': + ev.once('hashtagsStatsLog:' + msg.id, statsLog => { + connection.send(JSON.stringify({ + type: 'statsLog', + body: statsLog + })); + }); + ev.emit('requestHashtagsStatsLog', msg.id); + break; + } + }); + + ev.addListener('hashtagsStats', onStats); + + connection.on('close', () => { + ev.removeListener('hashtagsStats', onStats); + }); +} diff --git a/src/server/api/streaming.ts b/src/server/api/streaming.ts index 2d4cfc108..e4156096e 100644 --- a/src/server/api/streaming.ts +++ b/src/server/api/streaming.ts @@ -14,6 +14,7 @@ import othelloGameStream from './stream/othello-game'; import othelloStream from './stream/othello'; import serverStatsStream from './stream/server-stats'; import notesStatsStream from './stream/notes-stats'; +import hashtagsStatsStream from './stream/hashtags-stats'; import requestsStream from './stream/requests'; import { ParsedUrlQuery } from 'querystring'; import authenticate from './authenticate'; @@ -39,6 +40,11 @@ module.exports = (server: http.Server) => { return; } + if (request.resourceURL.pathname === '/hashtags-stats') { + hashtagsStatsStream(request, connection); + return; + } + if (request.resourceURL.pathname === '/requests') { requestsStream(request, connection); return; From 2ec25a7729e8d2fa6734bdf25ffb5a1b35ca2d5b Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 11 Jun 2018 09:11:32 +0900 Subject: [PATCH 2/4] wip --- .../app/common/views/widgets/hashtags.vue | 97 +++++++++++++++++++ src/client/app/common/views/widgets/index.ts | 2 + .../app/desktop/views/components/home.vue | 1 + .../views/pages/deck/deck.widgets-column.vue | 1 + src/client/app/mobile/views/pages/widgets.vue | 1 + src/index.ts | 2 - src/server/api/endpoints.ts | 5 + src/server/api/endpoints/hashtags/trend.ts | 78 +++++++++++++++ src/server/api/stream/hashtags-stats.ts | 35 ------- src/server/api/streaming.ts | 6 -- 10 files changed, 185 insertions(+), 43 deletions(-) create mode 100644 src/client/app/common/views/widgets/hashtags.vue create mode 100644 src/server/api/endpoints/hashtags/trend.ts delete mode 100644 src/server/api/stream/hashtags-stats.ts diff --git a/src/client/app/common/views/widgets/hashtags.vue b/src/client/app/common/views/widgets/hashtags.vue new file mode 100644 index 000000000..0ac62af70 --- /dev/null +++ b/src/client/app/common/views/widgets/hashtags.vue @@ -0,0 +1,97 @@ + + + + + diff --git a/src/client/app/common/views/widgets/index.ts b/src/client/app/common/views/widgets/index.ts index 0190393ba..7d548ef35 100644 --- a/src/client/app/common/views/widgets/index.ts +++ b/src/client/app/common/views/widgets/index.ts @@ -13,6 +13,7 @@ import wSlideshow from './slideshow.vue'; import wTips from './tips.vue'; import wDonation from './donation.vue'; import wNav from './nav.vue'; +import wHashtags from './hashtags.vue'; Vue.component('mkw-analog-clock', wAnalogClock); Vue.component('mkw-nav', wNav); @@ -27,3 +28,4 @@ Vue.component('mkw-posts-monitor', wPostsMonitor); Vue.component('mkw-memo', wMemo); Vue.component('mkw-rss', wRss); Vue.component('mkw-version', wVersion); +Vue.component('mkw-hashtags', wHashtags); diff --git a/src/client/app/desktop/views/components/home.vue b/src/client/app/desktop/views/components/home.vue index cac1fd935..8774aada6 100644 --- a/src/client/app/desktop/views/components/home.vue +++ b/src/client/app/desktop/views/components/home.vue @@ -23,6 +23,7 @@ + diff --git a/src/client/app/desktop/views/pages/deck/deck.widgets-column.vue b/src/client/app/desktop/views/pages/deck/deck.widgets-column.vue index 2a3a2472d..a41bf8c3c 100644 --- a/src/client/app/desktop/views/pages/deck/deck.widgets-column.vue +++ b/src/client/app/desktop/views/pages/deck/deck.widgets-column.vue @@ -23,6 +23,7 @@ + diff --git a/src/client/app/mobile/views/pages/widgets.vue b/src/client/app/mobile/views/pages/widgets.vue index ea8580b4d..294c80e7a 100644 --- a/src/client/app/mobile/views/pages/widgets.vue +++ b/src/client/app/mobile/views/pages/widgets.vue @@ -15,6 +15,7 @@ + diff --git a/src/index.ts b/src/index.ts index 27c5dd027..35cf5a243 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,6 @@ import MachineInfo from './utils/machineInfo'; import DependencyInfo from './utils/dependencyInfo'; import serverStats from './daemons/server-stats'; import notesStats from './daemons/notes-stats'; -import hashtagsStats from './daemons/hashtags-stats'; import loadConfig from './config/load'; import { Config } from './config/types'; @@ -53,7 +52,6 @@ function main() { ev.mount(); serverStats(); notesStats(); - hashtagsStats(); } else { workerMain(opt); } diff --git a/src/server/api/endpoints.ts b/src/server/api/endpoints.ts index 91e5298e7..5f0a020d6 100644 --- a/src/server/api/endpoints.ts +++ b/src/server/api/endpoints.ts @@ -628,6 +628,11 @@ const endpoints: Endpoint[] = [ withCredential: true }, + { + name: 'hashtags/trend', + withCredential: true + }, + { name: 'messaging/history', withCredential: true, diff --git a/src/server/api/endpoints/hashtags/trend.ts b/src/server/api/endpoints/hashtags/trend.ts new file mode 100644 index 000000000..3b95e1fa4 --- /dev/null +++ b/src/server/api/endpoints/hashtags/trend.ts @@ -0,0 +1,78 @@ +import Note from '../../../../models/note'; + +/** + * Get trends of hashtags + */ +module.exports = (params, user) => new Promise(async (res, rej) => { + // 10分 + const interval = 1000 * 60 * 10; + + const data = await Note.aggregate([{ + $match: { + createdAt: { + $gt: new Date(Date.now() - interval) + }, + tags: { + $exists: true, + $ne: [] + } + } + }, { + $unwind: '$tags' + }, { + $group: { + _id: '$tags', + count: { + $sum: 1 + } + } + }, { + $group: { + _id: null, + tags: { + $push: { + tag: '$_id', + count: '$count' + } + } + } + }, { + $project: { + _id: false, + tags: true + } + }]) as Array<{ + tags: Array<{ + tag: string; + count: number; + }> + }>; + + const hots = data[0].tags + .sort((a, b) => a.count - b.count) + .map(tag => tag.tag) + .slice(0, 10); + + const countPromises: Array> = []; + + for (let i = 0; i < 10; i++) { + countPromises.push(Promise.all(hots.map(tag => Note.count({ + tags: tag, + createdAt: { + $lt: new Date(Date.now() - (interval * i)), + $gt: new Date(Date.now() - (interval * (i + 1))) + } + })))); + } + + const countsLog = await Promise.all(countPromises); + + const stats = hots.map((tag, i) => ({ + tag, + chart: countsLog.map(counts => counts[i]) + })); + + console.log(stats); + + res(stats); +}); diff --git a/src/server/api/stream/hashtags-stats.ts b/src/server/api/stream/hashtags-stats.ts deleted file mode 100644 index 47183467f..000000000 --- a/src/server/api/stream/hashtags-stats.ts +++ /dev/null @@ -1,35 +0,0 @@ -import * as websocket from 'websocket'; -import Xev from 'xev'; - -const ev = new Xev(); - -export default function(request: websocket.request, connection: websocket.connection): void { - const onStats = stats => { - connection.send(JSON.stringify({ - type: 'stats', - body: stats - })); - }; - - connection.on('message', async data => { - const msg = JSON.parse(data.utf8Data); - - switch (msg.type) { - case 'requestLog': - ev.once('hashtagsStatsLog:' + msg.id, statsLog => { - connection.send(JSON.stringify({ - type: 'statsLog', - body: statsLog - })); - }); - ev.emit('requestHashtagsStatsLog', msg.id); - break; - } - }); - - ev.addListener('hashtagsStats', onStats); - - connection.on('close', () => { - ev.removeListener('hashtagsStats', onStats); - }); -} diff --git a/src/server/api/streaming.ts b/src/server/api/streaming.ts index e4156096e..2d4cfc108 100644 --- a/src/server/api/streaming.ts +++ b/src/server/api/streaming.ts @@ -14,7 +14,6 @@ import othelloGameStream from './stream/othello-game'; import othelloStream from './stream/othello'; import serverStatsStream from './stream/server-stats'; import notesStatsStream from './stream/notes-stats'; -import hashtagsStatsStream from './stream/hashtags-stats'; import requestsStream from './stream/requests'; import { ParsedUrlQuery } from 'querystring'; import authenticate from './authenticate'; @@ -40,11 +39,6 @@ module.exports = (server: http.Server) => { return; } - if (request.resourceURL.pathname === '/hashtags-stats') { - hashtagsStatsStream(request, connection); - return; - } - if (request.resourceURL.pathname === '/requests') { requestsStream(request, connection); return; From 7d7193cb63ac8811152707a6896e5cf7ae05258c Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 11 Jun 2018 11:24:29 +0900 Subject: [PATCH 3/4] :v: --- locales/ja.yml | 3 + .../common/views/widgets/hashtags.chart.vue | 78 +++++++++++++++++++ .../app/common/views/widgets/hashtags.vue | 57 +++++++------- src/server/api/endpoints/hashtags/trend.ts | 12 ++- 4 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 src/client/app/common/views/widgets/hashtags.chart.vue diff --git a/locales/ja.yml b/locales/ja.yml index 64ee00dba..da751c9e3 100644 --- a/locales/ja.yml +++ b/locales/ja.yml @@ -254,6 +254,9 @@ common/views/widgets/posts-monitor.vue: title: "投稿チャート" toggle: "表示を切り替え" +common/views/widgets/hashtags.vue: + title: "ハッシュタグ" + common/views/widgets/server.vue: title: "サーバー情報" toggle: "表示を切り替え" diff --git a/src/client/app/common/views/widgets/hashtags.chart.vue b/src/client/app/common/views/widgets/hashtags.chart.vue new file mode 100644 index 000000000..19b56ef28 --- /dev/null +++ b/src/client/app/common/views/widgets/hashtags.chart.vue @@ -0,0 +1,78 @@ + + + diff --git a/src/client/app/common/views/widgets/hashtags.vue b/src/client/app/common/views/widgets/hashtags.vue index 0ac62af70..c4647ee0f 100644 --- a/src/client/app/common/views/widgets/hashtags.vue +++ b/src/client/app/common/views/widgets/hashtags.vue @@ -6,7 +6,12 @@

%fa:spinner .pulse .fw%%i18n:common.loading%

- {{ stat.tag }} +
+
+ #{{ stat.tag }} +
+ +
@@ -15,12 +20,17 @@