From 0db54386cdef3f444f1afb4f3b8bfcaeab7ac68d Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 07:46:39 +0900 Subject: [PATCH] Resolve #3119 --- CHANGELOG.md | 13 ++++ package.json | 3 +- .../app/common/scripts/gen-search-query.ts | 31 ++++++++ src/client/app/common/scripts/search.ts | 2 +- .../app/common/views/components/user-list.vue | 8 +-- .../app/common/views/deck/deck.notes.vue | 8 +-- .../common/views/deck/deck.search-column.vue | 5 +- .../app/desktop/views/components/notes.vue | 6 +- src/client/app/desktop/views/home/search.vue | 5 +- .../app/mobile/views/components/notes.vue | 8 +-- src/client/app/mobile/views/pages/search.vue | 5 +- src/db/elasticsearch.ts | 72 +++++++------------ src/server/api/endpoints/notes/search.ts | 64 ++++++++++++----- src/services/note/create.ts | 7 +- 14 files changed, 147 insertions(+), 90 deletions(-) create mode 100644 src/client/app/common/scripts/gen-search-query.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e797664c1..f50bcef57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,19 @@ mongodb: 8. master ブランチに戻す 9. enjoy +11.4.0 (2019/04/25) +------------------- +### Improvements +* 検索でローカルの投稿のみに絞れるように +* 検索で特定のインスタンスの投稿のみに絞れるように +* 検索で特定のユーザーの投稿のみに絞れるように + +### Fixes +* 投稿が増殖する問題を修正 +* ストリームで過去の投稿が流れてくる問題を修正 +* モバイル版のユーザーページで遷移してもユーザー名が変わらない問題を修正 +* お知らせを切り替えても内容が変わらない問題を修正 + 11.3.1 (2019/04/24) ------------------- ### Fixes diff --git a/package.json b/package.json index 00cbdc73c..56a5ac0dc 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "format": "gulp format" }, "dependencies": { + "@elastic/elasticsearch": "7.0.0-rc.2", "@fortawesome/fontawesome-svg-core": "1.2.15", "@fortawesome/free-brands-svg-icons": "5.7.2", "@fortawesome/free-regular-svg-icons": "5.7.2", @@ -35,7 +36,6 @@ "@types/dateformat": "3.0.0", "@types/deep-equal": "1.0.1", "@types/double-ended-queue": "2.1.0", - "@types/elasticsearch": "5.0.32", "@types/file-type": "10.9.1", "@types/gulp": "4.0.6", "@types/gulp-mocha": "0.0.32", @@ -113,7 +113,6 @@ "deep-equal": "1.0.1", "diskusage": "1.1.0", "double-ended-queue": "2.1.0-0", - "elasticsearch": "15.4.1", "emojilib": "2.4.0", "eslint": "5.16.0", "eslint-plugin-vue": "5.2.2", diff --git a/src/client/app/common/scripts/gen-search-query.ts b/src/client/app/common/scripts/gen-search-query.ts new file mode 100644 index 000000000..fc26cb7f7 --- /dev/null +++ b/src/client/app/common/scripts/gen-search-query.ts @@ -0,0 +1,31 @@ +import parseAcct from '../../../../misc/acct/parse'; +import { host as localHost } from '../../config'; + +export async function genSearchQuery(v: any, q: string) { + let host: string; + let userId: string; + if (q.split(' ').some(x => x.startsWith('@'))) { + for (const at of q.split(' ').filter(x => x.startsWith('@')).map(x => x.substr(1))) { + if (at.includes('.')) { + if (at === localHost || at === '.') { + host = null; + } else { + host = at; + } + } else { + const user = await v.$root.api('users/show', parseAcct(at)).catch(x => null); + if (user) { + userId = user.id; + } else { + // todo: show error + } + } + } + + } + return { + query: q.split(' ').filter(x => !x.startsWith('/') && !x.startsWith('@')).join(' '), + host: host, + userId: userId + }; +} diff --git a/src/client/app/common/scripts/search.ts b/src/client/app/common/scripts/search.ts index c44581817..2897ed631 100644 --- a/src/client/app/common/scripts/search.ts +++ b/src/client/app/common/scripts/search.ts @@ -3,7 +3,7 @@ import { faHistory } from '@fortawesome/free-solid-svg-icons'; export async function search(v: any, q: string) { q = q.trim(); - if (q.startsWith('@')) { + if (q.startsWith('@') && !q.includes(' ')) { v.$router.push(`/${q}`); return; } diff --git a/src/client/app/common/views/components/user-list.vue b/src/client/app/common/views/components/user-list.vue index b8bcc35d8..53577bad0 100644 --- a/src/client/app/common/views/components/user-list.vue +++ b/src/client/app/common/views/components/user-list.vue @@ -60,9 +60,9 @@ export default Vue.extend({ }, methods: { - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.us = x; } else { @@ -76,9 +76,9 @@ export default Vue.extend({ }); }, - fetchMoreUsers() { + async fetchMoreUsers() { this.fetchingMoreUsers = true; - this.makePromise(this.cursor).then(x => { + await (this.makePromise(this.cursor)).then(x => { this.us = this.us.concat(x.users); this.cursor = x.cursor; this.fetchingMoreUsers = false; diff --git a/src/client/app/common/views/deck/deck.notes.vue b/src/client/app/common/views/deck/deck.notes.vue index bc67f4911..680b44bc8 100644 --- a/src/client/app/common/views/deck/deck.notes.vue +++ b/src/client/app/common/views/deck/deck.notes.vue @@ -110,11 +110,11 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.queue = []; this.notes = []; this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -129,10 +129,10 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching) return; this.moreFetching = true; - this.makePromise(this.notes[this.notes.length - 1].id).then(x => { + await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => { this.notes = this.notes.concat(x.notes); this.more = x.more; this.moreFetching = false; diff --git a/src/client/app/common/views/deck/deck.search-column.vue b/src/client/app/common/views/deck/deck.search-column.vue index ab19bdaab..17ee2ef45 100644 --- a/src/client/app/common/views/deck/deck.search-column.vue +++ b/src/client/app/common/views/deck/deck.search-column.vue @@ -14,6 +14,7 @@ import Vue from 'vue'; import XColumn from './deck.column.vue'; import XNotes from './deck.notes.vue'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -25,10 +26,10 @@ export default Vue.extend({ data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, offset: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/client/app/desktop/views/components/notes.vue b/src/client/app/desktop/views/components/notes.vue index 9044ad347..87fdc749d 100644 --- a/src/client/app/desktop/views/components/notes.vue +++ b/src/client/app/desktop/views/components/notes.vue @@ -105,9 +105,9 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -122,7 +122,7 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching || this.notes.length === 0) return; this.moreFetching = true; this.makePromise(this.notes[this.notes.length - 1].id).then(x => { diff --git a/src/client/app/desktop/views/home/search.vue b/src/client/app/desktop/views/home/search.vue index 84153d18c..50c645615 100644 --- a/src/client/app/desktop/views/home/search.vue +++ b/src/client/app/desktop/views/home/search.vue @@ -14,6 +14,7 @@ import Vue from 'vue'; import i18n from '../../../i18n'; import Progress from '../../../common/scripts/loading'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -21,10 +22,10 @@ export default Vue.extend({ i18n: i18n('desktop/views/pages/search.vue'), data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, offset: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/client/app/mobile/views/components/notes.vue b/src/client/app/mobile/views/components/notes.vue index 2e4230071..5ad80c286 100644 --- a/src/client/app/mobile/views/components/notes.vue +++ b/src/client/app/mobile/views/components/notes.vue @@ -106,9 +106,9 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -123,10 +123,10 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching || this.notes.length === 0) return; this.moreFetching = true; - this.makePromise(this.notes[this.notes.length - 1].id).then(x => { + await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => { this.notes = this.notes.concat(x.notes); this.more = x.more; this.moreFetching = false; diff --git a/src/client/app/mobile/views/pages/search.vue b/src/client/app/mobile/views/pages/search.vue index 0225dd9e9..45f383790 100644 --- a/src/client/app/mobile/views/pages/search.vue +++ b/src/client/app/mobile/views/pages/search.vue @@ -12,6 +12,7 @@ import Vue from 'vue'; import i18n from '../../../i18n'; import Progress from '../../../common/scripts/loading'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -19,10 +20,10 @@ export default Vue.extend({ i18n: i18n('mobile/views/pages/search.vue'), data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, untilId: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/db/elasticsearch.ts b/src/db/elasticsearch.ts index d54b01763..02c9e88d9 100644 --- a/src/db/elasticsearch.ts +++ b/src/db/elasticsearch.ts @@ -1,41 +1,30 @@ -import * as elasticsearch from 'elasticsearch'; +import * as elasticsearch from '@elastic/elasticsearch'; import config from '../config'; -import Logger from '../services/logger'; - -const esLogger = new Logger('es'); const index = { settings: { analysis: { - normalizer: { - lowercase_normalizer: { - type: 'custom', - filter: ['lowercase'] - } - }, analyzer: { - bigram: { - tokenizer: 'bigram_tokenizer' - } - }, - tokenizer: { - bigram_tokenizer: { - type: 'nGram', - min_gram: 2, - max_gram: 2 + ngram: { + tokenizer: 'ngram' } } } }, mappings: { - note: { - properties: { - text: { - type: 'text', - index: true, - analyzer: 'bigram', - normalizer: 'lowercase_normalizer' - } + properties: { + text: { + type: 'text', + index: true, + analyzer: 'ngram', + }, + userId: { + type: 'keyword', + index: true, + }, + userHost: { + type: 'keyword', + index: true, } } } @@ -43,31 +32,20 @@ const index = { // Init ElasticSearch connection const client = config.elasticsearch ? new elasticsearch.Client({ - host: `${config.elasticsearch.host}:${config.elasticsearch.port}` + node: `http://${config.elasticsearch.host}:${config.elasticsearch.port}`, + pingTimeout: 30000 }) : null; if (client) { - // Send a HEAD request - client.ping({ - // Ping usually has a 3000ms timeout - requestTimeout: 30000 - }, error => { - if (error) { - esLogger.error('elasticsearch is down!'); - } else { - esLogger.succ('elasticsearch is available!'); - } - }); - client.indices.exists({ - index: 'misskey' + index: 'misskey_note' }).then(exist => { - if (exist) return; - - client.indices.create({ - index: 'misskey', - body: index - }); + if (!exist.body) { + client.indices.create({ + index: 'misskey_note', + body: index + }); + } }); } diff --git a/src/server/api/endpoints/notes/search.ts b/src/server/api/endpoints/notes/search.ts index daf992b63..65ce20074 100644 --- a/src/server/api/endpoints/notes/search.ts +++ b/src/server/api/endpoints/notes/search.ts @@ -5,6 +5,7 @@ import { ApiError } from '../../error'; import { Notes } from '../../../../models'; import { In } from 'typeorm'; import { types, bool } from '../../../../misc/schema'; +import { ID } from '../../../../misc/cafy-id'; export const meta = { desc: { @@ -29,7 +30,17 @@ export const meta = { offset: { validator: $.optional.num.min(0), default: 0 - } + }, + + host: { + validator: $.optional.nullable.str, + default: undefined + }, + + userId: { + validator: $.optional.nullable.type(ID), + default: null + }, }, res: { @@ -54,30 +65,51 @@ export const meta = { export default define(meta, async (ps, me) => { if (es == null) throw new ApiError(meta.errors.searchingNotAvailable); - const response = await es.search({ - index: 'misskey', - type: 'note', + const userQuery = ps.userId != null ? [{ + term: { + userId: ps.userId + } + }] : []; + + const hostQuery = ps.userId == null ? + ps.host === null ? [{ + bool: { + must_not: { + exists: { + field: 'userHost' + } + } + } + }] : ps.host !== undefined ? [{ + term: { + userHost: ps.host + } + }] : [] + : []; + + const result = await es.search({ + index: 'misskey_note', body: { size: ps.limit!, from: ps.offset, query: { - simple_query_string: { - fields: ['text'], - query: ps.query, - default_operator: 'and' + bool: { + must: [{ + simple_query_string: { + fields: ['text'], + query: ps.query.toLowerCase(), + default_operator: 'and' + }, + }, ...hostQuery, ...userQuery] } }, - sort: [ - { _doc: 'desc' } - ] + sort: [{ + _doc: 'desc' + }] } }); - if (response.hits.total === 0) { - return []; - } - - const hits = response.hits.hits.map((hit: any) => hit.id); + const hits = result.body.hits.hits.map((hit: any) => hit._id); if (hits.length === 0) return []; diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 53e77b4ef..dd47632ca 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -435,11 +435,12 @@ function index(note: Note) { if (note.text == null || config.elasticsearch == null) return; es!.index({ - index: 'misskey', - type: 'note', + index: 'misskey_note', id: note.id.toString(), body: { - text: note.text + text: note.text.toLowerCase(), + userId: note.userId, + userHost: note.userHost } }); }