populateEmojisのリファクタと絵文字情報のキャッシュ (#7378)

* revert

* Refactor populateEmojis, Cache emojis

* ん

* fix typo

* コメント
This commit is contained in:
MeiMei 2021-03-22 00:44:38 +09:00 committed by GitHub
parent 2f2a8e537d
commit d1efe1d208
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 89 additions and 216 deletions

View file

@ -14,13 +14,30 @@ export class Cache<T> {
}); });
} }
public get(key: string | null): T | null { public get(key: string | null): T | undefined {
const cached = this.cache.get(key); const cached = this.cache.get(key);
if (cached == null) return null; if (cached == null) return undefined;
if ((Date.now() - cached.date) > this.lifetime) { if ((Date.now() - cached.date) > this.lifetime) {
this.cache.delete(key); this.cache.delete(key);
return null; return undefined;
} }
return cached.value; return cached.value;
} }
public delete(key: string | null) {
this.cache.delete(key);
}
public async fetch(key: string | null, fetcher: () => Promise<T>): Promise<T> {
const cachedValue = this.get(key);
if (cachedValue !== undefined) {
// Cache HIT
return cachedValue;
}
// Cache MISS
const value = await fetcher();
this.set(key, value);
return value;
}
} }

View file

@ -0,0 +1,58 @@
import { Emojis } from '../models';
import { Emoji } from '../models/entities/emoji';
import { Cache } from './cache';
import { isSelfHost, toPunyNullable } from './convert-host';
const cache = new Cache<Emoji | null>(1000 * 60 * 60);
/**
*
*/
type PopulatedEmoji = {
name: string;
url: string;
};
/**
*
* @param emojiName (:, @. (decodeReactionで可能))
* @param noteUserHost
* @returns , nullは未マッチを意味する
*/
export async function populateEmoji(emojiName: string, noteUserHost: string | null): Promise<PopulatedEmoji | null> {
const match = emojiName.match(/^(\w+)(?:@([\w.-]+))?$/);
if (!match) return null;
const name = match[1];
// クエリに使うホスト
let host = match[2] === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ)
: match[2] === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない)
: isSelfHost(match[2]) ? null // 自ホスト指定
: (match[2] || noteUserHost); // 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない)
host = toPunyNullable(host);
const queryOrNull = async () => (await Emojis.findOne({
name,
host
})) || null;
const emoji = await cache.fetch(`${name} ${host}`, queryOrNull);
if (emoji == null) return null;
return {
name: emojiName,
url: emoji.url,
};
}
/**
* (, )
*/
export async function populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<PopulatedEmoji[]> {
const emojis = await Promise.all(emojiNames.map(x => populateEmoji(x, noteUserHost)));
return emojis.filter((x): x is PopulatedEmoji => x != null);
}

View file

@ -1,15 +1,14 @@
import { EntityRepository, Repository, In } from 'typeorm'; import { EntityRepository, Repository, In } from 'typeorm';
import { Note } from '../entities/note'; import { Note } from '../entities/note';
import { User } from '../entities/user'; import { User } from '../entities/user';
import { Emojis, Users, PollVotes, DriveFiles, NoteReactions, Followings, Polls, Channels } from '..'; import { Users, PollVotes, DriveFiles, NoteReactions, Followings, Polls, Channels } from '..';
import { SchemaType } from '../../misc/schema'; import { SchemaType } from '../../misc/schema';
import { awaitAll } from '../../prelude/await-all'; import { awaitAll } from '../../prelude/await-all';
import { convertLegacyReaction, convertLegacyReactions, decodeReaction } from '../../misc/reaction-lib'; import { convertLegacyReaction, convertLegacyReactions, decodeReaction } from '../../misc/reaction-lib';
import { toString } from '../../mfm/to-string'; import { toString } from '../../mfm/to-string';
import { parse } from '../../mfm/parse'; import { parse } from '../../mfm/parse';
import { Emoji } from '../entities/emoji';
import { concat } from '../../prelude/array';
import { NoteReaction } from '../entities/note-reaction'; import { NoteReaction } from '../entities/note-reaction';
import { populateEmojis } from '../../misc/populate-emojis';
export type PackedNote = SchemaType<typeof packedNoteSchema>; export type PackedNote = SchemaType<typeof packedNoteSchema>;
@ -85,7 +84,6 @@ export class NoteRepository extends Repository<Note> {
detail?: boolean; detail?: boolean;
skipHide?: boolean; skipHide?: boolean;
_hint_?: { _hint_?: {
emojis: Emoji[] | null;
myReactions: Map<Note['id'], NoteReaction | null>; myReactions: Map<Note['id'], NoteReaction | null>;
}; };
} }
@ -135,93 +133,6 @@ export class NoteRepository extends Repository<Note> {
}; };
} }
/**
* emojisを解決する
* @param emojiNames Note等に添付されたカスタム絵文字名 (:)
* @param noteUserHost Noteのホスト
* @param reactionNames Note等にリアクションされたカスタム絵文字名 (:)
*/
async function populateEmojis(emojiNames: string[], noteUserHost: string | null, reactionNames: string[]) {
const customReactions = reactionNames?.map(x => decodeReaction(x)).filter(x => x.name);
let all = [] as {
name: string,
url: string
}[];
// 与えられたhintだけで十分(=新たにクエリする必要がない)かどうかを表すフラグ
let enough = true;
if (options?._hint_?.emojis) {
for (const name of emojiNames) {
const matched = options._hint_.emojis.find(x => x.name === name && x.host === noteUserHost);
if (matched) {
all.push({
name: matched.name,
url: matched.url,
});
} else {
enough = false;
}
}
for (const customReaction of customReactions) {
const matched = options._hint_.emojis.find(x => x.name === customReaction.name && x.host === customReaction.host);
if (matched) {
all.push({
name: `${matched.name}@${matched.host || '.'}`, // @host付きでローカルは.
url: matched.url,
});
} else {
enough = false;
}
}
} else {
enough = false;
}
if (enough) return all;
// カスタム絵文字
if (emojiNames?.length > 0) {
const tmp = await Emojis.find({
where: {
name: In(emojiNames),
host: noteUserHost
},
select: ['name', 'host', 'url']
}).then(emojis => emojis.map((emoji: Emoji) => {
return {
name: emoji.name,
url: emoji.url,
};
}));
all = concat([all, tmp]);
}
if (customReactions?.length > 0) {
const where = [] as {}[];
for (const customReaction of customReactions) {
where.push({
name: customReaction.name,
host: customReaction.host
});
}
const tmp = await Emojis.find({
where,
select: ['name', 'host', 'url']
}).then(emojis => emojis.map((emoji: Emoji) => {
return {
name: `${emoji.name}@${emoji.host || '.'}`, // @host付きでローカルは.
url: emoji.url,
};
}));
all = concat([all, tmp]);
}
return all;
}
async function populateMyReaction() { async function populateMyReaction() {
if (options?._hint_?.myReactions) { if (options?._hint_?.myReactions) {
const reaction = options._hint_.myReactions.get(note.id); const reaction = options._hint_.myReactions.get(note.id);
@ -257,15 +168,14 @@ export class NoteRepository extends Repository<Note> {
: await Channels.findOne(note.channelId) : await Channels.findOne(note.channelId)
: null; : null;
const reactionEmojiNames = Object.keys(note.reactions).filter(x => x?.startsWith(':')).map(x => decodeReaction(x).reaction).map(x => x.replace(/:/g, ''));
const packed = await awaitAll({ const packed = await awaitAll({
id: note.id, id: note.id,
createdAt: note.createdAt.toISOString(), createdAt: note.createdAt.toISOString(),
userId: note.userId, userId: note.userId,
user: Users.pack(note.user || note.userId, meId, { user: Users.pack(note.user || note.userId, meId, {
detail: false, detail: false,
_hint_: {
emojis: options?._hint_?.emojis || null
}
}), }),
text: text, text: text,
cw: note.cw, cw: note.cw,
@ -277,7 +187,7 @@ export class NoteRepository extends Repository<Note> {
repliesCount: note.repliesCount, repliesCount: note.repliesCount,
reactions: convertLegacyReactions(note.reactions), reactions: convertLegacyReactions(note.reactions),
tags: note.tags.length > 0 ? note.tags : undefined, tags: note.tags.length > 0 ? note.tags : undefined,
emojis: populateEmojis(note.emojis, host, Object.keys(note.reactions)), emojis: populateEmojis(note.emojis.concat(reactionEmojiNames), host),
fileIds: note.fileIds, fileIds: note.fileIds,
files: DriveFiles.packMany(note.fileIds), files: DriveFiles.packMany(note.fileIds),
replyId: note.replyId, replyId: note.replyId,
@ -350,48 +260,10 @@ export class NoteRepository extends Repository<Note> {
} }
} }
// TODO: ここら辺の処理をaggregateEmojisみたいな関数に切り出したい
let emojisWhere: any[] = [];
for (const note of notes) {
if (typeof note !== 'object') continue;
emojisWhere.push({
name: In(note.emojis),
host: note.userHost
});
if (note.renote) {
emojisWhere.push({
name: In(note.renote.emojis),
host: note.renote.userHost
});
if (note.renote.user) {
emojisWhere.push({
name: In(note.renote.user.emojis),
host: note.renote.userHost
});
}
}
const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name);
emojisWhere = emojisWhere.concat(customReactions.map(x => ({
name: x.name,
host: x.host
})));
if (note.user) {
emojisWhere.push({
name: In(note.user.emojis),
host: note.userHost
});
}
}
const emojis = emojisWhere.length > 0 ? await Emojis.find({
where: emojisWhere,
select: ['name', 'host', 'url']
}) : null;
return await Promise.all(notes.map(n => this.pack(n, me, { return await Promise.all(notes.map(n => this.pack(n, me, {
...options, ...options,
_hint_: { _hint_: {
myReactions: myReactionsMap, myReactions: myReactionsMap
emojis: emojis
} }
}))); })));
} }

View file

@ -1,13 +1,11 @@
import { EntityRepository, In, Repository } from 'typeorm'; import { EntityRepository, In, Repository } from 'typeorm';
import { Users, Notes, UserGroupInvitations, AccessTokens, NoteReactions, Emojis } from '..'; import { Users, Notes, UserGroupInvitations, AccessTokens, NoteReactions } from '..';
import { Notification } from '../entities/notification'; import { Notification } from '../entities/notification';
import { awaitAll } from '../../prelude/await-all'; import { awaitAll } from '../../prelude/await-all';
import { SchemaType } from '../../misc/schema'; import { SchemaType } from '../../misc/schema';
import { Note } from '../entities/note'; import { Note } from '../entities/note';
import { NoteReaction } from '../entities/note-reaction'; import { NoteReaction } from '../entities/note-reaction';
import { User } from '../entities/user'; import { User } from '../entities/user';
import { decodeReaction } from '../../misc/reaction-lib';
import { Emoji } from '../entities/emoji';
export type PackedNotification = SchemaType<typeof packedNotificationSchema>; export type PackedNotification = SchemaType<typeof packedNotificationSchema>;
@ -17,7 +15,6 @@ export class NotificationRepository extends Repository<Notification> {
src: Notification['id'] | Notification, src: Notification['id'] | Notification,
options: { options: {
_hintForEachNotes_?: { _hintForEachNotes_?: {
emojis: Emoji[] | null;
myReactions: Map<Note['id'], NoteReaction | null>; myReactions: Map<Note['id'], NoteReaction | null>;
}; };
} }
@ -101,47 +98,9 @@ export class NotificationRepository extends Repository<Notification> {
myReactionsMap.set(target, myReactions.find(reaction => reaction.noteId === target) || null); myReactionsMap.set(target, myReactions.find(reaction => reaction.noteId === target) || null);
} }
// TODO: ここら辺の処理をaggregateEmojisみたいな関数に切り出したい
let emojisWhere: any[] = [];
for (const note of notes) {
if (typeof note !== 'object') continue;
emojisWhere.push({
name: In(note.emojis),
host: note.userHost
});
if (note.renote) {
emojisWhere.push({
name: In(note.renote.emojis),
host: note.renote.userHost
});
if (note.renote.user) {
emojisWhere.push({
name: In(note.renote.user.emojis),
host: note.renote.userHost
});
}
}
const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name);
emojisWhere = emojisWhere.concat(customReactions.map(x => ({
name: x.name,
host: x.host
})));
if (note.user) {
emojisWhere.push({
name: In(note.user.emojis),
host: note.userHost
});
}
}
const emojis = emojisWhere.length > 0 ? await Emojis.find({
where: emojisWhere,
select: ['name', 'host', 'url']
}) : null;
return await Promise.all(notifications.map(x => this.pack(x, { return await Promise.all(notifications.map(x => this.pack(x, {
_hintForEachNotes_: { _hintForEachNotes_: {
myReactions: myReactionsMap, myReactions: myReactionsMap
emojis: emojis,
} }
}))); })));
} }

View file

@ -1,11 +1,11 @@
import $ from 'cafy'; import $ from 'cafy';
import { EntityRepository, Repository, In, Not } from 'typeorm'; import { EntityRepository, Repository, In, Not } from 'typeorm';
import { User, ILocalUser, IRemoteUser } from '../entities/user'; import { User, ILocalUser, IRemoteUser } from '../entities/user';
import { Emojis, Notes, NoteUnreads, FollowRequests, Notifications, MessagingMessages, UserNotePinings, Followings, Blockings, Mutings, UserProfiles, UserSecurityKeys, UserGroupJoinings, Pages, Announcements, AnnouncementReads, Antennas, AntennaNotes, ChannelFollowings, Instances } from '..'; import { Notes, NoteUnreads, FollowRequests, Notifications, MessagingMessages, UserNotePinings, Followings, Blockings, Mutings, UserProfiles, UserSecurityKeys, UserGroupJoinings, Pages, Announcements, AnnouncementReads, Antennas, AntennaNotes, ChannelFollowings, Instances } from '..';
import config from '../../config'; import config from '../../config';
import { SchemaType } from '../../misc/schema'; import { SchemaType } from '../../misc/schema';
import { awaitAll } from '../../prelude/await-all'; import { awaitAll } from '../../prelude/await-all';
import { Emoji } from '../entities/emoji'; import { populateEmojis } from '../../misc/populate-emojis';
export type PackedUser = SchemaType<typeof packedUserSchema>; export type PackedUser = SchemaType<typeof packedUserSchema>;
@ -150,9 +150,6 @@ export class UserRepository extends Repository<User> {
options?: { options?: {
detail?: boolean, detail?: boolean,
includeSecrets?: boolean, includeSecrets?: boolean,
_hint_?: {
emojis: Emoji[] | null;
};
} }
): Promise<PackedUser> { ): Promise<PackedUser> {
const opts = Object.assign({ const opts = Object.assign({
@ -170,34 +167,6 @@ export class UserRepository extends Repository<User> {
}) : []; }) : [];
const profile = opts.detail ? await UserProfiles.findOneOrFail(user.id) : null; const profile = opts.detail ? await UserProfiles.findOneOrFail(user.id) : null;
let emojis: Emoji[] = [];
if (user.emojis.length > 0) {
// 与えられたhintだけで十分(=新たにクエリする必要がない)かどうかを表すフラグ
let enough = true;
if (options?._hint_?.emojis) {
for (const name of user.emojis) {
const matched = options._hint_.emojis.find(x => x.name === name && x.host === user.host);
if (matched) {
emojis.push(matched);
} else {
enough = false;
}
}
} else {
enough = false;
}
if (!enough) {
emojis = await Emojis.find({
where: {
name: In(user.emojis),
host: user.host
},
select: ['name', 'host', 'url', 'aliases']
});
}
}
const falsy = opts.detail ? false : undefined; const falsy = opts.detail ? false : undefined;
const packed = { const packed = {
@ -220,9 +189,7 @@ export class UserRepository extends Repository<User> {
faviconUrl: instance.faviconUrl, faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor, themeColor: instance.themeColor,
} : undefined) : undefined, } : undefined) : undefined,
emojis: populateEmojis(user.emojis, user.host),
// カスタム絵文字添付
emojis: emojis,
...(opts.detail ? { ...(opts.detail ? {
url: profile!.url, url: profile!.url,