feat: カスタム絵文字エクスポート

This commit is contained in:
syuilo 2021-12-10 18:24:26 +09:00
parent 0b038f6477
commit bd903cdbb5
10 changed files with 298 additions and 5 deletions

View File

@ -13,6 +13,7 @@
- Added a user-level instance mute in user settings
- フォローエクスポートでミュートしているユーザーを含めないオプションを追加
- フォローエクスポートで使われていないアカウントを含めないオプションを追加
- カスタム絵文字エクスポート機能
### Bugfixes
- クライアント: タッチ機能付きディスプレイを使っていてマウス操作をしている場合に一部機能が動作しない問題を修正

View File

@ -74,6 +74,7 @@
"@typescript-eslint/eslint-plugin": "5.3.1",
"@typescript-eslint/parser": "5.1.0",
"abort-controller": "3.0.0",
"archiver": "5.3.0",
"autobind-decorator": "2.4.0",
"autosize": "4.0.4",
"autwh": "0.1.0",
@ -131,6 +132,7 @@
"koa-views": "7.0.2",
"langmap": "0.0.16",
"mfm-js": "0.20.0",
"mime-types": "2.1.34",
"misskey-js": "0.0.8",
"mocha": "8.4.0",
"ms": "3.0.0-canary.1",

View File

@ -11,7 +11,7 @@ const PrivateIp = require('private-ip');
const pipeline = util.promisify(stream.pipeline);
export async function downloadUrl(url: string, path: string) {
export async function downloadUrl(url: string, path: string): Promise<void> {
const logger = new Logger('download');
logger.info(`Downloading ${chalk.cyan(url)} ...`);

View File

@ -117,6 +117,15 @@ export function createDeleteDriveFilesJob(user: ThinUser) {
});
}
export function createExportCustomEmojisJob(user: ThinUser) {
return dbQueue.add('exportCustomEmojis', {
user: user,
}, {
removeOnComplete: true,
removeOnFail: true,
});
}
export function createExportNotesJob(user: ThinUser) {
return dbQueue.add('exportNotes', {
user: user,

View File

@ -0,0 +1,131 @@
import * as Bull from 'bull';
import * as tmp from 'tmp';
import * as fs from 'fs';
import { ulid } from 'ulid';
const mime = require('mime-types');
const archiver = require('archiver');
import { queueLogger } from '../../logger';
import addFile from '@/services/drive/add-file';
import * as dateFormat from 'dateformat';
import { Users, Emojis } from '@/models/index';
import { } from '@/queue/types';
import { downloadUrl } from '@/misc/download-url';
const logger = queueLogger.createSubLogger('export-custom-emojis');
export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> {
logger.info(`Exporting custom emojis ...`);
const user = await Users.findOne(job.data.user.id);
if (user == null) {
done();
return;
}
// Create temp dir
const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
tmp.dir((e, path, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
logger.info(`Temp dir is ${path}`);
const metaPath = path + '/meta.json';
fs.writeFileSync(metaPath, '', 'utf-8');
const metaStream = fs.createWriteStream(metaPath, { flags: 'a' });
await new Promise<void>((res, rej) => {
metaStream.write('[', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
const customEmojis = await Emojis.find({
where: {
host: null,
},
order: {
id: 'ASC',
},
});
for (const emoji of customEmojis) {
const exportId = ulid().toLowerCase();
const emojiPath = path + '/' + exportId + '.' + mime.extension(emoji.type);
fs.writeFileSync(emojiPath, '', 'binary');
let downloaded = false;
try {
await downloadUrl(emoji.url, emojiPath);
downloaded = true;
} catch (e) { // TODO: 何度か再試行
logger.error(e);
}
await new Promise<void>((res, rej) => {
const content = JSON.stringify({
id: exportId,
downloaded: downloaded,
emoji: emoji,
});
const isFirst = customEmojis.indexOf(emoji) === 0;
metaStream.write(isFirst ? content : ',\n' + content, err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
}
await new Promise<void>((res, rej) => {
metaStream.write(']', err => {
if (err) {
logger.error(err);
rej(err);
} else {
res();
}
});
});
metaStream.end();
// Create archive
const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
const archiveStream = fs.createWriteStream(archivePath);
const archive = archiver('zip', {
zlib: { level: 0 },
});
archiveStream.on('close', async () => {
logger.succ(`Exported to: ${archivePath}`);
const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip';
const driveFile = await addFile(user, archivePath, fileName, null, null, true);
logger.succ(`Exported to: ${driveFile.id}`);
cleanup();
archiveCleanup();
done();
});
archive.pipe(archiveStream);
archive.directory(path, false);
archive.finalize();
}

View File

@ -74,7 +74,8 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
}
const content = JSON.stringify(serialize(note, poll));
await new Promise<void>((res, rej) => {
stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
const isFirst = exportedNotesCount === 0;
stream.write(isFirst ? content : ',\n' + content, err => {
if (err) {
logger.error(err);
rej(err);

View File

@ -1,6 +1,7 @@
import * as Bull from 'bull';
import { DbJobData } from '@/queue/types';
import { deleteDriveFiles } from './delete-drive-files';
import { exportCustomEmojis } from './export-custom-emojis';
import { exportNotes } from './export-notes';
import { exportFollowing } from './export-following';
import { exportMute } from './export-mute';
@ -14,6 +15,7 @@ import { importBlocking } from './import-blocking';
const jobs = {
deleteDriveFiles,
exportCustomEmojis,
exportNotes,
exportFollowing,
exportMute,

View File

@ -0,0 +1,17 @@
import $ from 'cafy';
import define from '../define';
import { createExportCustomEmojisJob } from '@/queue/index';
import ms from 'ms';
export const meta = {
secure: true,
requireCredential: true as const,
limit: {
duration: ms('1hour'),
max: 1,
},
};
export default define(meta, async (ps, user) => {
createExportCustomEmojisJob(user);
});

View File

@ -1318,6 +1318,35 @@ aproba@^1.0.3:
resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==
archiver-utils@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2"
integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==
dependencies:
glob "^7.1.4"
graceful-fs "^4.2.0"
lazystream "^1.0.0"
lodash.defaults "^4.2.0"
lodash.difference "^4.5.0"
lodash.flatten "^4.4.0"
lodash.isplainobject "^4.0.6"
lodash.union "^4.6.0"
normalize-path "^3.0.0"
readable-stream "^2.0.0"
archiver@5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba"
integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg==
dependencies:
archiver-utils "^2.1.0"
async "^3.2.0"
buffer-crc32 "^0.2.1"
readable-stream "^3.6.0"
readdir-glob "^1.0.0"
tar-stream "^2.2.0"
zip-stream "^4.1.0"
are-we-there-yet@~1.1.2:
version "1.1.5"
resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21"
@ -1645,6 +1674,11 @@ buffer-alloc@^1.2.0:
buffer-alloc-unsafe "^1.1.0"
buffer-fill "^1.0.0"
buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
version "0.2.13"
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=
buffer-equal-constant-time@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
@ -2167,6 +2201,16 @@ compare-versions@3.6.0:
resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62"
integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==
compress-commons@^4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.1.tgz#df2a09a7ed17447642bad10a85cc9a19e5c42a7d"
integrity sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==
dependencies:
buffer-crc32 "^0.2.13"
crc32-stream "^4.0.2"
normalize-path "^3.0.0"
readable-stream "^3.6.0"
concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@ -2249,7 +2293,7 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
crc-32@1.2.0:
crc-32@1.2.0, crc-32@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208"
integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==
@ -2257,6 +2301,14 @@ crc-32@1.2.0:
exit-on-epipe "~1.0.1"
printj "~1.1.0"
crc32-stream@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-4.0.2.tgz#c922ad22b38395abe9d3870f02fa8134ed709007"
integrity sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==
dependencies:
crc-32 "^1.2.0"
readable-stream "^3.4.0"
create-require@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
@ -4901,6 +4953,13 @@ langmap@0.0.16:
resolved "https://registry.yarnpkg.com/langmap/-/langmap-0.0.16.tgz#2fe3e98a531fec0fec546624ebe168c2855bab56"
integrity sha512-AtYvBK7BsDvWwnSfmO7CfgeUy7GUT1wK3QX8eKH/Ey/eXodqoHuAtvdQ82hmWD9QVFVKnuiNjym9fGY4qSJeLA==
lazystream@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.1.tgz#494c831062f1f9408251ec44db1cba29242a2638"
integrity sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==
dependencies:
readable-stream "^2.0.5"
lcid@^3.0.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lcid/-/lcid-3.1.1.tgz#9030ec479a058fc36b5e8243ebaac8b6ac582fd0"
@ -4981,6 +5040,11 @@ lodash.defaults@^4.0.1, lodash.defaults@^4.2.0:
resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
lodash.difference@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c"
integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=
lodash.filter@^4.4.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
@ -5006,6 +5070,11 @@ lodash.isfinite@^3.3.2:
resolved "https://registry.yarnpkg.com/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz#fb89b65a9a80281833f0b7478b3a5104f898ebb3"
integrity sha1-+4m2WpqAKBgz8LdHizpRBPiY67M=
lodash.isplainobject@^4.0.6:
version "4.0.6"
resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
lodash.isregexp@3.0.5:
version "3.0.5"
resolved "https://registry.yarnpkg.com/lodash.isregexp/-/lodash.isregexp-3.0.5.tgz#e0f596242f2fa228a840086b6c8ad82e4b71fd2d"
@ -5051,6 +5120,11 @@ lodash.sortby@^4.7.0:
resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
lodash.union@^4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88"
integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=
lodash.uniq@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@ -5198,6 +5272,18 @@ mime-db@1.44.0:
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==
mime-db@1.51.0:
version "1.51.0"
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c"
integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==
mime-types@2.1.34:
version "2.1.34"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24"
integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==
dependencies:
mime-db "1.51.0"
mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24:
version "2.1.27"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
@ -6777,7 +6863,7 @@ readable-stream@1.1.x:
isarray "0.0.1"
string_decoder "~0.10.x"
readable-stream@^2.0.6, readable-stream@^2.2.2:
readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@ -6807,6 +6893,13 @@ readable-web-to-node-stream@^3.0.0:
"@types/readable-stream" "^2.3.9"
readable-stream "^3.6.0"
readdir-glob@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==
dependencies:
minimatch "^3.0.4"
readdirp@~3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.3.0.tgz#984458d13a1e42e2e9f5841b129e162f369aff17"
@ -7664,7 +7757,7 @@ tar-stream@^2.0.0:
inherits "^2.0.3"
readable-stream "^3.1.1"
tar-stream@^2.1.4:
tar-stream@^2.1.4, tar-stream@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
@ -8583,3 +8676,12 @@ zen-observable@^0.8.15:
version "0.8.15"
resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==
zip-stream@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79"
integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==
dependencies:
archiver-utils "^2.1.0"
compress-commons "^4.1.0"
readable-stream "^3.6.0"

View File

@ -21,10 +21,38 @@ export default defineComponent({
title: this.$ts.customEmojis,
icon: 'fas fa-laugh',
bg: 'var(--bg)',
actions: [{
icon: 'fas fa-ellipsis-h',
handler: this.menu
}],
})),
tab: 'category',
}
},
methods: {
menu(ev) {
os.popupMenu([{
icon: 'fas fa-download',
text: this.$ts.export,
action: async () => {
os.api('export-custom-emojis', {
})
.then(() => {
os.alert({
type: 'info',
text: this.$ts.exportRequested,
});
}).catch((e) => {
os.alert({
type: 'error',
text: e.message,
});
});
}
}], ev.currentTarget || ev.target);
}
}
});
</script>