diff --git a/CHANGELOG.md b/CHANGELOG.md index e45e02090..32ce14407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Added a user-level instance mute in user settings - フォローエクスポートでミュートしているユーザーを含めないオプションを追加 - フォローエクスポートで使われていないアカウントを含めないオプションを追加 +- カスタム絵文字エクスポート機能 ### Bugfixes - クライアント: タッチ機能付きディスプレイを使っていてマウス操作をしている場合に一部機能が動作しない問題を修正 diff --git a/packages/backend/package.json b/packages/backend/package.json index 5c6ffca3b..abcde0ab7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -74,6 +74,7 @@ "@typescript-eslint/eslint-plugin": "5.3.1", "@typescript-eslint/parser": "5.1.0", "abort-controller": "3.0.0", + "archiver": "5.3.0", "autobind-decorator": "2.4.0", "autosize": "4.0.4", "autwh": "0.1.0", @@ -131,6 +132,7 @@ "koa-views": "7.0.2", "langmap": "0.0.16", "mfm-js": "0.20.0", + "mime-types": "2.1.34", "misskey-js": "0.0.8", "mocha": "8.4.0", "ms": "3.0.0-canary.1", diff --git a/packages/backend/src/misc/download-url.ts b/packages/backend/src/misc/download-url.ts index 3dc640dc0..8e1f7b9e2 100644 --- a/packages/backend/src/misc/download-url.ts +++ b/packages/backend/src/misc/download-url.ts @@ -11,7 +11,7 @@ const PrivateIp = require('private-ip'); const pipeline = util.promisify(stream.pipeline); -export async function downloadUrl(url: string, path: string) { +export async function downloadUrl(url: string, path: string): Promise { const logger = new Logger('download'); logger.info(`Downloading ${chalk.cyan(url)} ...`); diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index c5a07673b..2fbc1b1c0 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -117,6 +117,15 @@ export function createDeleteDriveFilesJob(user: ThinUser) { }); } +export function createExportCustomEmojisJob(user: ThinUser) { + return dbQueue.add('exportCustomEmojis', { + user: user, + }, { + removeOnComplete: true, + removeOnFail: true, + }); +} + export function createExportNotesJob(user: ThinUser) { return dbQueue.add('exportNotes', { user: user, diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts new file mode 100644 index 000000000..ed0ad249a --- /dev/null +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -0,0 +1,131 @@ +import * as Bull from 'bull'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; + +import { ulid } from 'ulid'; +const mime = require('mime-types'); +const archiver = require('archiver'); +import { queueLogger } from '../../logger'; +import addFile from '@/services/drive/add-file'; +import * as dateFormat from 'dateformat'; +import { Users, Emojis } from '@/models/index'; +import { } from '@/queue/types'; +import { downloadUrl } from '@/misc/download-url'; + +const logger = queueLogger.createSubLogger('export-custom-emojis'); + +export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise { + logger.info(`Exporting custom emojis ...`); + + const user = await Users.findOne(job.data.user.id); + if (user == null) { + done(); + return; + } + + // Create temp dir + const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp dir is ${path}`); + + const metaPath = path + '/meta.json'; + + fs.writeFileSync(metaPath, '', 'utf-8'); + + const metaStream = fs.createWriteStream(metaPath, { flags: 'a' }); + + await new Promise((res, rej) => { + metaStream.write('[', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + const customEmojis = await Emojis.find({ + where: { + host: null, + }, + order: { + id: 'ASC', + }, + }); + + for (const emoji of customEmojis) { + const exportId = ulid().toLowerCase(); + const emojiPath = path + '/' + exportId + '.' + mime.extension(emoji.type); + fs.writeFileSync(emojiPath, '', 'binary'); + let downloaded = false; + + try { + await downloadUrl(emoji.url, emojiPath); + downloaded = true; + } catch (e) { // TODO: 何度か再試行 + logger.error(e); + } + + await new Promise((res, rej) => { + const content = JSON.stringify({ + id: exportId, + downloaded: downloaded, + emoji: emoji, + }); + const isFirst = customEmojis.indexOf(emoji) === 0; + metaStream.write(isFirst ? content : ',\n' + content, err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + } + + await new Promise((res, rej) => { + metaStream.write(']', err => { + if (err) { + logger.error(err); + rej(err); + } else { + res(); + } + }); + }); + + metaStream.end(); + + // Create archive + const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.file((e, path, fd, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + const archiveStream = fs.createWriteStream(archivePath); + const archive = archiver('zip', { + zlib: { level: 0 }, + }); + archiveStream.on('close', async () => { + logger.succ(`Exported to: ${archivePath}`); + + const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip'; + const driveFile = await addFile(user, archivePath, fileName, null, null, true); + + logger.succ(`Exported to: ${driveFile.id}`); + cleanup(); + archiveCleanup(); + done(); + }); + archive.pipe(archiveStream); + archive.directory(path, false); + archive.finalize(); +} diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts index 761f4d827..7220455ed 100644 --- a/packages/backend/src/queue/processors/db/export-notes.ts +++ b/packages/backend/src/queue/processors/db/export-notes.ts @@ -74,7 +74,8 @@ export async function exportNotes(job: Bull.Job, done: any): Prom } const content = JSON.stringify(serialize(note, poll)); await new Promise((res, rej) => { - stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => { + const isFirst = exportedNotesCount === 0; + stream.write(isFirst ? content : ',\n' + content, err => { if (err) { logger.error(err); rej(err); diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 97087642b..1542f401e 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -1,6 +1,7 @@ import * as Bull from 'bull'; import { DbJobData } from '@/queue/types'; import { deleteDriveFiles } from './delete-drive-files'; +import { exportCustomEmojis } from './export-custom-emojis'; import { exportNotes } from './export-notes'; import { exportFollowing } from './export-following'; import { exportMute } from './export-mute'; @@ -14,6 +15,7 @@ import { importBlocking } from './import-blocking'; const jobs = { deleteDriveFiles, + exportCustomEmojis, exportNotes, exportFollowing, exportMute, diff --git a/packages/backend/src/server/api/endpoints/export-custom-emojis.ts b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts new file mode 100644 index 000000000..92738c828 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/export-custom-emojis.ts @@ -0,0 +1,17 @@ +import $ from 'cafy'; +import define from '../define'; +import { createExportCustomEmojisJob } from '@/queue/index'; +import ms from 'ms'; + +export const meta = { + secure: true, + requireCredential: true as const, + limit: { + duration: ms('1hour'), + max: 1, + }, +}; + +export default define(meta, async (ps, user) => { + createExportCustomEmojisJob(user); +}); diff --git a/packages/backend/yarn.lock b/packages/backend/yarn.lock index 821d21d6f..96fbb26c9 100644 --- a/packages/backend/yarn.lock +++ b/packages/backend/yarn.lock @@ -1318,6 +1318,35 @@ aproba@^1.0.3: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== +archiver-utils@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2" + integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw== + dependencies: + glob "^7.1.4" + graceful-fs "^4.2.0" + lazystream "^1.0.0" + lodash.defaults "^4.2.0" + lodash.difference "^4.5.0" + lodash.flatten "^4.4.0" + lodash.isplainobject "^4.0.6" + lodash.union "^4.6.0" + normalize-path "^3.0.0" + readable-stream "^2.0.0" + +archiver@5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/archiver/-/archiver-5.3.0.tgz#dd3e097624481741df626267564f7dd8640a45ba" + integrity sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg== + dependencies: + archiver-utils "^2.1.0" + async "^3.2.0" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.0.0" + tar-stream "^2.2.0" + zip-stream "^4.1.0" + are-we-there-yet@~1.1.2: version "1.1.5" resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" @@ -1645,6 +1674,11 @@ buffer-alloc@^1.2.0: buffer-alloc-unsafe "^1.1.0" buffer-fill "^1.0.0" +buffer-crc32@^0.2.1, buffer-crc32@^0.2.13: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= + buffer-equal-constant-time@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" @@ -2167,6 +2201,16 @@ compare-versions@3.6.0: resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" integrity sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA== +compress-commons@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.1.tgz#df2a09a7ed17447642bad10a85cc9a19e5c42a7d" + integrity sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ== + dependencies: + buffer-crc32 "^0.2.13" + crc32-stream "^4.0.2" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -2249,7 +2293,7 @@ core-util-is@1.0.2, core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= -crc-32@1.2.0: +crc-32@1.2.0, crc-32@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.0.tgz#cb2db6e29b88508e32d9dd0ec1693e7b41a18208" integrity sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA== @@ -2257,6 +2301,14 @@ crc-32@1.2.0: exit-on-epipe "~1.0.1" printj "~1.1.0" +crc32-stream@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-4.0.2.tgz#c922ad22b38395abe9d3870f02fa8134ed709007" + integrity sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w== + dependencies: + crc-32 "^1.2.0" + readable-stream "^3.4.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -4901,6 +4953,13 @@ langmap@0.0.16: resolved "https://registry.yarnpkg.com/langmap/-/langmap-0.0.16.tgz#2fe3e98a531fec0fec546624ebe168c2855bab56" integrity sha512-AtYvBK7BsDvWwnSfmO7CfgeUy7GUT1wK3QX8eKH/Ey/eXodqoHuAtvdQ82hmWD9QVFVKnuiNjym9fGY4qSJeLA== +lazystream@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.1.tgz#494c831062f1f9408251ec44db1cba29242a2638" + integrity sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw== + dependencies: + readable-stream "^2.0.5" + lcid@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/lcid/-/lcid-3.1.1.tgz#9030ec479a058fc36b5e8243ebaac8b6ac582fd0" @@ -4981,6 +5040,11 @@ lodash.defaults@^4.0.1, lodash.defaults@^4.2.0: resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= +lodash.difference@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c" + integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw= + lodash.filter@^4.4.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace" @@ -5006,6 +5070,11 @@ lodash.isfinite@^3.3.2: resolved "https://registry.yarnpkg.com/lodash.isfinite/-/lodash.isfinite-3.3.2.tgz#fb89b65a9a80281833f0b7478b3a5104f898ebb3" integrity sha1-+4m2WpqAKBgz8LdHizpRBPiY67M= +lodash.isplainobject@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" + integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs= + lodash.isregexp@3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/lodash.isregexp/-/lodash.isregexp-3.0.5.tgz#e0f596242f2fa228a840086b6c8ad82e4b71fd2d" @@ -5051,6 +5120,11 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= +lodash.union@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" + integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg= + lodash.uniq@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" @@ -5198,6 +5272,18 @@ mime-db@1.44.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.51.0: + version "1.51.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c" + integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g== + +mime-types@2.1.34: + version "2.1.34" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24" + integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A== + dependencies: + mime-db "1.51.0" + mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -6777,7 +6863,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.6, readable-stream@^2.2.2: +readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -6807,6 +6893,13 @@ readable-web-to-node-stream@^3.0.0: "@types/readable-stream" "^2.3.9" readable-stream "^3.6.0" +readdir-glob@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4" + integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA== + dependencies: + minimatch "^3.0.4" + readdirp@~3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.3.0.tgz#984458d13a1e42e2e9f5841b129e162f369aff17" @@ -7664,7 +7757,7 @@ tar-stream@^2.0.0: inherits "^2.0.3" readable-stream "^3.1.1" -tar-stream@^2.1.4: +tar-stream@^2.1.4, tar-stream@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== @@ -8583,3 +8676,12 @@ zen-observable@^0.8.15: version "0.8.15" resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15" integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ== + +zip-stream@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-4.1.0.tgz#51dd326571544e36aa3f756430b313576dc8fc79" + integrity sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A== + dependencies: + archiver-utils "^2.1.0" + compress-commons "^4.1.0" + readable-stream "^3.6.0" diff --git a/packages/client/src/pages/emojis.vue b/packages/client/src/pages/emojis.vue index ae06fa793..2adb5345e 100644 --- a/packages/client/src/pages/emojis.vue +++ b/packages/client/src/pages/emojis.vue @@ -21,10 +21,38 @@ export default defineComponent({ title: this.$ts.customEmojis, icon: 'fas fa-laugh', bg: 'var(--bg)', + actions: [{ + icon: 'fas fa-ellipsis-h', + handler: this.menu + }], })), tab: 'category', } }, + + methods: { + menu(ev) { + os.popupMenu([{ + icon: 'fas fa-download', + text: this.$ts.export, + action: async () => { + os.api('export-custom-emojis', { + }) + .then(() => { + os.alert({ + type: 'info', + text: this.$ts.exportRequested, + }); + }).catch((e) => { + os.alert({ + type: 'error', + text: e.message, + }); + }); + } + }], ev.currentTarget || ev.target); + } + } });