From 7e5e763584acd80b1baef6af2d20621c89fa0af9 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 13 Jan 2022 00:47:40 +0900 Subject: [PATCH] feat: emojis import --- packages/backend/package.json | 1 + .../processors/db/export-custom-emojis.ts | 12 ++- .../processors/db/import-custom-emojis.ts | 84 ++++++++++++++++ .../backend/src/queue/processors/db/index.ts | 2 + packages/backend/yarn.lock | 95 ++++++++++++++++++- packages/client/src/pages/admin/emojis.vue | 20 ++++ 6 files changed, 205 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/queue/processors/db/import-custom-emojis.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 65da382e2..c940e9830 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -180,6 +180,7 @@ "typeorm": "0.2.39", "typescript": "4.4.4", "ulid": "2.3.0", + "unzipper": "0.10.11", "uuid": "8.3.2", "web-push": "3.4.5", "websocket": "1.0.34", diff --git a/packages/backend/src/queue/processors/db/export-custom-emojis.ts b/packages/backend/src/queue/processors/db/export-custom-emojis.ts index 3930b9d6d..a420866dc 100644 --- a/packages/backend/src/queue/processors/db/export-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/export-custom-emojis.ts @@ -52,7 +52,7 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi }); }; - await writeMeta(`{"metaVersion":1,"emojis":[`); + await writeMeta(`{"metaVersion":2,"emojis":[`); const customEmojis = await Emojis.find({ where: { @@ -64,9 +64,9 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi }); for (const emoji of customEmojis) { - const exportId = ulid().toLowerCase(); const ext = mime.extension(emoji.type); - const emojiPath = path + '/' + exportId + (ext ? '.' + ext : ''); + const fileName = emoji.name + (ext ? '.' + ext : ''); + const emojiPath = path + '/' + fileName; fs.writeFileSync(emojiPath, '', 'binary'); let downloaded = false; @@ -77,8 +77,12 @@ export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promi logger.error(e); } + if (!downloaded) { + fs.unlinkSync(emojiPath); + } + const content = JSON.stringify({ - id: exportId, + fileName: fileName, downloaded: downloaded, emoji: emoji, }); diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts new file mode 100644 index 000000000..eb386bbb4 --- /dev/null +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -0,0 +1,84 @@ +import * as Bull from 'bull'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +const unzipper = require('unzipper'); +import { getConnection } from 'typeorm'; + +import { queueLogger } from '../../logger'; +import { downloadUrl } from '@/misc/download-url'; +import { DriveFiles, Emojis } from '@/models/index'; +import { DbUserImportJobData } from '@/queue/types'; +import addFile from '@/services/drive/add-file'; +import { genId } from '@/misc/gen-id'; + +const logger = queueLogger.createSubLogger('import-custom-emojis'); + +// TODO: 名前衝突時の動作を選べるようにする +export async function importCustomEmojis(job: Bull.Job, done: any): Promise { + logger.info(`Importing custom emojis ...`); + + const file = await DriveFiles.findOne({ + id: job.data.fileId, + }); + if (file == null) { + done(); + return; + } + + // Create temp dir + const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => { + tmp.dir((e, path, cleanup) => { + if (e) return rej(e); + res([path, cleanup]); + }); + }); + + logger.info(`Temp dir is ${path}`); + + const destPath = path + '/emojis.zip'; + + try { + fs.writeFileSync(destPath, '', 'binary'); + await downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + logger.error(e); + throw e; + } + + const outputPath = path + '/emojis'; + const unzipStream = fs.createReadStream(destPath); + const extractor = unzipper.Extract({ path: outputPath }); + extractor.on('close', async () => { + const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8'); + const meta = JSON.parse(metaRaw); + + for (const record of meta.emojis) { + if (!record.downloaded) continue; + const emojiInfo = record.emoji; + const emojiPath = outputPath + '/' + record.fileName; + await Emojis.delete({ + name: emojiInfo.name, + }); + const driveFile = await addFile(null, emojiPath, record.fileName, null, null, true); + const emoji = await Emojis.insert({ + id: genId(), + updatedAt: new Date(), + name: emojiInfo.name, + category: emojiInfo.category, + host: null, + aliases: emojiInfo.aliases, + url: driveFile.url, + type: driveFile.type, + }).then(x => Emojis.findOneOrFail(x.identifiers[0])); + } + + await getConnection().queryResultCache!.remove(['meta_emojis']); + + cleanup(); + + logger.succ('Imported'); + done(); + }); + unzipStream.pipe(extractor); + logger.succ(`Unzipping to ${outputPath}`); +} diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts index 1542f401e..5fffa378f 100644 --- a/packages/backend/src/queue/processors/db/index.ts +++ b/packages/backend/src/queue/processors/db/index.ts @@ -12,6 +12,7 @@ import { importUserLists } from './import-user-lists'; import { deleteAccount } from './delete-account'; import { importMuting } from './import-muting'; import { importBlocking } from './import-blocking'; +import { importCustomEmojis } from './import-custom-emojis'; const jobs = { deleteDriveFiles, @@ -25,6 +26,7 @@ const jobs = { importMuting, importBlocking, importUserLists, + importCustomEmojis, deleteAccount, } as Record | Bull.ProcessPromiseFunction>; diff --git a/packages/backend/yarn.lock b/packages/backend/yarn.lock index 16144b6d5..9e21fb29e 100644 --- a/packages/backend/yarn.lock +++ b/packages/backend/yarn.lock @@ -1522,6 +1522,11 @@ big-integer@^1.6.16: resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e" integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w== +big-integer@^1.6.17: + version "1.6.51" + resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.51.tgz#0df92a5d9880560d3ff2d5fd20245c889d130686" + integrity sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg== + big.js@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" @@ -1532,6 +1537,14 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c" integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow== +binary@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79" + integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk= + dependencies: + buffers "~0.1.1" + chainsaw "~0.1.0" + bl@^4.0.1, bl@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489" @@ -1546,6 +1559,11 @@ bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +bluebird@~3.4.1: + version "3.4.7" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" + integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM= + blurhash@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.4.tgz#a7010ceb3019cd2c9809b17c910ebf6175d29244" @@ -1677,6 +1695,11 @@ buffer-from@^1.0.0, buffer-from@^1.1.1: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer-indexof-polyfill@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz#d2732135c5999c64b277fcf9b1abe3498254729c" + integrity sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A== + buffer-writer@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04" @@ -1707,6 +1730,11 @@ buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" +buffers@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s= + bufferutil@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.1.tgz#3a177e8e5819a1243fe16b63a199951a7ad8d4a7" @@ -1875,6 +1903,13 @@ cbor@8.1.0: dependencies: nofilter "^3.1.0" +chainsaw@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98" + integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg= + dependencies: + traverse ">=0.3.0 <0.4" + chalk@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.0.0.tgz#6e98081ed2d17faab615eb52ac66ec1fe6209e72" @@ -2789,6 +2824,13 @@ dotenv@^8.2.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw== +duplexer2@~0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME= + dependencies: + readable-stream "^2.0.2" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -3480,6 +3522,16 @@ fsevents@~2.1.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fstream@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045" + integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg== + dependencies: + graceful-fs "^4.1.2" + inherits "~2.0.0" + mkdirp ">=0.5 0" + rimraf "2" + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -3690,7 +3742,7 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.4: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== -graceful-fs@^4.2.0: +graceful-fs@^4.2.0, graceful-fs@^4.2.2: version "4.2.8" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== @@ -4007,7 +4059,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -4800,6 +4852,11 @@ lilconfig@^2.0.3: resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.0.3.tgz#68f3005e921dafbd2a2afb48379986aa6d2579fd" integrity sha512-EHKqr/+ZvdKCifpNrJCKxBTgk5XupZA3y/aCPY9mxfgBzmgh93Mt/WqjjQ38oMxXuvDokaKiM3lAgvSH2sjtHg== +listenercount@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937" + integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc= + loader-runner@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" @@ -5204,7 +5261,7 @@ mkdirp-classic@^0.5.3: resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -mkdirp@0.x, mkdirp@^0.5.4: +mkdirp@0.x, "mkdirp@>=0.5 0", mkdirp@^0.5.4: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== @@ -6598,7 +6655,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2: +readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -6780,6 +6837,13 @@ reusify@^1.0.4: resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== +rimraf@2: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -6914,7 +6978,7 @@ set-blocking@^2.0.0, set-blocking@~2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= -setimmediate@^1.0.5: +setimmediate@^1.0.5, setimmediate@~1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU= @@ -7584,6 +7648,11 @@ trace-redirect@1.0.6: resolved "https://registry.yarnpkg.com/trace-redirect/-/trace-redirect-1.0.6.tgz#ac629b5bf8247d30dde5a35fe9811b811075b504" integrity sha512-UUfa1DjjU5flcjMdaFIiIEGDTyu2y/IiMjOX4uGXa7meKBS4vD4f2Uy/tken9Qkd4Jsm4sRsfZcIIPqrRVF3Mg== +"traverse@>=0.3.0 <0.4": + version "0.3.9" + resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" + integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk= + ts-jest@^25.2.1: version "25.5.1" resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-25.5.1.tgz#2913afd08f28385d54f2f4e828be4d261f4337c7" @@ -7827,6 +7896,22 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= +unzipper@0.10.11: + version "0.10.11" + resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.10.11.tgz#0b4991446472cbdb92ee7403909f26c2419c782e" + integrity sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw== + dependencies: + big-integer "^1.6.17" + binary "~0.3.0" + bluebird "~3.4.1" + buffer-indexof-polyfill "~1.0.0" + duplexer2 "~0.1.4" + fstream "^1.0.12" + graceful-fs "^4.2.2" + listenercount "~1.0.1" + readable-stream "~2.3.6" + setimmediate "~1.0.4" + uri-js@^4.2.2: version "4.2.2" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" diff --git a/packages/client/src/pages/admin/emojis.vue b/packages/client/src/pages/admin/emojis.vue index f995460f1..38182e5be 100644 --- a/packages/client/src/pages/admin/emojis.vue +++ b/packages/client/src/pages/admin/emojis.vue @@ -182,6 +182,26 @@ const menu = (ev: MouseEvent) => { }); }); } + }, { + icon: 'fas fa-upload', + text: i18n.locale.import, + action: async () => { + const file = await selectFile(ev.currentTarget || ev.target); + os.api('admin/emoji/import-zip', { + fileId: file.id, + }) + .then(() => { + os.alert({ + type: 'info', + text: i18n.locale.importRequested, + }); + }).catch((e) => { + os.alert({ + type: 'error', + text: e.message, + }); + }); + } }], ev.currentTarget || ev.target); };