From 8cde66b8ac10fcce8259495cf61d9725ec43eade Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Thu, 17 Nov 2022 00:14:16 -0500 Subject: [PATCH 1/3] backend: Add LibreTranslate support --- .../1668661888188-add-libretranslate.js | 19 +++ packages/backend/src/models/entities/meta.ts | 23 ++++ .../src/server/api/common/translator.ts | 12 ++ .../src/server/api/endpoints/admin/meta.ts | 3 +- .../server/api/endpoints/admin/update-meta.ts | 24 ++++ .../backend/src/server/api/endpoints/meta.ts | 3 +- .../server/api/endpoints/notes/translate.ts | 130 ++++++++++++------ 7 files changed, 173 insertions(+), 41 deletions(-) create mode 100644 packages/backend/migration/1668661888188-add-libretranslate.js create mode 100644 packages/backend/src/server/api/common/translator.ts diff --git a/packages/backend/migration/1668661888188-add-libretranslate.js b/packages/backend/migration/1668661888188-add-libretranslate.js new file mode 100644 index 000000000..cc5f47939 --- /dev/null +++ b/packages/backend/migration/1668661888188-add-libretranslate.js @@ -0,0 +1,19 @@ +export class addLibretranslate1668661888188 { + name = 'addLibretranslate1668661888188' + + async up(queryRunner) { + await queryRunner.query(`CREATE TYPE "public"."meta_translationservice_enum" AS ENUM('deepl', 'libretranslate')`); + await queryRunner.query(`ALTER TABLE "meta" ADD "translationService" "public"."meta_translationservice_enum"`); + await queryRunner.query(`ALTER TABLE "meta" ADD "libreTranslateAuthKey" character varying(128)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "libreTranslateEndpoint" character varying(2048)`); + // Set translationService to 'deepl' if auth key is already set + await queryRunner.query(`UPDATE "meta" SET "translationService"='deepl' WHERE "deeplAuthKey" IS NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "libreTranslateEndpoint"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "libreTranslateAuthKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "translationService"`); + await queryRunner.query(`DROP TYPE "public"."meta_translationservice_enum"`); + } +} diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index 0afb7c7fc..565d95a32 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -3,6 +3,11 @@ import { id } from '../id.js'; import { User } from './user.js'; import { Clip } from './clip.js'; +export enum TranslationService { + DeepL = 'deepl', + LibreTranslate = 'libretranslate', +} + @Entity() export class Meta { @PrimaryColumn({ @@ -299,6 +304,12 @@ export class Meta { }) public discordClientSecret: string | null; + @Column('enum', { + enum: TranslationService, + nullable: true, + }) + public translationService: TranslationService | null; + @Column('varchar', { length: 128, nullable: true, @@ -310,6 +321,18 @@ export class Meta { }) public deeplIsPro: boolean; + @Column('varchar', { + length: 128, + nullable: true, + }) + public libreTranslateAuthKey: string | null; + + @Column('varchar', { + length: 2048, + nullable: true, + }) + public libreTranslateEndpoint: string | null; + @Column('varchar', { length: 512, nullable: true, diff --git a/packages/backend/src/server/api/common/translator.ts b/packages/backend/src/server/api/common/translator.ts new file mode 100644 index 000000000..dde4686b3 --- /dev/null +++ b/packages/backend/src/server/api/common/translator.ts @@ -0,0 +1,12 @@ +import { Meta, TranslationService } from '@/models/entities/meta.js'; + +export function translatorAvailable(instance: Meta): boolean { + switch (instance.translationService) { + case TranslationService.DeepL: + return instance.deeplAuthKey != null; + case TranslationService.LibreTranslate: + return instance.libreTranslateEndpoint != null; + default: + return false; + } +} diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 0c53b1382..5107cfccf 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,5 +1,6 @@ import config from '@/config/index.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; +import { translatorAvailable } from '../../common/translator.js'; import define from '../../define.js'; export const meta = { @@ -317,7 +318,7 @@ export default define(meta, paramDef, async (ps, me) => { enableGithubIntegration: instance.enableGithubIntegration, enableDiscordIntegration: instance.enableDiscordIntegration, enableServiceWorker: instance.enableServiceWorker, - translatorAvailable: instance.deeplAuthKey != null, + translatorAvailable: translatorAvailable(instance), pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, cacheRemoteFiles: instance.cacheRemoteFiles, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index e83252c29..5958b3c98 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -1,5 +1,6 @@ import { insertModerationLog } from '@/services/insert-moderation-log.js'; import { fetchMeta, setMeta } from '@/misc/fetch-meta.js'; +import { TranslationService } from '@/models/entities/meta.js'; import define from '../../define.js'; export const meta = { @@ -55,8 +56,11 @@ export const paramDef = { type: 'string', } }, summalyProxy: { type: 'string', nullable: true }, + translationService: { type: 'string', nullable: true, enum: [null, ...Object.values(TranslationService)] }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, + libreTranslateAuthKey: { type: 'string', nullable: true }, + libreTranslateEndpoint: { type: 'string', nullable: true }, enableTwitterIntegration: { type: 'boolean' }, twitterConsumerKey: { type: 'string', nullable: true }, twitterConsumerSecret: { type: 'string', nullable: true }, @@ -362,6 +366,10 @@ export default define(meta, paramDef, async (ps, me) => { set.objectStorageS3ForcePathStyle = ps.objectStorageS3ForcePathStyle; } + if (ps.translationService !== undefined) { + set.translationService = ps.translationService; + } + if (ps.deeplAuthKey !== undefined) { if (ps.deeplAuthKey === '') { set.deeplAuthKey = null; @@ -374,6 +382,22 @@ export default define(meta, paramDef, async (ps, me) => { set.deeplIsPro = ps.deeplIsPro; } + if (ps.libreTranslateEndpoint !== undefined) { + if (ps.libreTranslateEndpoint === '') { + set.libreTranslateEndpoint = null; + } else { + set.libreTranslateEndpoint = ps.libreTranslateEndpoint; + } + } + + if (ps.libreTranslateAuthKey !== undefined) { + if (ps.libreTranslateAuthKey === '') { + set.libreTranslateAuthKey = null; + } else { + set.libreTranslateAuthKey = ps.libreTranslateAuthKey; + } + } + const meta = await fetchMeta(); await setMeta({ ...meta, diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 4357b8cad..511c97ddc 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -3,6 +3,7 @@ import config from '@/config/index.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { Emojis, Users } from '@/models/index.js'; import define from '../define.js'; +import { translatorAvailable } from '../common/translator.js'; export const meta = { tags: ['meta'], @@ -322,7 +323,7 @@ export default define(meta, paramDef, async (ps, me) => { enableServiceWorker: instance.enableServiceWorker, - translatorAvailable: instance.deeplAuthKey != null, + translatorAvailable: translatorAvailable(instance), pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index f62e07b4e..44057415a 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -4,6 +4,7 @@ import config from '@/config/index.js'; import { getAgentByUrl } from '@/misc/fetch.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; import { Notes } from '@/models/index.js'; +import { TranslationService } from '@/models/entities/meta.js'; import { ApiError } from '../../error.js'; import { getNote } from '../../common/getters.js'; import define from '../../define.js'; @@ -113,50 +114,101 @@ export default define(meta, paramDef, async (ps, user) => { if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError('NO_SUCH_NOTE'); throw err; }); + const instance = await fetchMeta(); - if (note.text == null) { + if (instance.translationService == null) { return 204; } - const instance = await fetchMeta(); + type Translation = { + sourceLang: string; + text: string; + }; - if (instance.deeplAuthKey == null) { - return 204; // TODO: 良い感じのエラー返す + async function translateDeepL(): Promise { + if (note.text == null || instance.deeplAuthKey == null) { + return 204; // TODO: Return a better error + } + + const sourceLang = ps.sourceLang; + const targetLang = ps.targetLang; + + const params = new URLSearchParams(); + params.append('auth_key', instance.deeplAuthKey); + params.append('text', note.text); + params.append('target_lang', targetLang); + if (sourceLang) params.append('source_lang', sourceLang); + + const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; + + const res = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': config.userAgent, + Accept: 'application/json, */*', + }, + body: params, + // TODO + //timeout: 10000, + agent: getAgentByUrl, + }); + + const json = (await res.json()) as { + translations: { + detected_source_language: string; + text: string; + }[]; + }; + + return { + sourceLang: json.translations[0].detected_source_language, + text: json.translations[0].text, + }; + } + + async function translateLibreTranslate(): Promise { + if (note.text == null || instance.libreTranslateEndpoint == null) { + return 204; + } + + // LibteTranslate only understands 2-letter codes + const source = ps.sourceLang?.toLowerCase().split('-', 1)[0] ?? 'auto'; + const target = ps.targetLang.toLowerCase().split('-', 1)[0]; + const api_key = instance.libreTranslateAuthKey ?? undefined; + const endpoint = instance.libreTranslateEndpoint; + + const params = { + q: note.text, + source, + target, + api_key, + }; + + const res = await fetch(endpoint, { + method: 'POST', + body: JSON.stringify(params), + headers: { 'Content-Type': 'application/json' }, + }); + + const json = (await res.json()) as { + detectedLanguage?: { + confidence: number; + language: string; + }; + translatedText: string; + }; + + return { + sourceLang: (json.detectedLanguage?.language ?? source).toUpperCase(), + text: json.translatedText, + }; } - const sourceLang = ps.sourceLang; - const targetLang = ps.targetLang; - - const params = new URLSearchParams(); - params.append('auth_key', instance.deeplAuthKey); - params.append('text', note.text); - params.append('target_lang', targetLang); - if (sourceLang) params.append('source_lang', sourceLang); - - const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - - const res = await fetch(endpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'User-Agent': config.userAgent, - Accept: 'application/json, */*', - }, - body: params, - // TODO - //timeout: 10000, - agent: getAgentByUrl, - }); - - const json = (await res.json()) as { - translations: { - detected_source_language: string; - text: string; - }[]; - }; - - return { - sourceLang: json.translations[0].detected_source_language, - text: json.translations[0].text, - }; + switch (instance.translationService) { + case TranslationService.DeepL: + return await translateDeepL(); + case TranslationService.LibreTranslate: + return await translateLibreTranslate(); + } }); From cfe0f3ca671339c5c440f99e771f142f67335296 Mon Sep 17 00:00:00 2001 From: Francis Dinh Date: Tue, 1 Nov 2022 01:40:54 -0400 Subject: [PATCH 2/3] client: Add LibreTranslate support This adds a new "Translation Settings" page to the admin interface where the admin can configure the instance's translation settings. The existing settigns for DeepL translation settings will now be located in that page alongside the new LibreTranslate stuff. Also made the translation service settings localizable, which funnily enough was not already the case. --- locales/en-US.yml | 9 ++ packages/client/src/pages/admin/index.vue | 6 ++ packages/client/src/pages/admin/settings.vue | 20 +---- .../src/pages/admin/translation-settings.vue | 86 +++++++++++++++++++ 4 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 packages/client/src/pages/admin/translation-settings.vue diff --git a/locales/en-US.yml b/locales/en-US.yml index 28cabce03..c746e460e 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -849,6 +849,8 @@ misskeyUpdated: "FoundKey has been updated!" whatIsNew: "Show changes" translate: "Translate" translatedFrom: "Translated from {x}" +translationSettings: "Translation Settings" +translationService: "Translation Service" accountDeletionInProgress: "Account deletion is currently in progress." usernameInfo: "A name that identifies your account from others on this server. You\ \ can use the alphabet (a~z, A~Z), digits (0~9) or underscores (_). Usernames cannot\ @@ -1525,3 +1527,10 @@ _services: _github: connected: "GitHub: @{login} connected to FoundKey: @{userName}!" disconnected: "GitHub linkage has been removed." +_translationService: + _deepl: + authKey: "DeepL Auth Key" + pro: "Pro Account" + _libreTranslate: + endpoint: "LibreTranslate API Endpoint" + authKey: "LibreTranslate Auth Key (optional)" diff --git a/packages/client/src/pages/admin/index.vue b/packages/client/src/pages/admin/index.vue index 441b4953a..e45ac1772 100644 --- a/packages/client/src/pages/admin/index.vue +++ b/packages/client/src/pages/admin/index.vue @@ -172,6 +172,11 @@ const menuDef = $computed(() => [{ text: i18n.ts.proxyAccount, to: '/admin/proxy-account', active: props.initialPage === 'proxy-account', + }, { + icon: 'fas fa-language', + text: i18n.ts.translationSettings, + to: '/admin/translation-settings', + active: props.initialPage === 'translation-settings', }], }, { title: i18n.ts.info, @@ -202,6 +207,7 @@ const component = $computed(() => { case 'integrations': return defineAsyncComponent(() => import('./integrations.vue')); case 'instance-block': return defineAsyncComponent(() => import('./instance-block.vue')); case 'proxy-account': return defineAsyncComponent(() => import('./proxy-account.vue')); + case 'translation-settings': return defineAsyncComponent(() => import('./translation-settings.vue')); default: return null; } }); diff --git a/packages/client/src/pages/admin/settings.vue b/packages/client/src/pages/admin/settings.vue index a6c98f49a..daf5a58fb 100644 --- a/packages/client/src/pages/admin/settings.vue +++ b/packages/client/src/pages/admin/settings.vue @@ -128,18 +128,6 @@ - - - - - - - - - - - - @@ -182,8 +170,6 @@ let emailRequiredForSignup: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); -let deeplAuthKey: string = $ref(''); -let deeplIsPro: boolean = $ref(false); async function init(): Promise { const meta = await os.api('admin/meta'); @@ -209,11 +195,9 @@ async function init(): Promise { enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; - deeplAuthKey = meta.deeplAuthKey; - deeplIsPro = meta.deeplIsPro; } -function save() { +function save(): void { os.apiWithDialog('admin/update-meta', { name, description, @@ -237,8 +221,6 @@ function save() { enableServiceWorker, swPublicKey, swPrivateKey, - deeplAuthKey, - deeplIsPro, }).then(() => { fetchInstance(); }); diff --git a/packages/client/src/pages/admin/translation-settings.vue b/packages/client/src/pages/admin/translation-settings.vue new file mode 100644 index 000000000..15085940b --- /dev/null +++ b/packages/client/src/pages/admin/translation-settings.vue @@ -0,0 +1,86 @@ + + + From aefb11959f2c871d58c14b37f69b9661b88262be Mon Sep 17 00:00:00 2001 From: Johann150 Date: Sun, 20 Nov 2022 10:37:50 +0100 Subject: [PATCH 3/3] fix: translator settings on admin/meta endpoint --- .../src/server/api/endpoints/admin/meta.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 5107cfccf..f4ee7c480 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -1,5 +1,6 @@ import config from '@/config/index.js'; import { fetchMeta } from '@/misc/fetch-meta.js'; +import { TranslationService } from '@/models/entities/meta.js'; import { translatorAvailable } from '../../common/translator.js'; import define from '../../define.js'; @@ -270,6 +271,27 @@ export const meta = { type: 'boolean', optional: true, nullable: false, }, + translatorService: { + type: 'string', + enum: [null, ...Object.values(TranslationService)], + optional: false, nullable: true, + }, + deeplAuthKey: { + type: 'string', + optional: true, nullable: true, + }, + deeplIsPro: { + type: 'boolean', + optional: true, nullable: false, + }, + libreTranslateEndpoint: { + type: 'string', + optional: true, nullable: true, + }, + libreTranslateAuthKey: { + type: 'string', + optional: true, nullable: true, + }, }, }, } as const; @@ -318,7 +340,6 @@ export default define(meta, paramDef, async (ps, me) => { enableGithubIntegration: instance.enableGithubIntegration, enableDiscordIntegration: instance.enableDiscordIntegration, enableServiceWorker: instance.enableServiceWorker, - translatorAvailable: translatorAvailable(instance), pinnedPages: instance.pinnedPages, pinnedClipId: instance.pinnedClipId, cacheRemoteFiles: instance.cacheRemoteFiles, @@ -357,7 +378,12 @@ export default define(meta, paramDef, async (ps, me) => { objectStorageUseProxy: instance.objectStorageUseProxy, objectStorageSetPublicRead: instance.objectStorageSetPublicRead, objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle, + + translatorAvailable: translatorAvailable(instance), + translationService: instance.translationService, deeplAuthKey: instance.deeplAuthKey, deeplIsPro: instance.deeplIsPro, + libreTranslateEndpoint: instance.libreTranslateEndpoint, + libreTranslateAuthKey: instance.libreTranslateAuthKey, }; });