期限切れ/未保存リモートファイルのローカルプロキシ (#5655)

* Media Proxy を実装

* サンプルを追加

* https://github.com/syuilo/misskey/pull/5649#discussion_r359967471 の修正

* https://github.com/syuilo/misskey/pull/5649#discussion_r359967966 の修正

* https://github.com/syuilo/misskey/pull/5649#discussion_r359968219 の修正

* 期限切れ/未保存リモートファイルのローカルプロキシ

* 設定

* 説明

* comment out

* fix

Co-authored-by: 和風ドレッシング <37681609+CookieRamen@users.noreply.github.com>
This commit is contained in:
MeiMei 2019-12-31 17:23:47 +09:00 committed by GitHub
parent 307fc18138
commit b0bb5d8dfc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 151 additions and 12 deletions

View file

@ -142,4 +142,4 @@ autoAdmin: true
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Media Proxy # Media Proxy
#mediaProxy: http://127.0.0.1:3000 #mediaProxy: https://example.com/proxy

View file

@ -1410,7 +1410,9 @@ admin/views/instance.vue:
object-storage-s3-info-here: "こちら" object-storage-s3-info-here: "こちら"
object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。" object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。"
cache-remote-files: "リモートのファイルをキャッシュする" cache-remote-files: "リモートのファイルをキャッシュする"
cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにしておくことをおすすめします。" cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにするか次のリモートファイルのプロキシを有効にすることをおすすめします。"
proxy-remote-files: "リモートのファイルをプロキシする"
proxy-remote-files-desc: "この設定を有効にすると、未保存または保存容量超過で削除されたリモートファイルをローカルでプロキシし、サムネイルも生成するようになります。"
local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量" local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量"
remote-drive-capacity-mb: "リモートユーザーひとりあたりのドライブ容量" remote-drive-capacity-mb: "リモートユーザーひとりあたりのドライブ容量"
mb: "メガバイト単位" mb: "メガバイト単位"

View file

@ -0,0 +1,14 @@
import {MigrationInterface, QueryRunner} from "typeorm";
export class ProxyRemoteFiles1576869585998 implements MigrationInterface {
name = 'ProxyRemoteFiles1576869585998'
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT false`, undefined);
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "proxyRemoteFiles"`, undefined);
}
}

View file

@ -81,6 +81,7 @@
</section> </section>
<section> <section>
<ui-switch v-model="cacheRemoteFiles">{{ $t('cache-remote-files') }}<template #desc>{{ $t('cache-remote-files-desc') }}</template></ui-switch> <ui-switch v-model="cacheRemoteFiles">{{ $t('cache-remote-files') }}<template #desc>{{ $t('cache-remote-files-desc') }}</template></ui-switch>
<ui-switch v-model="proxyRemoteFiles">{{ $t('proxy-remote-files') }}<template #desc>{{ $t('proxy-remote-files-desc') }}</template></ui-switch>
</section> </section>
<section class="fit-top fit-bottom"> <section class="fit-top fit-bottom">
<ui-input v-model="localDriveCapacityMb" type="number">{{ $t('local-drive-capacity-mb') }}<template #suffix>MB</template><template #desc>{{ $t('mb') }}</template></ui-input> <ui-input v-model="localDriveCapacityMb" type="number">{{ $t('local-drive-capacity-mb') }}<template #suffix>MB</template><template #desc>{{ $t('mb') }}</template></ui-input>
@ -275,6 +276,7 @@ export default Vue.extend({
description: null, description: null,
languages: null, languages: null,
cacheRemoteFiles: false, cacheRemoteFiles: false,
proxyRemoteFiles: false,
localDriveCapacityMb: null, localDriveCapacityMb: null,
remoteDriveCapacityMb: null, remoteDriveCapacityMb: null,
maxNoteTextLength: null, maxNoteTextLength: null,
@ -339,6 +341,7 @@ export default Vue.extend({
this.description = meta.description; this.description = meta.description;
this.languages = meta.langs.join(' '); this.languages = meta.langs.join(' ');
this.cacheRemoteFiles = meta.cacheRemoteFiles; this.cacheRemoteFiles = meta.cacheRemoteFiles;
this.proxyRemoteFiles = meta.proxyRemoteFiles;
this.localDriveCapacityMb = meta.driveCapacityPerLocalUserMb; this.localDriveCapacityMb = meta.driveCapacityPerLocalUserMb;
this.remoteDriveCapacityMb = meta.driveCapacityPerRemoteUserMb; this.remoteDriveCapacityMb = meta.driveCapacityPerRemoteUserMb;
this.maxNoteTextLength = meta.maxNoteTextLength; this.maxNoteTextLength = meta.maxNoteTextLength;
@ -463,6 +466,7 @@ export default Vue.extend({
description: this.description, description: this.description,
langs: this.languages ? this.languages.split(' ') : [], langs: this.languages ? this.languages.split(' ') : [],
cacheRemoteFiles: this.cacheRemoteFiles, cacheRemoteFiles: this.cacheRemoteFiles,
proxyRemoteFiles: this.proxyRemoteFiles,
localDriveCapacityMb: parseInt(this.localDriveCapacityMb, 10), localDriveCapacityMb: parseInt(this.localDriveCapacityMb, 10),
remoteDriveCapacityMb: parseInt(this.remoteDriveCapacityMb, 10), remoteDriveCapacityMb: parseInt(this.remoteDriveCapacityMb, 10),
maxNoteTextLength: parseInt(this.maxNoteTextLength, 10), maxNoteTextLength: parseInt(this.maxNoteTextLength, 10),

View file

@ -115,6 +115,11 @@ export class Meta {
}) })
public cacheRemoteFiles: boolean; public cacheRemoteFiles: boolean;
@Column('boolean', {
default: false,
})
public proxyRemoteFiles: boolean;
@Column('varchar', { @Column('varchar', {
length: 128, length: 128,
nullable: true nullable: true

View file

@ -7,6 +7,9 @@ import { ensure } from '../../prelude/ensure';
import { awaitAll } from '../../prelude/await-all'; import { awaitAll } from '../../prelude/await-all';
import { SchemaType } from '../../misc/schema'; import { SchemaType } from '../../misc/schema';
import config from '../../config'; import config from '../../config';
import { query, appendQuery } from '../../prelude/url';
import { Meta } from '../entities/meta';
import { fetchMeta } from '../../misc/fetch-meta';
export type PackedDriveFile = SchemaType<typeof packedDriveFileSchema>; export type PackedDriveFile = SchemaType<typeof packedDriveFileSchema>;
@ -22,12 +25,39 @@ export class DriveFileRepository extends Repository<DriveFile> {
); );
} }
public getPublicUrl(file: DriveFile, thumbnail = false): string | null { public getPublicUrl(file: DriveFile, thumbnail = false, meta?: Meta): string | null {
let url = thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url); // リモートかつメディアプロキシ
if (file.src !== null && file.userHost !== null && config.mediaProxy !== null) { if (file.uri != null && file.userHost != null && config.mediaProxy != null) {
url = `${config.mediaProxy}/${thumbnail ? 'thumbnail' : ''}?url=${file.src}`; return appendQuery(config.mediaProxy, query({
url: file.uri,
thumbnail: thumbnail ? '1' : undefined
}));
} }
return url;
// リモートかつ期限切れはローカルプロキシを試みる
if (file.uri != null && file.isLink && meta && meta.proxyRemoteFiles) {
const key = thumbnail ? file.thumbnailAccessKey : file.webpublicAccessKey;
if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外
let ext = '';
if (file.name) {
[ext] = (file.name.match(/\.(\w+)$/) || ['']);
}
if (ext === '') {
if (file.type === 'image/jpeg') ext = '.jpg';
if (file.type === 'image/png') ext = '.png';
if (file.type === 'image/webp') ext = '.webp';
if (file.type === 'image/apng') ext = '.apng';
if (file.type === 'image/vnd.mozilla.apng') ext = '.apng';
}
return `/files/${key}/${key}${ext}`;
}
}
return thumbnail ? (file.thumbnailUrl || file.webpublicUrl || null) : (file.webpublicUrl || file.url);
} }
public async clacDriveUsageOf(user: User['id'] | User): Promise<number> { public async clacDriveUsageOf(user: User['id'] | User): Promise<number> {
@ -87,6 +117,8 @@ export class DriveFileRepository extends Repository<DriveFile> {
const file = typeof src === 'object' ? src : await this.findOne(src).then(ensure); const file = typeof src === 'object' ? src : await this.findOne(src).then(ensure);
const meta = await fetchMeta();
return await awaitAll({ return await awaitAll({
id: file.id, id: file.id,
createdAt: file.createdAt.toISOString(), createdAt: file.createdAt.toISOString(),
@ -96,8 +128,8 @@ export class DriveFileRepository extends Repository<DriveFile> {
size: file.size, size: file.size,
isSensitive: file.isSensitive, isSensitive: file.isSensitive,
properties: file.properties, properties: file.properties,
url: opts.self ? file.url : this.getPublicUrl(file, false), url: opts.self ? file.url : this.getPublicUrl(file, false, meta),
thumbnailUrl: this.getPublicUrl(file, true), thumbnailUrl: this.getPublicUrl(file, true, meta),
folderId: file.folderId, folderId: file.folderId,
folder: opts.detail && file.folderId ? DriveFolders.pack(file.folderId, { folder: opts.detail && file.folderId ? DriveFolders.pack(file.folderId, {
detail: true detail: true

View file

@ -151,6 +151,13 @@ export const meta = {
} }
}, },
proxyRemoteFiles: {
validator: $.optional.bool,
desc: {
'ja-JP': 'ローカルにないリモートのファイルをプロキシするか否か'
}
},
enableRecaptcha: { enableRecaptcha: {
validator: $.optional.bool, validator: $.optional.bool,
desc: { desc: {
@ -478,6 +485,10 @@ export default define(meta, async (ps, me) => {
set.cacheRemoteFiles = ps.cacheRemoteFiles; set.cacheRemoteFiles = ps.cacheRemoteFiles;
} }
if (ps.proxyRemoteFiles !== undefined) {
set.proxyRemoteFiles = ps.proxyRemoteFiles;
}
if (ps.enableRecaptcha !== undefined) { if (ps.enableRecaptcha !== undefined) {
set.enableRecaptcha = ps.enableRecaptcha; set.enableRecaptcha = ps.enableRecaptcha;
} }

View file

@ -143,6 +143,7 @@ export default define(meta, async (ps, me) => {
driveCapacityPerLocalUserMb: instance.localDriveCapacityMb, driveCapacityPerLocalUserMb: instance.localDriveCapacityMb,
driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb, driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb,
cacheRemoteFiles: instance.cacheRemoteFiles, cacheRemoteFiles: instance.cacheRemoteFiles,
proxyRemoteFiles: instance.proxyRemoteFiles,
enableRecaptcha: instance.enableRecaptcha, enableRecaptcha: instance.enableRecaptcha,
recaptchaSiteKey: instance.recaptchaSiteKey, recaptchaSiteKey: instance.recaptchaSiteKey,
swPublickey: instance.swPublicKey, swPublickey: instance.swPublicKey,

View file

@ -1,10 +1,16 @@
import * as Koa from 'koa'; import * as Koa from 'koa';
import * as send from 'koa-send'; import * as send from 'koa-send';
import * as rename from 'rename'; import * as rename from 'rename';
import * as tmp from 'tmp';
import * as fs from 'fs';
import { serverLogger } from '..'; import { serverLogger } from '..';
import { contentDisposition } from '../../misc/content-disposition'; import { contentDisposition } from '../../misc/content-disposition';
import { DriveFiles } from '../../models'; import { DriveFiles } from '../../models';
import { InternalStorage } from '../../services/drive/internal-storage'; import { InternalStorage } from '../../services/drive/internal-storage';
import { downloadUrl } from '../../misc/donwload-url';
import { detectMine } from '../../misc/detect-mine';
import { convertToJpeg, convertToPng, convertToGif, convertToApng } from '../../services/drive/image-processor';
import { GenerateVideoThumbnail } from '../../services/drive/generate-video-thumbnail';
const assets = `${__dirname}/../../server/file/assets/`; const assets = `${__dirname}/../../server/file/assets/`;
@ -31,15 +37,70 @@ export default async function(ctx: Koa.Context) {
return; return;
} }
const isThumbnail = file.thumbnailAccessKey === key;
const isWebpublic = file.webpublicAccessKey === key;
if (!file.storedInternal) { if (!file.storedInternal) {
if (file.isLink && file.uri) { // 期限切れリモートファイル
const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
tmp.file((e, path, fd, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
});
});
try {
await downloadUrl(file.uri, path);
const [type, ext] = await detectMine(path);
const convertFile = async () => {
if (isThumbnail) {
if (['image/jpeg', 'image/webp'].includes(type)) {
return await convertToJpeg(path, 498, 280);
} else if (['image/png'].includes(type)) {
return await convertToPng(path, 498, 280);
} else if (['image/gif'].includes(type)) {
return await convertToGif(path);
} else if (['image/apng', 'image/vnd.mozilla.apng'].includes(type)) {
return await convertToApng(path);
} else if (type.startsWith('video/')) {
return await GenerateVideoThumbnail(path);
}
}
return {
data: fs.readFileSync(path),
ext,
type,
};
};
const image = await convertFile();
ctx.body = image.data;
ctx.set('Content-Type', file.type);
ctx.set('Cache-Control', 'max-age=31536000, immutable');
} catch (e) {
serverLogger.error(e);
if (typeof e == 'number' && e >= 400 && e < 500) {
ctx.status = e;
ctx.set('Cache-Control', 'max-age=86400');
} else {
ctx.status = 500;
ctx.set('Cache-Control', 'max-age=300');
}
} finally {
cleanup();
}
return;
}
ctx.status = 204; ctx.status = 204;
ctx.set('Cache-Control', 'max-age=86400'); ctx.set('Cache-Control', 'max-age=86400');
return; return;
} }
const isThumbnail = file.thumbnailAccessKey === key;
const isWebpublic = file.webpublicAccessKey === key;
if (isThumbnail) { if (isThumbnail) {
ctx.body = InternalStorage.read(key); ctx.body = InternalStorage.read(key);
ctx.set('Content-Type', 'image/jpeg'); ctx.set('Content-Type', 'image/jpeg');

View file

@ -424,6 +424,10 @@ export default async function(
file.url = url; file.url = url;
file.thumbnailUrl = url; file.thumbnailUrl = url;
file.webpublicUrl = url; file.webpublicUrl = url;
// ローカルプロキシ用
file.accessKey = uuid();
file.thumbnailAccessKey = 'thumbnail-' + uuid();
file.webpublicAccessKey = 'webpublic-' + uuid();
} }
} }

View file

@ -5,6 +5,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '../chart';
import { createDeleteObjectStorageFileJob } from '../../queue'; import { createDeleteObjectStorageFileJob } from '../../queue';
import { fetchMeta } from '../../misc/fetch-meta'; import { fetchMeta } from '../../misc/fetch-meta';
import { getS3 } from './s3'; import { getS3 } from './s3';
import { v4 as uuid } from 'uuid';
export async function deleteFile(file: DriveFile, isExpired = false) { export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) { if (file.storedInternal) {
@ -71,6 +72,10 @@ function postProcess(file: DriveFile, isExpired = false) {
thumbnailUrl: file.uri, thumbnailUrl: file.uri,
webpublicUrl: file.uri, webpublicUrl: file.uri,
size: 0, size: 0,
// ローカルプロキシ用
accessKey: uuid(),
thumbnailAccessKey: 'thumbnail-' + uuid(),
webpublicAccessKey: 'webpublic-' + uuid(),
}); });
} else { } else {
DriveFiles.delete(file.id); DriveFiles.delete(file.id);