Merge remote-tracking branch 'origin/master' into gs-master

Conflicts:
 	app/javascript/styles/mastodon/components.scss
 	app/models/media_attachment.rb
This commit is contained in:
David Yip 2018-03-27 10:26:47 -05:00
commit f61aa8e0f7
No known key found for this signature in database
GPG key ID: 7DA0036508FCC0CC
24 changed files with 151 additions and 54 deletions

View file

@ -61,7 +61,7 @@ module JsonLdHelper
def fetch_resource_without_id_validation(uri) def fetch_resource_without_id_validation(uri)
build_request(uri).perform do |response| build_request(uri).perform do |response|
response.code == 200 ? body_to_json(response.to_s) : nil response.code == 200 ? body_to_json(response.body_with_limit) : nil
end end
end end

View file

@ -39,7 +39,7 @@ export function importFetchedAccounts(accounts) {
pushUnique(normalAccounts, normalizeAccount(account)); pushUnique(normalAccounts, normalizeAccount(account));
if (account.moved) { if (account.moved) {
processAccount(account); processAccount(account.moved);
} }
} }

View file

@ -10,6 +10,10 @@ export function normalizeAccount(account) {
account.display_name_html = emojify(escapeTextContentForBrowser(displayName)); account.display_name_html = emojify(escapeTextContentForBrowser(displayName));
account.note_emojified = emojify(account.note); account.note_emojified = emojify(account.note);
if (account.moved) {
account.moved = account.moved.id;
}
return account; return account;
} }

View file

@ -158,7 +158,7 @@ export default class StatusContent extends React.PureComponent {
<p style={{ marginBottom: hidden && status.get('mentions').isEmpty() ? '0px' : null }}> <p style={{ marginBottom: hidden && status.get('mentions').isEmpty() ? '0px' : null }}>
<span dangerouslySetInnerHTML={spoilerContent} /> <span dangerouslySetInnerHTML={spoilerContent} />
{' '} {' '}
<button tabIndex='0' className='status__content__spoiler-link' onClick={this.handleSpoilerClick}>{toggleText}</button> <button tabIndex='0' className={`status__content__spoiler-link ${hidden ? 'status__content__spoiler-link--show-more' : 'status__content__spoiler-link--show-less'}`} onClick={this.handleSpoilerClick}>{toggleText}</button>
</p> </p>
{mentionsPlaceholder} {mentionsPlaceholder}

View file

@ -1435,14 +1435,19 @@
position: relative; position: relative;
width: 100%; width: 100%;
height: 100%; height: 100%;
display: flex;
align-items: center;
justify-content: center;
&.image-loader--loading { .image-loader__preview-canvas {
display: flex; max-width: $media-modal-media-max-width;
align-content: center; max-height: $media-modal-media-max-height;
background: url('~images/void.png') repeat;
object-fit: contain;
}
.image-loader__preview-canvas { &.image-loader--loading .image-loader__preview-canvas {
filter: blur(2px); filter: blur(2px);
}
} }
&.image-loader--amorphous .image-loader__preview-canvas { &.image-loader--amorphous .image-loader__preview-canvas {
@ -1455,7 +1460,16 @@
width: 100%; width: 100%;
height: 100%; height: 100%;
display: flex; display: flex;
align-content: center; align-items: center;
justify-content: center;
img {
max-width: $media-modal-media-max-width;
max-height: $media-modal-media-max-height;
width: auto;
height: auto;
object-fit: contain;
}
} }
.navigation-bar { .navigation-bar {
@ -3422,27 +3436,6 @@ a.status-card {
width: 100%; width: 100%;
height: 100%; height: 100%;
position: relative; position: relative;
img,
canvas,
video {
max-width: 100%;
/*
put margins on top and bottom of image to avoid the screen coverd by
image.
*/
max-height: 80%;
width: auto;
height: auto;
margin: auto;
}
img,
canvas {
display: block;
background: url('~images/void.png') repeat;
object-fit: contain;
}
} }
.media-modal__closer { .media-modal__closer {

View file

@ -30,3 +30,8 @@ $ui-highlight-color: $classic-highlight-color !default; // Vibrant
// Language codes that uses CJK fonts // Language codes that uses CJK fonts
$cjk-langs: ja, ko, zh-CN, zh-HK, zh-TW; $cjk-langs: ja, ko, zh-CN, zh-HK, zh-TW;
// Variables for components
$media-modal-media-max-width: 100%;
// put margins on top and bottom of image to avoid the screen covered by image.
$media-modal-media-max-height: 80%;

View file

@ -5,6 +5,7 @@ module Mastodon
class NotPermittedError < Error; end class NotPermittedError < Error; end
class ValidationError < Error; end class ValidationError < Error; end
class HostValidationError < ValidationError; end class HostValidationError < ValidationError; end
class LengthValidationError < ValidationError; end
class RaceConditionError < Error; end class RaceConditionError < Error; end
class UnexpectedResponseError < Error class UnexpectedResponseError < Error

View file

@ -18,7 +18,7 @@ class ProviderDiscovery < OEmbed::ProviderDiscovery
else else
Request.new(:get, url).perform do |res| Request.new(:get, url).perform do |res|
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
Nokogiri::HTML(res.to_s) Nokogiri::HTML(res.body_with_limit)
end end
end end

View file

@ -40,7 +40,7 @@ class Request
end end
begin begin
yield response yield response.extend(ClientLimit)
ensure ensure
http_client.close http_client.close
end end
@ -99,6 +99,33 @@ class Request
@http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2) @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
end end
module ClientLimit
def body_with_limit(limit = 1.megabyte)
raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
if charset.nil?
encoding = Encoding::BINARY
else
begin
encoding = Encoding.find(charset)
rescue ArgumentError
encoding = Encoding::BINARY
end
end
contents = String.new(encoding: encoding)
while (chunk = readpartial)
contents << chunk
chunk.clear
raise Mastodon::LengthValidationError if contents.bytesize > limit
end
contents
end
end
class Socket < TCPSocket class Socket < TCPSocket
class << self class << self
def open(host, *args) def open(host, *args)
@ -118,5 +145,5 @@ class Request
end end
end end
private_constant :Socket private_constant :ClientLimit, :Socket
end end

View file

@ -55,7 +55,6 @@ class Account < ApplicationRecord
include AccountHeader include AccountHeader
include AccountInteractions include AccountInteractions
include Attachmentable include Attachmentable
include Remotable
include Paginable include Paginable
MAX_NOTE_LENGTH = 500 MAX_NOTE_LENGTH = 500

View file

@ -2,4 +2,5 @@
class ApplicationRecord < ActiveRecord::Base class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true self.abstract_class = true
include Remotable
end end

View file

@ -4,6 +4,7 @@ module AccountAvatar
extend ActiveSupport::Concern extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 2.megabytes
class_methods do class_methods do
def avatar_styles(file) def avatar_styles(file)
@ -19,7 +20,8 @@ module AccountAvatar
# Avatar upload # Avatar upload
has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
validates_attachment_size :avatar, less_than: 2.megabytes validates_attachment_size :avatar, less_than: LIMIT
remotable_attachment :avatar, LIMIT
end end
def avatar_original_url def avatar_original_url

View file

@ -4,6 +4,7 @@ module AccountHeader
extend ActiveSupport::Concern extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 2.megabytes
class_methods do class_methods do
def header_styles(file) def header_styles(file)
@ -19,7 +20,8 @@ module AccountHeader
# Header upload # Header upload
has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
validates_attachment_size :header, less_than: 2.megabytes validates_attachment_size :header, less_than: LIMIT
remotable_attachment :header, LIMIT
end end
def header_original_url def header_original_url

View file

@ -3,8 +3,8 @@
module Remotable module Remotable
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do class_methods do
attachment_definitions.each_key do |attachment_name| def remotable_attachment(attachment_name, limit)
attribute_name = "#{attachment_name}_remote_url".to_sym attribute_name = "#{attachment_name}_remote_url".to_sym
method_name = "#{attribute_name}=".to_sym method_name = "#{attribute_name}=".to_sym
alt_method_name = "reset_#{attachment_name}!".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym
@ -33,7 +33,7 @@ module Remotable
File.extname(filename) File.extname(filename)
end end
send("#{attachment_name}=", StringIO.new(response.to_s)) send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
send("#{attachment_name}_file_name=", basename + extname) send("#{attachment_name}_file_name=", basename + extname)
self[attribute_name] = url if has_attribute?(attribute_name) self[attribute_name] = url if has_attribute?(attribute_name)

View file

@ -19,6 +19,8 @@
# #
class CustomEmoji < ApplicationRecord class CustomEmoji < ApplicationRecord
LIMIT = 50.kilobytes
SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}' SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
SCAN_RE = /(?<=[^[:alnum:]:]|\n|^) SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
@ -29,14 +31,14 @@ class CustomEmoji < ApplicationRecord
has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } } has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes } validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT }
validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 } validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
scope :local, -> { where(domain: nil) } scope :local, -> { where(domain: nil) }
scope :remote, -> { where.not(domain: nil) } scope :remote, -> { where.not(domain: nil) }
scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) } scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
include Remotable remotable_attachment :image, LIMIT
def local? def local?
domain.nil? domain.nil?

View file

@ -74,6 +74,8 @@ class MediaAttachment < ApplicationRecord
}, },
}.freeze }.freeze
LIMIT = 8.megabytes
belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :account, inverse_of: :media_attachments, optional: true
belongs_to :status, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true
@ -85,7 +87,8 @@ class MediaAttachment < ApplicationRecord
include Remotable include Remotable
validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
validates_attachment_size :file, less_than: 8.megabytes validates_attachment_size :file, less_than: LIMIT
remotable_attachment :file, LIMIT
validates :account, presence: true validates :account, presence: true
validates :description, length: { maximum: 420 }, if: :local? validates :description, length: { maximum: 420 }, if: :local?

View file

@ -26,6 +26,7 @@
class PreviewCard < ApplicationRecord class PreviewCard < ApplicationRecord
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 1.megabytes
self.inheritance_column = false self.inheritance_column = false
@ -36,11 +37,11 @@ class PreviewCard < ApplicationRecord
has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' } has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' }
include Attachmentable include Attachmentable
include Remotable
validates :url, presence: true, uniqueness: true validates :url, presence: true, uniqueness: true
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
validates_attachment_size :image, less_than: 1.megabytes validates_attachment_size :image, less_than: LIMIT
remotable_attachment :image, LIMIT
before_save :extract_dimensions, if: :link? before_save :extract_dimensions, if: :link?

View file

@ -38,13 +38,14 @@ class FetchAtomService < BaseService
return nil if response.code != 200 return nil if response.code != 200
if response.mime_type == 'application/atom+xml' if response.mime_type == 'application/atom+xml'
[@url, { prefetched_body: response.to_s }, :ostatus] [@url, { prefetched_body: response.body_with_limit }, :ostatus]
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
json = body_to_json(response.to_s) body = response.body_with_limit
json = body_to_json(body)
if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present? if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] [json['id'], { prefetched_body: body, id: true }, :activitypub]
elsif supported_context?(json) && json['type'] == 'Note' elsif supported_context?(json) && json['type'] == 'Note'
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] [json['id'], { prefetched_body: body, id: true }, :activitypub]
else else
@unsupported_activity = true @unsupported_activity = true
nil nil
@ -61,7 +62,7 @@ class FetchAtomService < BaseService
end end
def process_html(response) def process_html(response)
page = Nokogiri::HTML(response.to_s) page = Nokogiri::HTML(response.body_with_limit)
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }

View file

@ -45,7 +45,7 @@ class FetchLinkCardService < BaseService
Request.new(:get, @url).perform do |res| Request.new(:get, @url).perform do |res|
if res.code == 200 && res.mime_type == 'text/html' if res.code == 200 && res.mime_type == 'text/html'
@html = res.to_s @html = res.body_with_limit
@html_charset = res.charset @html_charset = res.charset
else else
@html = nil @html = nil

View file

@ -181,7 +181,7 @@ class ResolveAccountService < BaseService
@atom_body = Request.new(:get, atom_url).perform do |response| @atom_body = Request.new(:get, atom_url).perform do |response|
raise Mastodon::UnexpectedResponseError, response unless response.code == 200 raise Mastodon::UnexpectedResponseError, response unless response.code == 200
response.to_s response.body_with_limit
end end
end end

View file

@ -57,7 +57,7 @@ class Pubsubhubbub::ConfirmationWorker
def callback_get_with_params def callback_get_with_params
Request.new(:get, subscription.callback_url, params: callback_params).perform do |response| Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
@callback_response_body = response.body.to_s @callback_response_body = response.body_with_limit
end end
end end

View file

@ -21,6 +21,10 @@
"description": "The secret key base", "description": "The secret key base",
"generator": "secret" "generator": "secret"
}, },
"OTP_SECRET": {
"description": "One-time password secret",
"generator": "secret"
},
"SINGLE_USER_MODE": { "SINGLE_USER_MODE": {
"description": "Should the instance run in single user mode? (Disable registrations, redirect to front page)", "description": "Should the instance run in single user mode? (Disable registrations, redirect to front page)",
"value": "false", "value": "false",

View file

@ -1,6 +1,7 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'rails_helper' require 'rails_helper'
require 'securerandom'
describe Request do describe Request do
subject { Request.new(:get, 'http://example.com') } subject { Request.new(:get, 'http://example.com') }
@ -64,6 +65,12 @@ describe Request do
expect_any_instance_of(HTTP::Client).to receive(:close) expect_any_instance_of(HTTP::Client).to receive(:close)
expect { |block| subject.perform &block }.to yield_control expect { |block| subject.perform &block }.to yield_control
end end
it 'returns response which implements body_with_limit' do
subject.perform do |response|
expect(response).to respond_to :body_with_limit
end
end
end end
context 'with private host' do context 'with private host' do
@ -81,4 +88,46 @@ describe Request do
end end
end end
end end
describe "response's body_with_limit method" do
it 'rejects body more than 1 megabyte by default' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes))
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'accepts body less than 1 megabyte by default' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
expect { subject.perform { |response| response.body_with_limit } }.not_to raise_error
end
it 'rejects body by given size' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
expect { subject.perform { |response| response.body_with_limit(1.kilobyte) } }.to raise_error Mastodon::LengthValidationError
end
it 'rejects too large chunked body' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Transfer-Encoding' => 'chunked' })
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'rejects too large monolithic body' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes })
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'uses binary encoding if Content-Type does not tell encoding' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
end
it 'uses binary encoding if Content-Type tells unknown encoding' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=unknown' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
end
it 'uses encoding specified by Content-Type' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=UTF-8' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::UTF_8
end
end
end end

View file

@ -29,7 +29,10 @@ RSpec.describe Remotable do
context 'Remotable module is included' do context 'Remotable module is included' do
before do before do
class Foo; include Remotable; end class Foo
include Remotable
remotable_attachment :hoge, 1.kilobyte
end
end end
let(:attribute_name) { "#{hoge}_remote_url".to_sym } let(:attribute_name) { "#{hoge}_remote_url".to_sym }