Remove HEAD request from fetching link previews (#12028)

It is not really necessary and we need to reduce requests
This commit is contained in:
Eugen Rochko 2019-10-01 04:54:10 +02:00 committed by GitHub
parent b258583d2b
commit 9ba40a6bfd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 16 deletions

View file

@ -39,12 +39,6 @@ class FetchLinkCardService < BaseService
def process_url def process_url
@card ||= PreviewCard.new(url: @url) @card ||= PreviewCard.new(url: @url)
failed = Request.new(:head, @url).perform do |res|
res.code != 405 && res.code != 501 && (res.code != 200 || res.mime_type != 'text/html')
end
return if failed
Request.new(:get, @url).perform do |res| Request.new(:get, @url).perform do |res|
if res.code == 200 && res.mime_type == 'text/html' if res.code == 200 && res.mime_type == 'text/html'
@html = res.body_with_limit @html = res.body_with_limit

View file

@ -4,20 +4,13 @@ RSpec.describe FetchLinkCardService, type: :service do
subject { FetchLinkCardService.new } subject { FetchLinkCardService.new }
before do before do
stub_request(:head, 'http://example.xn--fiqs8s/').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt'))
stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt')) stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt'))
stub_request(:head, 'http://example.com/sjis_with_wrong_charset').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
stub_request(:head, 'http://example.com/日本語').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt')) stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt'))
stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404)
stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
subject.call(status) subject.call(status)
@ -90,11 +83,11 @@ RSpec.describe FetchLinkCardService, type: :service do
let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen? Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') } let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen? Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
it 'parses out URLs' do it 'parses out URLs' do
expect(a_request(:head, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once expect(a_request(:get, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once
end end
it 'ignores URLs to hashtags' do it 'ignores URLs to hashtags' do
expect(a_request(:head, 'https://quitter.se/tag/wannacry')).to_not have_been_made expect(a_request(:get, 'https://quitter.se/tag/wannacry')).to_not have_been_made
end end
end end
end end