From 79b00d294d5098e530640f271d0b30dfb9bde9be Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 11 Mar 2019 13:27:57 +0100 Subject: [PATCH 1/5] Increase DNS timeouts (#10238) --- app/lib/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/request.rb b/app/lib/request.rb index ef4aeaf29..e555ae6a1 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -171,7 +171,7 @@ class Request outer_e = nil Resolv::DNS.open do |dns| - dns.timeouts = 1 + dns.timeouts = 5 addresses = dns.getaddresses(host).take(2) time_slot = 10.0 / addresses.size From 0a2e5abece415ebda6855de09d41db832a75f4cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 11 Mar 2019 13:29:53 +0100 Subject: [PATCH 2/5] Bump active_record_query_trace from 1.6 to 1.6.1 (#10237) Bumps [active_record_query_trace](https://github.com/brunofacca/active-record-query-trace) from 1.6 to 1.6.1. - [Release notes](https://github.com/brunofacca/active-record-query-trace/releases) - [Changelog](https://github.com/brunofacca/active-record-query-trace/blob/master/HISTORY.md) - [Commits](https://github.com/brunofacca/active-record-query-trace/compare/v1.6...v1.6.1) Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b33482d4e..4de4923c6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -43,7 +43,7 @@ GEM activemodel (>= 4.1, < 6) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) - active_record_query_trace (1.6) + active_record_query_trace (1.6.1) activejob (5.2.2) activesupport (= 5.2.2) globalid (>= 0.3.6) From a558ef45d8839a54a2087a34bdb925a528cf2886 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 11 Mar 2019 21:31:16 +0900 Subject: [PATCH 3/5] Bump pkg-config from 1.3.5 to 1.3.6 (#10235) Bumps [pkg-config](https://github.com/ruby-gnome2/pkg-config) from 1.3.5 to 1.3.6. - [Release notes](https://github.com/ruby-gnome2/pkg-config/releases) - [Changelog](https://github.com/ruby-gnome2/pkg-config/blob/master/NEWS) - [Commits](https://github.com/ruby-gnome2/pkg-config/compare/1.3.5...1.3.6) Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4de4923c6..dc2512212 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -400,7 +400,7 @@ GEM pg (1.1.4) pghero (2.2.0) activerecord - pkg-config (1.3.5) + pkg-config (1.3.6) powerpack (0.1.2) premailer (1.11.1) addressable From 67665023c3088eca032ecfe4b55771a90d972300 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 11 Mar 2019 13:45:17 +0100 Subject: [PATCH 4/5] Add `tootctl statuses remove` to sweep unreferenced statuses (#10063) Query by @tateisu Fix #1554 --- lib/cli.rb | 4 +++ lib/mastodon/statuses_cli.rb | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 lib/mastodon/statuses_cli.rb diff --git a/lib/cli.rb b/lib/cli.rb index 6036adfbe..59f0f3076 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -6,6 +6,7 @@ require_relative 'mastodon/emoji_cli' require_relative 'mastodon/accounts_cli' require_relative 'mastodon/feeds_cli' require_relative 'mastodon/settings_cli' +require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' require_relative 'mastodon/version' @@ -30,6 +31,9 @@ module Mastodon desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings' subcommand 'settings', Mastodon::SettingsCLI + desc 'statuses SUBCOMMAND ...ARGS', 'Manage statuses' + subcommand 'statuses', Mastodon::StatusesCLI + desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains' subcommand 'domains', Mastodon::DomainsCLI diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb new file mode 100644 index 000000000..5881ba260 --- /dev/null +++ b/lib/mastodon/statuses_cli.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class StatusesCLI < Thor + include ActionView::Helpers::NumberHelper + + def self.exit_on_failure? + true + end + + option :days, type: :numeric, default: 90 + desc 'remove', 'Remove statuses' + def remove + say('Creating temporary database indices...') + + ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local) + ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id) + ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url) + + max_id = Mastodon::Snowflake.id_at(options[:days].days.ago) + start_at = Time.now.to_f + + say('Beginning removal... This might take a while...') + + Status.remote + .where('id < ?', max_id) + .where(reblog_of_id: nil) # Skip reblogs + .where(in_reply_to_id: nil) # Skip replies + .where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') # Skip statuses that are pinned on profiles + .where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') # Skip statuses that mention local accounts + .where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') # Skip statuses favourited by local accounts + .where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND statuses1.account_id IN (SELECT accounts.id FROM accounts WHERE accounts.domain IS NULL))') # Skip statuses reblogged by local accounts + .where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') # Skip accounts followed by local accounts + .in_batches + .delete_all + + say('Beginning removal of now-orphaned media attachments to free up disk space...') + + Scheduler::MediaCleanupScheduler.new.perform + + say("Done after #{Time.now.to_f - start_at}s", :green) + ensure + say('Removing temporary database indices to restore write performance...') + + ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local) if ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local) + ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id) if ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id) + ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url) if ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url) + end + end +end From c8122e84cc140f49d7bce0540b6e389781c0fdaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 11 Mar 2019 15:29:07 +0100 Subject: [PATCH 5/5] Bump aws-sdk-s3 from 1.30.1 to 1.31.0 (#10236) Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.30.1 to 1.31.0. - [Release notes](https://github.com/aws/aws-sdk-ruby/releases) - [Changelog](https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-ruby/compare/1.30.1...1.31.0) Signed-off-by: dependabot[bot] --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 9f8647ff6..9c00e90c1 100644 --- a/Gemfile +++ b/Gemfile @@ -15,7 +15,7 @@ gem 'makara', '~> 0.4' gem 'pghero', '~> 2.2' gem 'dotenv-rails', '~> 2.7' -gem 'aws-sdk-s3', '~> 1.30', require: false +gem 'aws-sdk-s3', '~> 1.31', require: false gem 'fog-core', '<= 2.1.0' gem 'fog-openstack', '~> 0.3', require: false gem 'paperclip', '~> 6.0' diff --git a/Gemfile.lock b/Gemfile.lock index dc2512212..a94da45d3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,8 +76,8 @@ GEM av (0.9.0) cocaine (~> 0.5.3) aws-eventstream (1.0.1) - aws-partitions (1.131.0) - aws-sdk-core (3.45.0) + aws-partitions (1.143.0) + aws-sdk-core (3.46.2) aws-eventstream (~> 1.0) aws-partitions (~> 1.0) aws-sigv4 (~> 1.0) @@ -85,7 +85,7 @@ GEM aws-sdk-kms (1.13.0) aws-sdk-core (~> 3, >= 3.39.0) aws-sigv4 (~> 1.0) - aws-sdk-s3 (1.30.1) + aws-sdk-s3 (1.31.0) aws-sdk-core (~> 3, >= 3.39.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.0) @@ -659,7 +659,7 @@ DEPENDENCIES active_record_query_trace (~> 1.6) addressable (~> 2.6) annotate (~> 2.7) - aws-sdk-s3 (~> 1.30) + aws-sdk-s3 (~> 1.31) better_errors (~> 2.5) binding_of_caller (~> 0.7) bootsnap (~> 1.4)