806 lines
34 KiB
Ruby
806 lines
34 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'tty-prompt'
|
|
require_relative '../../config/boot'
|
|
require_relative '../../config/environment'
|
|
require_relative 'cli_helper'
|
|
|
|
module Mastodon
|
|
class MaintenanceCLI < Thor
|
|
include CLIHelper
|
|
|
|
def self.exit_on_failure?
|
|
true
|
|
end
|
|
|
|
MIN_SUPPORTED_VERSION = 2019_10_01_213028
|
|
MAX_SUPPORTED_VERSION = 2021_05_26_193025
|
|
|
|
# Stubs to enjoy ActiveRecord queries while not depending on a particular
|
|
# version of the code/database
|
|
|
|
class Status < ApplicationRecord; end
|
|
class StatusPin < ApplicationRecord; end
|
|
class Poll < ApplicationRecord; end
|
|
class Report < ApplicationRecord; end
|
|
class Tombstone < ApplicationRecord; end
|
|
class Favourite < ApplicationRecord; end
|
|
class Follow < ApplicationRecord; end
|
|
class FollowRequest < ApplicationRecord; end
|
|
class Block < ApplicationRecord; end
|
|
class Mute < ApplicationRecord; end
|
|
class AccountIdentityProof < ApplicationRecord; end
|
|
class AccountModerationNote < ApplicationRecord; end
|
|
class AccountPin < ApplicationRecord; end
|
|
class ListAccount < ApplicationRecord; end
|
|
class PollVote < ApplicationRecord; end
|
|
class Mention < ApplicationRecord; end
|
|
class AccountDomainBlock < ApplicationRecord; end
|
|
class AnnouncementReaction < ApplicationRecord; end
|
|
class FeaturedTag < ApplicationRecord; end
|
|
class CustomEmoji < ApplicationRecord; end
|
|
class CustomEmojiCategory < ApplicationRecord; end
|
|
class Bookmark < ApplicationRecord; end
|
|
class WebauthnCredential < ApplicationRecord; end
|
|
class FollowRecommendationSuppression < ApplicationRecord; end
|
|
class CanonicalEmailBlock < ApplicationRecord; end
|
|
|
|
class PreviewCard < ApplicationRecord
|
|
self.inheritance_column = false
|
|
end
|
|
|
|
class MediaAttachment < ApplicationRecord
|
|
self.inheritance_column = nil
|
|
end
|
|
|
|
class AccountStat < ApplicationRecord
|
|
belongs_to :account, inverse_of: :account_stat
|
|
end
|
|
|
|
# Dummy class, to make migration possible across version changes
|
|
class Account < ApplicationRecord
|
|
has_one :user, inverse_of: :account
|
|
has_one :account_stat, inverse_of: :account
|
|
|
|
scope :local, -> { where(domain: nil) }
|
|
|
|
def local?
|
|
domain.nil?
|
|
end
|
|
|
|
def acct
|
|
local? ? username : "#{username}@#{domain}"
|
|
end
|
|
|
|
# This is a duplicate of the AccountMerging concern because we need it to
|
|
# be independent from code version.
|
|
def merge_with!(other_account)
|
|
# Since it's the same remote resource, the remote resource likely
|
|
# already believes we are following/blocking, so it's safe to
|
|
# re-attribute the relationships too. However, during the presence
|
|
# of the index bug users could have *also* followed the reference
|
|
# account already, therefore mass update will not work and we need
|
|
# to check for (and skip past) uniqueness errors
|
|
|
|
owned_classes = [
|
|
Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
|
|
Follow, FollowRequest, Block, Mute, AccountIdentityProof,
|
|
AccountModerationNote, AccountPin, AccountStat, ListAccount,
|
|
PollVote, Mention
|
|
]
|
|
owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests)
|
|
owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
|
|
owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
|
|
|
|
owned_classes.each do |klass|
|
|
klass.where(account_id: other_account.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:account_id, id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
|
|
target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
|
|
target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
|
|
|
|
target_classes.each do |klass|
|
|
klass.where(target_account_id: other_account.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:target_account_id, id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
|
|
if ActiveRecord::Base.connection.table_exists?(:canonical_email_blocks)
|
|
CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
|
|
record.update_attribute(:reference_account_id, id)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
class User < ApplicationRecord
|
|
belongs_to :account, inverse_of: :user
|
|
end
|
|
|
|
desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes'
|
|
long_desc <<~LONG_DESC
|
|
Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes.
|
|
|
|
This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes
|
|
|
|
Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
|
|
LONG_DESC
|
|
def fix_duplicates
|
|
@prompt = TTY::Prompt.new
|
|
|
|
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
|
|
@prompt.warn 'Your version of the database schema is too old and is not supported by this script.'
|
|
@prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.'
|
|
exit(1)
|
|
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
|
|
@prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.'
|
|
exit(1) unless @prompt.yes?('Continue anyway?')
|
|
end
|
|
|
|
@prompt.warn 'This task will take a long time to run and is potentially destructive.'
|
|
@prompt.warn 'Please make sure to stop Mastodon and have a backup.'
|
|
exit(1) unless @prompt.yes?('Continue?')
|
|
|
|
deduplicate_users!
|
|
deduplicate_account_domain_blocks!
|
|
deduplicate_account_identity_proofs!
|
|
deduplicate_announcement_reactions!
|
|
deduplicate_conversations!
|
|
deduplicate_custom_emojis!
|
|
deduplicate_custom_emoji_categories!
|
|
deduplicate_domain_allows!
|
|
deduplicate_domain_blocks!
|
|
deduplicate_unavailable_domains!
|
|
deduplicate_email_domain_blocks!
|
|
deduplicate_media_attachments!
|
|
deduplicate_preview_cards!
|
|
deduplicate_statuses!
|
|
deduplicate_accounts!
|
|
deduplicate_tags!
|
|
deduplicate_webauthn_credentials!
|
|
|
|
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
|
|
Rails.cache.clear
|
|
|
|
@prompt.say 'Finished!'
|
|
end
|
|
|
|
MEDIA_TYPES = %w(accounts custom_emojis media_attachments).freeze
|
|
|
|
option :domain, type: :string, required: true, desc: 'Target server domain'
|
|
option :only, type: :array, enum: MEDIA_TYPES, desc: 'Only process these media types'
|
|
option :force, type: :boolean, desc: 'Skip new remote_url check'
|
|
desc 'fix-remote-url FROM TO', 'Fix remote_url of existing media'
|
|
long_desc <<-DESC
|
|
Fix the remote_url of the existing media when the remote media server is changed.
|
|
|
|
It will automatically check if the media can be obtained from the changed URL. To avoid this, specify the --force option.
|
|
DESC
|
|
def fix_remote_url(from, to)
|
|
@prompt = TTY::Prompt.new
|
|
|
|
@from = from
|
|
@to = to
|
|
@domain = options[:domain]
|
|
@types = options[:only] || MEDIA_TYPES
|
|
@force = options[:force]
|
|
|
|
@prompt.say "Start fixing the remote_url of #{@types.join ', '} on the remote server \"#{@domain}\" from \"#{@from}\" to \"#{@to}\"."
|
|
exit(1) unless @prompt.yes?('Continue?')
|
|
|
|
fix_accounts_remote_url!
|
|
fix_custom_emojis_remote_url!
|
|
fix_media_attachments_remote_url!
|
|
|
|
Rails.cache.clear
|
|
|
|
@prompt.say 'Finished!'
|
|
end
|
|
|
|
private
|
|
|
|
def fix_accounts_remote_url!
|
|
return unless @types.include? 'accounts'
|
|
|
|
@prompt.say 'Fixing accounts…'
|
|
|
|
unless @force
|
|
accounts_avatar_remote_url = ::Account.where(domain: @domain).where('avatar_remote_url like ?', "#{@from}%").reorder(avatar_updated_at: :desc).first&.avatar_remote_url&.sub(/^#{@from}/, @to)
|
|
|
|
return @prompt.warn "There is no corresponding remote avatar image, so skip it." unless accounts_avatar_remote_url
|
|
return @prompt.error "The remote avatar image cannot be reached with the changed URL." unless reachable_url?(accounts_avatar_remote_url)
|
|
end
|
|
|
|
fixed_avatars = ActiveRecord::Base.connection.exec_update(<<-SQL.squish, 'SQL', [[nil, @domain], [nil, @from], [nil, @to], [nil, "#{@from}%"]])
|
|
WITH new AS (
|
|
SELECT
|
|
id,
|
|
replace(avatar_remote_url, $2, $3) AS avatar_remote_url
|
|
FROM
|
|
accounts c
|
|
WHERE
|
|
domain = $1
|
|
AND avatar_remote_url LIKE $4)
|
|
UPDATE
|
|
accounts
|
|
SET
|
|
avatar_remote_url = new.avatar_remote_url
|
|
FROM
|
|
new
|
|
WHERE
|
|
accounts.id = new.id
|
|
SQL
|
|
fixed_headers = ActiveRecord::Base.connection.exec_update(<<-SQL.squish, 'SQL', [[nil, @domain], [nil, @from], [nil, @to], [nil, "#{@from}%"]])
|
|
WITH new AS (
|
|
SELECT
|
|
id,
|
|
replace(header_remote_url, $2, $3) AS header_remote_url
|
|
FROM
|
|
accounts c
|
|
WHERE
|
|
domain = $1
|
|
AND header_remote_url LIKE $4)
|
|
UPDATE
|
|
accounts
|
|
SET
|
|
header_remote_url = new.header_remote_url
|
|
FROM
|
|
new
|
|
WHERE
|
|
accounts.id = new.id
|
|
SQL
|
|
@prompt.ok "Fixed #{fixed_avatars} avatars, #{fixed_headers} headers"
|
|
end
|
|
|
|
def fix_custom_emojis_remote_url!
|
|
return unless @types.include? 'custom_emojis'
|
|
|
|
@prompt.say 'Fixing custon emojis…'
|
|
|
|
unless @force
|
|
custom_emojis_image_remote_url = ::CustomEmoji.where(domain: @domain).where('image_remote_url like ?', "#{@from}%").reorder(image_updated_at: :desc).first&.image_remote_url&.sub(/^#{@from}/, @to)
|
|
|
|
return @prompt.warn "There is no corresponding remote custom emoji, so skip it." unless custom_emojis_image_remote_url
|
|
return @prompt.error "The remote custom emoji cannot be reached with the modified URL." unless reachable_url?(custom_emojis_image_remote_url)
|
|
end
|
|
|
|
fixed_emojis = ActiveRecord::Base.connection.exec_update(<<-SQL.squish, 'SQL', [[nil, @domain], [nil, @from], [nil, @to], [nil, "#{@from}%"]])
|
|
WITH new AS (
|
|
SELECT
|
|
id,
|
|
replace(image_remote_url, $2, $3) AS image_remote_url
|
|
FROM
|
|
custom_emojis c
|
|
WHERE
|
|
domain = $1
|
|
AND image_remote_url LIKE $4)
|
|
UPDATE
|
|
custom_emojis
|
|
SET
|
|
image_remote_url = new.image_remote_url
|
|
FROM
|
|
new
|
|
WHERE
|
|
custom_emojis.id = new.id
|
|
SQL
|
|
@prompt.ok "Fixed #{fixed_emojis} custom emojis"
|
|
end
|
|
|
|
def fix_media_attachments_remote_url!
|
|
return unless @types.include? 'media_attachments'
|
|
|
|
@prompt.say 'Fixing media attachments…'
|
|
|
|
unless @force
|
|
attatchments_remote_url = ::MediaAttachment.joins(status: :account).where(statuses: { accounts: { domain: @domain } }).where('remote_url like ?', "#{@from}%").reorder(file_updated_at: :desc).first&.remote_url&.sub(/^#{@from}/, @to)
|
|
|
|
return @prompt.warn "There is no corresponding remote media attachment, so skip it." unless attatchments_remote_url
|
|
return @prompt.error "The remote media attachment cannot be reached with the modified URL." unless reachable_url?(attatchments_remote_url)
|
|
end
|
|
|
|
fixed_attachments = ActiveRecord::Base.connection.exec_update(<<-SQL.squish, 'SQL', [[nil, @domain], [nil, @from], [nil, @to], [nil, "#{@from}%"]])
|
|
WITH new AS (
|
|
SELECT
|
|
m.id,
|
|
replace(m.remote_url, $2, $3) AS remote_url,
|
|
replace(m.thumbnail_remote_url, $2, $3) AS thumbnail_remote_url
|
|
FROM
|
|
media_attachments m
|
|
INNER JOIN statuses s ON m.status_id = s.id
|
|
AND s.deleted_at IS NULL
|
|
INNER JOIN accounts a ON s.account_id = a.id
|
|
WHERE
|
|
a.domain = $1
|
|
AND m.remote_url LIKE $4)
|
|
UPDATE
|
|
media_attachments
|
|
SET
|
|
remote_url = new.remote_url,
|
|
thumbnail_remote_url = new.thumbnail_remote_url
|
|
FROM
|
|
new
|
|
WHERE
|
|
media_attachments.id = new.id;
|
|
SQL
|
|
@prompt.ok "Fixed #{fixed_attachments} media attachments"
|
|
end
|
|
|
|
def reachable_url?(url)
|
|
return false if url.blank?
|
|
|
|
code = 0
|
|
code = Request.new(:head, url).perform(&:code)
|
|
(200...300).cover?(code)
|
|
rescue HTTP::ConnectionError
|
|
false
|
|
end
|
|
|
|
def deduplicate_accounts!
|
|
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
|
|
|
|
@prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
|
|
|
|
find_duplicate_accounts.each do |row|
|
|
accounts = Account.where(id: row['ids'].split(',')).to_a
|
|
|
|
if accounts.first.local?
|
|
deduplicate_local_accounts!(accounts)
|
|
else
|
|
deduplicate_remote_accounts!(accounts)
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
|
|
if ActiveRecord::Migrator.current_version < 20200620164023
|
|
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
|
|
else
|
|
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
|
|
end
|
|
|
|
@prompt.say 'Reindexing textual indexes on accounts…'
|
|
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
|
|
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
|
|
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
|
|
end
|
|
|
|
def deduplicate_users!
|
|
remove_index_if_exists!(:users, 'index_users_on_confirmation_token')
|
|
remove_index_if_exists!(:users, 'index_users_on_email')
|
|
remove_index_if_exists!(:users, 'index_users_on_remember_token')
|
|
remove_index_if_exists!(:users, 'index_users_on_reset_password_token')
|
|
|
|
@prompt.say 'Deduplicating user records…'
|
|
|
|
# Deduplicating email
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
|
|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
|
|
ref_user = users.shift
|
|
@prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
|
|
@prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
|
|
@prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'
|
|
|
|
i = 0
|
|
users.each do |user|
|
|
user.update!(email: "#{i} " + user.email)
|
|
end
|
|
end
|
|
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
|
|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
|
|
@prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
|
|
|
|
users.each do |user|
|
|
user.update!(confirmation_token: nil)
|
|
end
|
|
end
|
|
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
|
|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
|
|
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
|
|
|
|
users.each do |user|
|
|
user.update!(remember_token: nil)
|
|
end
|
|
end
|
|
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
|
|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
|
|
@prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
|
|
|
|
users.each do |user|
|
|
user.update!(reset_password_token: nil)
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring users indexes…'
|
|
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
|
|
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
|
|
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true
|
|
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
|
|
end
|
|
|
|
def deduplicate_account_domain_blocks!
|
|
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
|
|
|
|
@prompt.say 'Removing duplicate account domain blocks…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
|
|
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
|
|
end
|
|
|
|
@prompt.say 'Restoring account domain blocks indexes…'
|
|
ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_account_identity_proofs!
|
|
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
|
|
|
|
@prompt.say 'Removing duplicate account identity proofs…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
|
|
AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring account identity proofs indexes…'
|
|
ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
|
|
end
|
|
|
|
def deduplicate_announcement_reactions!
|
|
return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions)
|
|
|
|
remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
|
|
|
|
@prompt.say 'Removing duplicate account identity proofs…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
|
|
AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring announcement_reactions indexes…'
|
|
ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
|
|
end
|
|
|
|
def deduplicate_conversations!
|
|
remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
|
|
|
|
@prompt.say 'Deduplicating conversations…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
|
|
conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
|
|
|
ref_conversation = conversations.shift
|
|
|
|
conversations.each do |other|
|
|
merge_conversations!(ref_conversation, other)
|
|
other.destroy
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring conversations indexes…'
|
|
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
|
|
end
|
|
|
|
def deduplicate_custom_emojis!
|
|
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
|
|
|
|
@prompt.say 'Deduplicating custom_emojis…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
|
|
emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
|
|
|
ref_emoji = emojis.shift
|
|
|
|
emojis.each do |other|
|
|
merge_custom_emojis!(ref_emoji, other)
|
|
other.destroy
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring custom_emojis indexes…'
|
|
ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_custom_emoji_categories!
|
|
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
|
|
|
|
@prompt.say 'Deduplicating custom_emoji_categories…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
|
|
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
|
|
|
ref_category = categories.shift
|
|
|
|
categories.each do |other|
|
|
merge_custom_emoji_categories!(ref_category, other)
|
|
other.destroy
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring custom_emoji_categories indexes…'
|
|
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
|
|
end
|
|
|
|
def deduplicate_domain_allows!
|
|
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
|
|
|
|
@prompt.say 'Deduplicating domain_allows…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
|
|
DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring domain_allows indexes…'
|
|
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_domain_blocks!
|
|
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
|
|
|
|
@prompt.say 'Deduplicating domain_allows…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
|
|
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
|
|
|
|
reject_media = domain_blocks.any?(&:reject_media?)
|
|
reject_reports = domain_blocks.any?(&:reject_reports?)
|
|
|
|
reference_block = domain_blocks.shift
|
|
|
|
private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence }
|
|
public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence }
|
|
|
|
reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment)
|
|
|
|
domain_blocks.each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring domain_blocks indexes…'
|
|
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_unavailable_domains!
|
|
return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains)
|
|
|
|
remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
|
|
|
|
@prompt.say 'Deduplicating unavailable_domains…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
|
|
UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring domain_allows indexes…'
|
|
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_email_domain_blocks!
|
|
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
|
|
|
|
@prompt.say 'Deduplicating email_domain_blocks…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
|
|
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
|
|
domain_blocks.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring email_domain_blocks indexes…'
|
|
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
|
|
end
|
|
|
|
def deduplicate_media_attachments!
|
|
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
|
|
|
|
@prompt.say 'Deduplicating media_attachments…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
|
|
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
|
|
end
|
|
|
|
@prompt.say 'Restoring media_attachments indexes…'
|
|
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
|
|
end
|
|
|
|
def deduplicate_preview_cards!
|
|
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
|
|
|
|
@prompt.say 'Deduplicating preview_cards…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
|
|
PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring preview_cards indexes…'
|
|
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
|
|
end
|
|
|
|
def deduplicate_statuses!
|
|
remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
|
|
|
|
@prompt.say 'Deduplicating statuses…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
|
|
statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
|
|
ref_status = statuses.shift
|
|
statuses.each do |status|
|
|
merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id
|
|
status.destroy
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring statuses indexes…'
|
|
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
|
|
end
|
|
|
|
def deduplicate_tags!
|
|
remove_index_if_exists!(:tags, 'index_tags_on_name_lower')
|
|
|
|
@prompt.say 'Deduplicating tags…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
|
|
tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
|
|
ref_tag = tags.shift
|
|
tags.each do |tag|
|
|
merge_tags!(ref_tag, tag)
|
|
tag.destroy
|
|
end
|
|
end
|
|
|
|
@prompt.say 'Restoring tags indexes…'
|
|
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
|
|
|
|
if ActiveRecord::Base.connection.indexes(:tags).any? { |i| i.name == 'index_tags_on_name_lower_btree' }
|
|
@prompt.say 'Reindexing textual indexes on tags…'
|
|
ActiveRecord::Base.connection.execute('REINDEX INDEX index_tags_on_name_lower_btree;')
|
|
end
|
|
end
|
|
|
|
def deduplicate_webauthn_credentials!
|
|
return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials)
|
|
|
|
remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
|
|
|
|
@prompt.say 'Deduplicating webauthn_credentials…'
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
|
|
WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
|
end
|
|
|
|
@prompt.say 'Restoring webauthn_credentials indexes…'
|
|
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
|
|
end
|
|
|
|
def deduplicate_local_accounts!(accounts)
|
|
accounts = accounts.sort_by(&:id).reverse
|
|
|
|
@prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
|
|
@prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.'
|
|
|
|
accounts.each_with_index do |account, idx|
|
|
@prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A']
|
|
end
|
|
|
|
@prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
|
|
|
|
ref_id = @prompt.ask('Account to keep unchanged:') do |q|
|
|
q.required true
|
|
q.default 0
|
|
q.convert :int
|
|
end
|
|
|
|
accounts.delete_at(ref_id)
|
|
|
|
i = 0
|
|
accounts.each do |account|
|
|
i += 1
|
|
username = account.username + "_#{i}"
|
|
|
|
while Account.local.exists?(username: username)
|
|
i += 1
|
|
username = account.username + "_#{i}"
|
|
end
|
|
|
|
account.update!(username: username)
|
|
end
|
|
end
|
|
|
|
def deduplicate_remote_accounts!(accounts)
|
|
accounts = accounts.sort_by(&:updated_at).reverse
|
|
|
|
reference_account = accounts.shift
|
|
|
|
accounts.each do |other_account|
|
|
if other_account.public_key == reference_account.public_key
|
|
# The accounts definitely point to the same resource, so
|
|
# it's safe to re-attribute content and relationships
|
|
reference_account.merge_with!(other_account)
|
|
end
|
|
|
|
other_account.destroy
|
|
end
|
|
end
|
|
|
|
def merge_conversations!(main_conv, duplicate_conv)
|
|
owned_classes = [ConversationMute, AccountConversation]
|
|
owned_classes.each do |klass|
|
|
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:account_id, main_conv.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def merge_custom_emojis!(main_emoji, duplicate_emoji)
|
|
owned_classes = [AnnouncementReaction]
|
|
owned_classes.each do |klass|
|
|
klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id)
|
|
end
|
|
end
|
|
|
|
def merge_custom_emoji_categories!(main_category, duplicate_category)
|
|
owned_classes = [CustomEmoji]
|
|
owned_classes.each do |klass|
|
|
klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id)
|
|
end
|
|
end
|
|
|
|
def merge_statuses!(main_status, duplicate_status)
|
|
owned_classes = [Favourite, Mention, Poll]
|
|
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
|
|
owned_classes.each do |klass|
|
|
klass.where(status_id: duplicate_status.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:status_id, main_status.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
|
|
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:status_id, main_status.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
|
|
Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:in_reply_to_id, main_status.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
|
|
Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:reblog_of_id, main_status.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
|
|
def merge_tags!(main_tag, duplicate_tag)
|
|
[FeaturedTag].each do |klass|
|
|
klass.where(tag_id: duplicate_tag.id).find_each do |record|
|
|
begin
|
|
record.update_attribute(:tag_id, main_tag.id)
|
|
rescue ActiveRecord::RecordNotUnique
|
|
next
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def find_duplicate_accounts
|
|
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
|
|
end
|
|
|
|
def remove_index_if_exists!(table, name)
|
|
ActiveRecord::Base.connection.remove_index(table, name: name)
|
|
rescue ArgumentError
|
|
nil
|
|
rescue ActiveRecord::StatementInvalid
|
|
nil
|
|
end
|
|
end
|
|
end
|