Fix a performance issue with status expiration

This commit is contained in:
noellabo 2021-06-28 14:56:26 +09:00
parent 284b128e50
commit a115072b6a
15 changed files with 78 additions and 74 deletions

View file

@ -32,7 +32,7 @@ class Api::V1::Statuses::BookmarksController < Api::BaseController
private
def set_status
@status = Status.include_expired(current_account).find(params[:status_id])
@status = Status.include_expired.find(params[:status_id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -65,7 +65,7 @@ class Api::V1::Statuses::EmojiReactionedByAccountsController < Api::BaseControll
end
def set_status
@status = Status.include_expired(current_account).find(params[:status_id])
@status = Status.include_expired.find(params[:status_id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -65,7 +65,7 @@ class Api::V1::Statuses::FavouritedByAccountsController < Api::BaseController
end
def set_status
@status = Status.include_expired(current_account).find(params[:status_id])
@status = Status.include_expired.find(params[:status_id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -31,7 +31,7 @@ class Api::V1::Statuses::FavouritesController < Api::BaseController
private
def set_status
@status = Status.include_expired(current_account).find(params[:status_id])
@status = Status.include_expired.find(params[:status_id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -61,7 +61,7 @@ class Api::V1::Statuses::RebloggedByAccountsController < Api::BaseController
end
def set_status
@status = Status.include_expired(current_account).find(params[:status_id])
@status = Status.include_expired.find(params[:status_id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -71,7 +71,7 @@ class Api::V1::StatusesController < Api::BaseController
private
def set_status
@status = Status.include_expired(current_account).find(params[:id])
@status = Status.include_expired.find(params[:id])
authorize @status, :show?
rescue Mastodon::NotPermittedError
not_found

View file

@ -30,7 +30,7 @@ module Expireable
end
def expires?
!expires_at.nil?
!expires_at.nil? && expires_at != ::Float::INFINITY
end
end
end

View file

@ -24,7 +24,7 @@
# poll_id :bigint(8)
# quote_id :bigint(8)
# deleted_at :datetime
# expires_at :datetime
# expires_at :datetime default(Infinity), not null
# expires_action :integer default("delete"), not null
#
@ -102,42 +102,8 @@ class Status < ApplicationRecord
scope :remote, -> { where(local: false).where.not(uri: nil) }
scope :local, -> { where(local: true).or(where(uri: nil)) }
scope :not_expired, -> { where(statuses: {expires_at: nil} ).or(where('statuses.expires_at >= ?', Time.now.utc)) }
scope :include_expired, ->(account = nil) {
if account.nil?
unscoped.recent.kept
else
unscoped.recent.kept.where(<<-SQL, account_id: account.id, current_utc: Time.now.utc)
(
statuses.expires_at IS NULL
) OR
NOT (
statuses.account_id != :account_id
AND NOT EXISTS (
SELECT *
FROM bookmarks b
WHERE b.status_id = statuses.id
AND b.account_id = :account_id
)
AND NOT EXISTS (
SELECT *
FROM favourites f
WHERE f.status_id = statuses.id
AND f.account_id = :account_id
)
AND NOT EXISTS (
SELECT *
FROM emoji_reactions r
WHERE r.status_id = statuses.id
AND r.account_id = :account_id
)
AND statuses.expires_at IS NOT NULL
AND statuses.expires_at < :current_utc
)
SQL
end
}
scope :not_expired, -> { where("COALESCE(statuses.expires_at,'infinity') >= CURRENT_TIMESTAMP") }
scope :include_expired, -> { unscoped.recent.kept }
scope :with_accounts, ->(ids) { where(id: ids).includes(:account) }
scope :without_replies, -> { where('statuses.reply = FALSE OR statuses.in_reply_to_account_id = statuses.account_id') }
scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') }
@ -187,6 +153,10 @@ class Status < ApplicationRecord
REAL_TIME_WINDOW = 6.hours
def expires_at=(val)
super(val.nil? ? 'infinity' : val)
end
def searchable_by(preloaded = nil)
ids = []

View file

@ -127,7 +127,7 @@ class PostStatusService < BaseService
DistributionWorker.perform_async(@status.id)
ActivityPub::DistributionWorker.perform_async(@status.id)
PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll
DeleteExpiredStatusWorker.perform_at(@status.expires_at, @status.id) if !@status.expires_at.nil? && @status.expires_delete?
DeleteExpiredStatusWorker.perform_at(@status.expires_at, @status.id) if @status.expires? && @status.expires_delete?
end
def validate_media!

View file

@ -7,7 +7,7 @@ class ExpiresValidator < ActiveModel::Validator
def validate(status)
current_time = Time.now.utc
status.errors.add(:expires_at, I18n.t('statuses.errors.duration_too_long')) if status.expires_at.present? && status.expires_at - current_time > MAX_EXPIRATION
status.errors.add(:expires_at, I18n.t('statuses.errors.duration_too_short')) if status.expires_at.present? && (status.expires_at - current_time).ceil < MIN_EXPIRATION
status.errors.add(:expires_at, I18n.t('statuses.errors.duration_too_long')) if status.expires? && status.expires_at - current_time > MAX_EXPIRATION
status.errors.add(:expires_at, I18n.t('statuses.errors.duration_too_short')) if status.expires? && (status.expires_at - current_time).ceil < MIN_EXPIRATION
end
end

View file

@ -0,0 +1,19 @@
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class FixExpiredAtInStatusesNullToInfinity < ActiveRecord::Migration[6.1]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
safety_assured do
change_column_default :statuses, :expires_at, 'infinity'
end
end
def down
safety_assured do
change_column_default :statuses, :expires_at, nil
end
end
end

View file

@ -0,0 +1,13 @@
class UpdateStatusesIndexWithExpiredAt < ActiveRecord::Migration[6.1]
disable_ddl_transaction!
def up
safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at, :expires_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20210627 }
remove_index :statuses, name: :index_statuses_20190820
end
def down
safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20190820 }
remove_index :statuses, name: :index_statuses_20210627
end
end

View file

@ -0,0 +1,25 @@
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class MigrationExpiredAtInStatuses < ActiveRecord::Migration[6.1]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
safety_assured do
update_column_in_batches(:statuses, :expires_at, 'infinity') do |table, query|
query.where(table[:expires_at].eq(nil))
end
change_column_null :statuses, :expires_at, false
end
end
def down
safety_assured do
change_column_null :statuses, :expires_at, true
update_column_in_batches(:statuses, :expires_at, nil) do |table, query|
query.where(table[:expires_at].eq('infinity'))
end
end
end
end

View file

@ -425,27 +425,6 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.index ["list_id"], name: "index_domain_subscribes_on_list_id"
end
create_table "domains", force: :cascade do |t|
t.string "domain", default: "", null: false
t.string "title", default: "", null: false
t.string "short_description", default: "", null: false
t.string "email", default: "", null: false
t.string "version", default: "", null: false
t.string "thumbnail_remote_url", default: "", null: false
t.string "languages", array: true
t.boolean "registrations"
t.boolean "approval_required"
t.bigint "contact_account_id"
t.string "software", default: "", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "thumbnail_file_name"
t.string "thumbnail_content_type"
t.integer "thumbnail_file_size"
t.datetime "thumbnail_updated_at"
t.index ["contact_account_id"], name: "index_domains_on_contact_account_id"
end
create_table "email_domain_blocks", force: :cascade do |t|
t.string "domain", default: "", null: false
t.datetime "created_at", null: false
@ -466,7 +445,7 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.index ["account_id"], name: "index_emoji_reactions_on_account_id"
t.index ["custom_emoji_id"], name: "index_emoji_reactions_on_custom_emoji_id"
t.index ["status_id"], name: "index_emoji_reactions_on_status_id"
end
end
create_table "encrypted_messages", id: :bigint, default: -> { "timestamp_id('encrypted_messages'::text)" }, force: :cascade do |t|
t.bigint "device_id"
@ -986,9 +965,9 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
t.bigint "poll_id"
t.bigint "quote_id"
t.datetime "deleted_at"
t.datetime "expires_at"
t.datetime "expires_at", default: ::Float::INFINITY, null: false
t.integer "expires_action", default: 0, null: false
t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
t.index ["account_id", "id", "visibility", "updated_at", "expires_at"], name: "index_statuses_20210627", order: { id: :desc }, where: "(deleted_at IS NULL)"
t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
@ -1178,7 +1157,6 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do
add_foreign_key "devices", "oauth_access_tokens", column: "access_token_id", on_delete: :cascade
add_foreign_key "domain_subscribes", "accounts", on_delete: :cascade
add_foreign_key "domain_subscribes", "lists", on_delete: :cascade
add_foreign_key "domains", "accounts", column: "contact_account_id"
add_foreign_key "email_domain_blocks", "email_domain_blocks", column: "parent_id", on_delete: :cascade
add_foreign_key "emoji_reactions", "accounts", on_delete: :cascade
add_foreign_key "emoji_reactions", "custom_emojis", on_delete: :cascade

View file

@ -318,8 +318,7 @@ module Mastodon
media_attachments m
INNER JOIN statuses s ON m.status_id = s.id
AND s.deleted_at IS NULL
AND (s.expires_at IS NULL
OR s.expires_at >= CURRENT_TIMESTAMP)
AND s.expires_at >= CURRENT_TIMESTAMP
INNER JOIN accounts a ON s.account_id = a.id
WHERE
a.domain = $1