Migrate glitch-soc keyword mutes to Mastodon's

Completely remove glitch-soc's Keyword Mutes, migrate
existing database records to CustomFilters.

Handling of client-side filters is still not implemented
in the glitch-soc front-end.
This commit is contained in:
Thibaut Girka 2018-07-07 21:40:49 +02:00
parent ee41947e9d
commit 402da8065c
14 changed files with 55 additions and 565 deletions

View file

@ -1,61 +0,0 @@
# frozen_string_literal: true
class Settings::KeywordMutesController < Settings::BaseController
before_action :load_keyword_mute, only: [:edit, :update, :destroy]
def index
@keyword_mutes = paginated_keyword_mutes_for_account
end
def new
@keyword_mute = keyword_mutes_for_account.build
end
def create
@keyword_mute = keyword_mutes_for_account.create(keyword_mute_params)
if @keyword_mute.persisted?
redirect_to settings_keyword_mutes_path, notice: I18n.t('generic.changes_saved_msg')
else
render :new
end
end
def update
if @keyword_mute.update(keyword_mute_params)
redirect_to settings_keyword_mutes_path, notice: I18n.t('generic.changes_saved_msg')
else
render :edit
end
end
def destroy
@keyword_mute.destroy!
redirect_to settings_keyword_mutes_path, notice: I18n.t('generic.changes_saved_msg')
end
def destroy_all
keyword_mutes_for_account.delete_all
redirect_to settings_keyword_mutes_path, notice: I18n.t('generic.changes_saved_msg')
end
private
def keyword_mutes_for_account
Glitch::KeywordMute.where(account: current_account)
end
def load_keyword_mute
@keyword_mute = keyword_mutes_for_account.find(params[:id])
end
def keyword_mute_params
params.require(:keyword_mute).permit(:keyword, :whole_word, :apply_to_mentions)
end
def paginated_keyword_mutes_for_account
keyword_mutes_for_account.order(:keyword).page params[:page]
end
end

View file

@ -153,7 +153,6 @@ class FeedManager
def filter_from_home?(status, receiver_id)
return false if receiver_id == status.account_id
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
return true if keyword_filter_from_home?(status, receiver_id)
return true if phrase_filtered?(status, receiver_id, :home)
check_for_blocks = status.mentions.pluck(:account_id)
@ -181,26 +180,6 @@ class FeedManager
false
end
def keyword_filter_from_home?(status, receiver_id)
# If this status mentions the receiver, use the mentions scope: it's
# possible that the status will show up in the receiver's mentions, which
# means it ought to show up in the home feed as well.
#
# If it doesn't mention the receiver but is still headed for the home feed,
# use the home feed scope.
scope = if status.mentions.pluck(:account_id).include?(receiver_id)
Glitch::KeywordMute::Scopes::Mentions
else
Glitch::KeywordMute::Scopes::HomeFeed
end
return true if keyword_filter?(status, receiver_id, scope)
end
def keyword_filter?(status, receiver_id, scope)
Glitch::KeywordMuteHelper.new(receiver_id).matches?(status, scope)
end
def filter_from_mentions?(status, receiver_id)
return true if receiver_id == status.account_id
return true if phrase_filtered?(status, receiver_id, :notifications)
@ -213,7 +192,6 @@ class FeedManager
should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted)
should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them
should_filter ||= keyword_filter?(status, receiver_id, Glitch::KeywordMute::Scopes::Mentions) # or if the mention contains a muted keyword
should_filter
end

View file

@ -1,7 +0,0 @@
# frozen_string_literal: true
module Glitch
def self.table_name_prefix
'glitch_'
end
end

View file

@ -1,123 +0,0 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: glitch_keyword_mutes
#
# id :bigint(8) not null, primary key
# account_id :bigint(8) not null
# keyword :string not null
# whole_word :boolean default(TRUE), not null
# created_at :datetime not null
# updated_at :datetime not null
# apply_to_mentions :boolean default(TRUE), not null
#
class Glitch::KeywordMute < ApplicationRecord
belongs_to :account, required: true
validates_presence_of :keyword
after_commit :invalidate_cached_matchers
module Scopes
Unscoped = 0b00
HomeFeed = 0b01
Mentions = 0b10
end
def self.text_matcher_for(account_id)
TextMatcher.new(account_id)
end
def self.tag_matcher_for(account_id)
TagMatcher.new(account_id)
end
def scope
s = Scopes::Unscoped
s |= Scopes::HomeFeed
s |= Scopes::Mentions if apply_to_mentions?
s
end
private
def invalidate_cached_matchers
Rails.cache.delete(TextMatcher.cache_key(account_id))
Rails.cache.delete(TagMatcher.cache_key(account_id))
end
class CachedKeywordMute
attr_reader :keyword
attr_reader :whole_word
attr_reader :scope
def initialize(keyword, whole_word, scope)
@keyword = keyword
@whole_word = whole_word
@scope = scope
end
def boundary_regex_for_keyword
sb = keyword =~ /\A[[:word:]]/ ? '\b' : ''
eb = keyword =~ /[[:word:]]\Z/ ? '\b' : ''
/(?mix:#{sb}#{Regexp.escape(keyword)}#{eb})/
end
def matches?(str, required_scope)
((required_scope & scope) == required_scope) && \
str =~ (whole_word ? boundary_regex_for_keyword : /#{Regexp.escape(keyword)}/i)
end
end
class Matcher
attr_reader :account_id
attr_reader :keywords
def initialize(account_id)
@account_id = account_id
@keywords = Rails.cache.fetch(self.class.cache_key(account_id)) { fetch_keywords }
end
protected
def fetch_keywords
Glitch::KeywordMute.select(:whole_word, :keyword, :apply_to_mentions)
.where(account_id: account_id)
.map { |kw| CachedKeywordMute.new(transform_keyword(kw.keyword), kw.whole_word, kw.scope) }
end
def transform_keyword(keyword)
keyword
end
end
class TextMatcher < Matcher
def self.cache_key(account_id)
format('keyword_mutes:regex:text:%s', account_id)
end
def matches?(str, scope)
keywords.any? { |kw| kw.matches?(str, scope) }
end
end
class TagMatcher < Matcher
def self.cache_key(account_id)
format('keyword_mutes:regex:tag:%s', account_id)
end
def matches?(tags, scope)
tags.pluck(:name).any? do |n|
keywords.any? { |kw| kw.matches?(n, scope) }
end
end
protected
def transform_keyword(kw)
Tag::HASHTAG_RE =~ kw ? $1 : kw
end
end
end

View file

@ -1,27 +0,0 @@
require 'html2text'
class Glitch::KeywordMuteHelper
attr_reader :text_matcher
attr_reader :tag_matcher
def initialize(receiver_id)
@text_matcher = Glitch::KeywordMute.text_matcher_for(receiver_id)
@tag_matcher = Glitch::KeywordMute.tag_matcher_for(receiver_id)
end
def matches?(status, scope)
matchers_match?(status, scope) || (status.reblog? && matchers_match?(status.reblog, scope))
end
private
def matchers_match?(status, scope)
text_matcher.matches?(prepare_text(status.text), scope) ||
text_matcher.matches?(prepare_text(status.spoiler_text), scope) ||
tag_matcher.matches?(status.tags, scope)
end
def prepare_text(text)
Html2Text.convert(text)
end
end

View file

@ -7,7 +7,6 @@ SimpleNavigation::Configuration.run do |navigation|
primary.item :settings, safe_join([fa_icon('cog fw'), t('settings.settings')]), settings_profile_url do |settings|
settings.item :profile, safe_join([fa_icon('user fw'), t('settings.edit_profile')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration}
settings.item :preferences, safe_join([fa_icon('sliders fw'), t('settings.preferences')]), settings_preferences_url
settings.item :keyword_mutes, safe_join([fa_icon('volume-off fw'), t('settings.keyword_mutes')]), settings_keyword_mutes_url
settings.item :notifications, safe_join([fa_icon('bell fw'), t('settings.notifications')]), settings_notifications_url
settings.item :password, safe_join([fa_icon('lock fw'), t('auth.security')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete}
settings.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_url, highlights_on: %r{/settings/two_factor_authentication}

View file

@ -77,12 +77,6 @@ Rails.application.routes.draw do
namespace :settings do
resource :profile, only: [:show, :update]
resources :keyword_mutes do
collection do
delete :destroy_all
end
end
resource :preferences, only: [:show, :update]
resource :notifications, only: [:show, :update]
resource :import, only: [:show, :create]

View file

@ -0,0 +1,54 @@
class MigrateFilters < ActiveRecord::Migration[5.2]
class GlitchKeywordMute < ApplicationRecord
# Dummy class, as we removed Glitch::KeywordMute
belongs_to :account, required: true
validates_presence_of :keyword
end
class CustomFilter < ApplicationRecord
# Dummy class, in case CustomFilter gets altered in the future
belongs_to :account
validates :phrase, :context, presence: true
before_validation :clean_up_contexts
private
def clean_up_contexts
self.context = Array(context).map(&:strip).map(&:presence).compact
end
end
disable_ddl_transaction!
def up
GlitchKeywordMute.find_each do |filter|
filter.account.custom_filters.create!(
phrase: filter.keyword,
context: filter.apply_to_mentions ? %w(home public notifications) : %w(home public),
whole_word: filter.whole_word,
irreversible: true)
end
drop_table :glitch_keyword_mutes
end
def down
create_table "glitch_keyword_mutes" do |t|
t.references :account, null: false
t.string :keyword, null: false
t.boolean :whole_word, default: true, null: false
t.boolean :apply_to_mentions, default: true, null: false
t.timestamps
end
add_foreign_key :glitch_keyword_mutes, :accounts, on_delete: :cascade
CustomFilter.where(irreversible: true).find_each do |filter|
GlitchKeywordMute.where(account: filter.account).create!(
keyword: filter.phrase,
whole_word: filter.whole_word,
apply_to_mentions: filter.context.include?('notifications'))
end
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2018_07_07_154237) do
ActiveRecord::Schema.define(version: 2018_07_07_193142) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -211,16 +211,6 @@ ActiveRecord::Schema.define(version: 2018_07_07_154237) do
t.index ["account_id", "target_account_id"], name: "index_follows_on_account_id_and_target_account_id", unique: true
end
create_table "glitch_keyword_mutes", force: :cascade do |t|
t.bigint "account_id", null: false
t.string "keyword", null: false
t.boolean "whole_word", default: true, null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.boolean "apply_to_mentions", default: true, null: false
t.index ["account_id"], name: "index_glitch_keyword_mutes_on_account_id"
end
create_table "identities", id: :serial, force: :cascade do |t|
t.integer "user_id"
t.string "provider", default: "", null: false
@ -603,7 +593,6 @@ ActiveRecord::Schema.define(version: 2018_07_07_154237) do
add_foreign_key "follow_requests", "accounts", name: "fk_76d644b0e7", on_delete: :cascade
add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade
add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade
add_foreign_key "glitch_keyword_mutes", "accounts", on_delete: :cascade
add_foreign_key "identities", "users", on_delete: :cascade
add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade
add_foreign_key "invites", "users", on_delete: :cascade

View file

@ -1,5 +0,0 @@
require 'rails_helper'
RSpec.describe Settings::KeywordMutesController, type: :controller do
end

View file

@ -1,2 +0,0 @@
Fabricator('Glitch::KeywordMute') do
end

View file

@ -134,68 +134,6 @@ RSpec.describe FeedManager do
expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true
end
it 'returns true for a status containing a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take')
status = Fabricate(:status, text: 'This is a hot take', account: bob)
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
it 'returns true for a reply containing a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take')
s1 = Fabricate(:status, text: 'Something', account: alice)
s2 = Fabricate(:status, text: 'This is a hot take', thread: s1, account: bob)
expect(FeedManager.instance.filter?(:home, s2, alice.id)).to be true
end
it 'returns true for a status whose spoiler text contains a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take')
status = Fabricate(:status, spoiler_text: 'This is a hot take', account: bob)
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
it 'returns true for a reblog containing a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take')
status = Fabricate(:status, text: 'This is a hot take', account: bob)
reblog = Fabricate(:status, reblog: status, account: jeff)
expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true
end
it 'returns true for a reblog whose spoiler text contains a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take')
status = Fabricate(:status, spoiler_text: 'This is a hot take', account: bob)
reblog = Fabricate(:status, reblog: status, account: jeff)
expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true
end
it 'returns true for a status with a tag that matches a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'jorts')
status = Fabricate(:status, account: bob)
status.tags << Fabricate(:tag, name: 'jorts')
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
it 'returns true for a status with a tag that matches an octothorpe-prefixed muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: '#jorts')
status = Fabricate(:status, account: bob)
status.tags << Fabricate(:tag, name: 'jorts')
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
it 'returns false if the status is muted by a keyword mute that does not apply to mentions' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'take', apply_to_mentions: false)
status = Fabricate(:status, spoiler_text: 'This is a hot take', account: bob)
status.mentions.create!(account_id: alice.id)
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be false
end
context 'for irreversibly muted phrases' do
it 'considers word boundaries when matching' do
alice.custom_filters.create!(phrase: 'bob', context: %w(home), irreversible: true)
@ -247,20 +185,6 @@ RSpec.describe FeedManager do
bob.follow!(alice)
expect(FeedManager.instance.filter?(:mentions, status, bob.id)).to be false
end
it 'returns true for status that contains a muted keyword' do
Fabricate('Glitch::KeywordMute', account: bob, keyword: 'take')
status = Fabricate(:status, text: 'This is a hot take', account: alice)
bob.follow!(alice)
expect(FeedManager.instance.filter?(:mentions, status, bob.id)).to be true
end
it 'returns false for a mention that contains a word muted by a keyword that does not apply to mentions' do
Fabricate('Glitch::KeywordMute', account: bob, keyword: 'take', apply_to_mentions: false)
status = Fabricate(:status, text: 'This is a hot take', account: alice)
bob.follow!(alice)
expect(FeedManager.instance.filter?(:mentions, status, bob.id)).to be false
end
end
end

View file

@ -1,52 +0,0 @@
require 'rails_helper'
RSpec.describe Glitch::KeywordMuteHelper do
describe '#matches?' do
Unscoped = Glitch::KeywordMute::Scopes::Unscoped
let(:alice) { Fabricate(:account, username: 'alice').tap(&:save!) }
let(:helper) { Glitch::KeywordMuteHelper.new(alice) }
it 'ignores names of HTML tags in status text' do
status = Fabricate(:status, text: '<addr>uh example</addr>')
Glitch::KeywordMute.create!(account: alice, keyword: 'addr')
expect(helper.matches?(status, Unscoped)).to be false
end
it 'ignores properties of HTML tags in status text' do
status = Fabricate(:status, text: '<a href="https://www.example.org">uh example</a>')
Glitch::KeywordMute.create!(account: alice, keyword: 'href')
expect(helper.matches?(status, Unscoped)).to be false
end
it 'matches text inside HTML tags' do
status = Fabricate(:status, text: '<p>HEY THIS IS SOMETHING ANNOYING</p>')
Glitch::KeywordMute.create!(account: alice, keyword: 'annoying')
expect(helper.matches?(status, Unscoped)).to be true
end
it 'matches < in HTML-stripped text' do
status = Fabricate(:status, text: '<p>I <3 oats</p>')
Glitch::KeywordMute.create!(account: alice, keyword: '<3')
expect(helper.matches?(status, Unscoped)).to be true
end
it 'matches &lt; in HTML text' do
status = Fabricate(:status, text: '<p>I &lt;3 oats</p>')
Glitch::KeywordMute.create!(account: alice, keyword: '<3')
expect(helper.matches?(status, Unscoped)).to be true
end
it 'matches link hrefs in HTML text' do
status = Fabricate(:status, text: '<p><a href="https://example.com/it-was-milk">yep</a></p>')
Glitch::KeywordMute.create!(account: alice, keyword: 'milk')
expect(helper.matches?(status, Unscoped)).to be true
end
end
end

View file

@ -1,171 +0,0 @@
require 'rails_helper'
RSpec.describe Glitch::KeywordMute, type: :model do
let(:alice) { Fabricate(:account, username: 'alice').tap(&:save!) }
let(:bob) { Fabricate(:account, username: 'bob').tap(&:save!) }
Unscoped = Glitch::KeywordMute::Scopes::Unscoped
describe '.text_matcher_for' do
let(:matcher) { Glitch::KeywordMute.text_matcher_for(alice.id) }
describe 'with no mutes' do
before do
Glitch::KeywordMute.delete_all
end
it 'does not match' do
expect(matcher.matches?('This is a hot take', Unscoped)).to be_falsy
end
end
describe 'with mutes' do
it 'does not match keywords set by a different account' do
Glitch::KeywordMute.create!(account: bob, keyword: 'take')
expect(matcher.matches?('This is a hot take', Unscoped)).to be_falsy
end
it 'does not match if no keywords match the status text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'cold')
expect(matcher.matches?('This is a hot take', Unscoped)).to be_falsy
end
it 'considers word boundaries when matching' do
Glitch::KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true)
expect(matcher.matches?('bobcats', Unscoped)).to be_falsy
end
it 'matches substrings if whole_word is false' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take', whole_word: false)
expect(matcher.matches?('This is a shiitake mushroom', Unscoped)).to be_truthy
end
it 'matches keywords at the beginning of the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take')
expect(matcher.matches?('Take this', Unscoped)).to be_truthy
end
it 'matches keywords at the end of the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take')
expect(matcher.matches?('This is a hot take', Unscoped)).to be_truthy
end
it 'matches if at least one keyword case-insensitively matches the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
expect(matcher.matches?('This is a HOT take', Unscoped)).to be_truthy
end
it 'matches if at least one non-whole-word keyword case-insensitively matches the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot', whole_word: false)
expect(matcher.matches?('This is a HOTTY take', Unscoped)).to be_truthy
end
it 'maintains case-insensitivity when combining keywords into a single matcher' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
Glitch::KeywordMute.create!(account: alice, keyword: 'cold')
expect(matcher.matches?('This is a HOT take', Unscoped)).to be_truthy
end
it 'matches keywords surrounded by non-alphanumeric ornamentation' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
expect(matcher.matches?('(hot take)', Unscoped)).to be_truthy
end
it 'escapes metacharacters in whole-word keywords' do
Glitch::KeywordMute.create!(account: alice, keyword: '(hot take)')
expect(matcher.matches?('(hot take)', Unscoped)).to be_truthy
end
it 'escapes metacharacters in non-whole-word keywords' do
Glitch::KeywordMute.create!(account: alice, keyword: '(-', whole_word: false)
expect(matcher.matches?('bad (-)', Unscoped)).to be_truthy
end
it 'uses case-folding rules appropriate for more than just English' do
Glitch::KeywordMute.create!(account: alice, keyword: 'großeltern')
expect(matcher.matches?('besuch der grosseltern', Unscoped)).to be_truthy
end
it 'matches keywords that are composed of multiple words' do
Glitch::KeywordMute.create!(account: alice, keyword: 'a shiitake')
expect(matcher.matches?('This is a shiitake', Unscoped)).to be_truthy
expect(matcher.matches?('This is shiitake', Unscoped)).to_not be_truthy
end
end
end
describe '.tag_matcher_for' do
let(:matcher) { Glitch::KeywordMute.tag_matcher_for(alice.id) }
let(:status) { Fabricate(:status) }
describe 'with no mutes' do
before do
Glitch::KeywordMute.delete_all
end
it 'does not match' do
status.tags << Fabricate(:tag, name: 'xyzzy')
expect(matcher.matches?(status.tags, Unscoped)).to be false
end
end
describe 'with mutes' do
it 'does not match keywords set by a different account' do
status.tags << Fabricate(:tag, name: 'xyzzy')
Glitch::KeywordMute.create!(account: bob, keyword: 'take')
expect(matcher.matches?(status.tags, Unscoped)).to be false
end
it 'matches #xyzzy when given the mute "#xyzzy"' do
status.tags << Fabricate(:tag, name: 'xyzzy')
Glitch::KeywordMute.create!(account: alice, keyword: '#xyzzy')
expect(matcher.matches?(status.tags, Unscoped)).to be true
end
it 'matches #thingiverse when given the non-whole-word mute "#thing"' do
status.tags << Fabricate(:tag, name: 'thingiverse')
Glitch::KeywordMute.create!(account: alice, keyword: '#thing', whole_word: false)
expect(matcher.matches?(status.tags, Unscoped)).to be true
end
it 'matches #hashtag when given the mute "##hashtag""' do
status.tags << Fabricate(:tag, name: 'hashtag')
Glitch::KeywordMute.create!(account: alice, keyword: '##hashtag')
expect(matcher.matches?(status.tags, Unscoped)).to be true
end
it 'matches #oatmeal when given the non-whole-word mute "oat"' do
status.tags << Fabricate(:tag, name: 'oatmeal')
Glitch::KeywordMute.create!(account: alice, keyword: 'oat', whole_word: false)
expect(matcher.matches?(status.tags, Unscoped)).to be true
end
it 'does not match #oatmeal when given the mute "#oat"' do
status.tags << Fabricate(:tag, name: 'oatmeal')
Glitch::KeywordMute.create!(account: alice, keyword: 'oat')
expect(matcher.matches?(status.tags, Unscoped)).to be false
end
end
end
end