Add synchronization of remote featured_tag

app/controllers/api/v1/featured_tags_controller.rb

@@ -13,14 +13,12 @@ class Api::V1::FeaturedTagsController < Api::BaseController
   end
 
   def create
-    @featured_tag = current_account.featured_tags.new(featured_tag_params)
+    featured_tag = CreateFeaturedTagService.new.call(current_account, featured_tag_params[:name])
-    @featured_tag.reset_data
+    render json: featured_tag, serializer: REST::FeaturedTagSerializer
-    @featured_tag.save!
-    render json: @featured_tag, serializer: REST::FeaturedTagSerializer
   end
 
   def destroy
-    @featured_tag.destroy!
+    RemoveFeaturedTagWorker.perform_async(current_account.id, @featured_tag.id)
     render_empty
   end
 

app/controllers/settings/featured_tags_controller.rb

@@ -10,10 +10,9 @@ class Settings::FeaturedTagsController < Settings::BaseController
   end
 
   def create
-    @featured_tag = current_account.featured_tags.new(featured_tag_params)
+    @featured_tag = CreateFeaturedTagService.new.call(current_account, featured_tag_params[:name], force: false)
-    @featured_tag.reset_data
 
-    if @featured_tag.save
+    if @featured_tag.valid?
       redirect_to settings_featured_tags_path
     else
       set_featured_tags
@@ -24,7 +23,7 @@ class Settings::FeaturedTagsController < Settings::BaseController
   end
 
   def destroy
-    @featured_tag.destroy!
+    RemoveFeaturedTagWorker.perform_async(current_account.id, @featured_tag.id)
     redirect_to settings_featured_tags_path
   end
 

app/lib/activitypub/activity/add.rb

@@ -2,13 +2,32 @@
 
 class ActivityPub::Activity::Add < ActivityPub::Activity
   def perform
-    return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url
+    return if @json['target'].blank?
 
-    status   = status_from_uri(object_uri)
+    case value_or_id(@json['target'])
-    status ||= fetch_remote_original_status
+    when @account.featured_collection_url
+      case @object['type']
+      when 'Hashtag'
+        add_featured_tags
+      else
+        add_featured
+      end
+    end
+  end
+
+  private
+
+  def add_featured
+    status = status_from_object
 
     return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
 
     StatusPin.create!(account: @account, status: status)
   end
+
+  def add_featured_tags
+    name = @object['name']&.delete_prefix('#')
+
+    FeaturedTag.create!(account: @account, name: name) if name.present?
+  end
 end

app/lib/activitypub/activity/remove.rb

@@ -2,8 +2,22 @@
 
 class ActivityPub::Activity::Remove < ActivityPub::Activity
   def perform
-    return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url
+    return if @json['target'].blank?
 
+    case value_or_id(@json['target'])
+    when @account.featured_collection_url
+      case @object['type']
+      when 'Hashtag'
+        remove_featured_tags
+      else
+        remove_featured
+      end
+    end
+  end
+
+  private
+
+  def remove_featured
     status = status_from_uri(object_uri)
 
     return unless !status.nil? && status.account_id == @account.id
@@ -11,4 +25,13 @@ class ActivityPub::Activity::Remove < ActivityPub::Activity
     pin = StatusPin.find_by(account: @account, status: status)
     pin&.destroy!
   end
+
+  def remove_featured_tags
+    name = @object['name']&.delete_prefix('#')
+
+    return if name.blank?
+
+    featured_tag = FeaturedTag.by_name(name).find_by(account: @account)
+    featured_tag&.destroy!
+  end
 end

app/models/account.rb

@@ -50,6 +50,7 @@
 #  settings                      :jsonb            default("{}"), not null
 #  silence_mode                  :integer          default(0), not null
 #  searchability                 :integer          default(3), not null
+#  featured_tags_collection_url  :string
 #
 
 class Account < ApplicationRecord

app/models/featured_tag.rb

@@ -13,17 +13,30 @@
 #
 
 class FeaturedTag < ApplicationRecord
-  belongs_to :account, inverse_of: :featured_tags, required: true
+  belongs_to :account, inverse_of: :featured_tags
-  belongs_to :tag, inverse_of: :featured_tags, required: true
+  belongs_to :tag, inverse_of: :featured_tags, optional: true # Set after validation
 
-  delegate :name, to: :tag, allow_nil: true
+  validate :validate_tag_name, on: :create
-
-  validates_associated :tag, on: :create
-  validates :name, presence: true, on: :create
   validate :validate_featured_tags_limit, on: :create
 
-  def name=(str)
+  before_create :set_tag
-    self.tag = Tag.find_or_create_by_names(str.strip)&.first
+  before_create :reset_data
+
+  # scope :by_name, ->(name) { joins(:tag).where(tag: { name: HashtagNormalizer.new.normalize(name) }) }
+  scope :by_name, ->(name) { joins(:tag).where(tag: { name: name }) }
+
+  # delegate :display_name, to: :tag
+
+  attr_writer :name
+
+  LIMIT = 10
+
+  def sign?
+    true
+  end
+
+  def name
+    tag_id.present? ? tag.name : @name
   end
 
   def increment(timestamp)
@@ -34,14 +47,27 @@ class FeaturedTag < ApplicationRecord
     update(statuses_count: [0, statuses_count - 1].max, last_status_at: account.statuses.where(visibility: %i(public unlisted)).tagged_with(tag).where.not(id: deleted_status_id).select(:created_at).first&.created_at)
   end
 
+  private
+
+  def set_tag
+    self.tag = Tag.find_or_create_by_names(@name)&.first
+  end
+
   def reset_data
     self.statuses_count = account.statuses.where(visibility: %i(public unlisted)).tagged_with(tag).count
     self.last_status_at = account.statuses.where(visibility: %i(public unlisted)).tagged_with(tag).select(:created_at).first&.created_at
   end
 
-  private
-
   def validate_featured_tags_limit
-    errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10
+    return unless account.local?
+
+    errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT
   end
+
+  def validate_tag_name
+    errors.add(:name, :blank) if @name.blank?
+    errors.add(:name, :invalid) unless @name.match?(/\A(#{Tag::HASHTAG_NAME_RE})\z/i)
+    errors.add(:name, :taken) if FeaturedTag.by_name(@name).where(account_id: account_id).exists?
+  end
+
 end

app/serializers/activitypub/add_serializer.rb

@@ -1,10 +1,29 @@
 # frozen_string_literal: true
 
 class ActivityPub::AddSerializer < ActivityPub::Serializer
+  class UriSerializer < ActiveModel::Serializer
+    include RoutingHelper
+
+    def serializable_hash(*_args)
+      ActivityPub::TagManager.instance.uri_for(object)
+    end
+  end
+
+  def self.serializer_for(model, options)
+    case model.class.name
+    when 'Status'
+      UriSerializer
+    when 'FeaturedTag'
+      ActivityPub::HashtagSerializer
+    else
+      super
+    end
+  end
+
   include RoutingHelper
 
   attributes :type, :actor, :target
-  attribute :proper_object, key: :object
+  has_one :proper_object, key: :object
 
   def type
     'Add'
@@ -15,7 +34,7 @@ class ActivityPub::AddSerializer < ActivityPub::Serializer
   end
 
   def proper_object
-    ActivityPub::TagManager.instance.uri_for(object)
+    object
   end
 
   def target

app/serializers/activitypub/hashtag_serializer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::HashtagSerializer < ActivityPub::Serializer
+  context_extensions :hashtag
+
   include RoutingHelper
 
   attributes :type, :href, :name

app/serializers/activitypub/remove_serializer.rb

@@ -1,10 +1,29 @@
 # frozen_string_literal: true
 
 class ActivityPub::RemoveSerializer < ActivityPub::Serializer
+  class UriSerializer < ActiveModel::Serializer
+    include RoutingHelper
+
+    def serializable_hash(*_args)
+      ActivityPub::TagManager.instance.uri_for(object)
+    end
+  end
+
+  def self.serializer_for(model, options)
+    case model.class.name
+    when 'Status'
+      UriSerializer
+    when 'FeaturedTag'
+      ActivityPub::HashtagSerializer
+    else
+      super
+    end
+  end
+
   include RoutingHelper
 
   attributes :type, :actor, :target
-  attribute :proper_object, key: :object
+  has_one :proper_object, key: :object
 
   def type
     'Remove'
@@ -15,7 +34,7 @@ class ActivityPub::RemoveSerializer < ActivityPub::Serializer
   end
 
   def proper_object
-    ActivityPub::TagManager.instance.uri_for(object)
+    object
   end
 
   def target

app/serializers/rest/instance_serializer.rb

@@ -57,6 +57,10 @@ class REST::InstanceSerializer < ActiveModel::Serializer
 
   def configuration
     {
+      accounts: {
+        max_featured_tags: FeaturedTag::LIMIT,
+      },
+
       statuses: {
         max_characters: StatusLengthValidator::MAX_CHARS,
         max_media_attachments: 4,

app/services/activitypub/fetch_featured_collection_service.rb

@@ -3,28 +3,60 @@
 class ActivityPub::FetchFeaturedCollectionService < BaseService
   include JsonLdHelper
 
-  def call(account)
+  def call(account, **options)
     return if account.featured_collection_url.blank? || account.suspended? || account.local?
 
     @account = account
-    @json    = fetch_resource(@account.featured_collection_url, true)
+    @options = options
+    @json    = fetch_resource(@account.featured_collection_url, true, local_follower)
 
-    return unless supported_context?
+    return unless supported_context?(@json)
 
-    case @json['type']
+    process_items(collection_items(@json))
-    when 'Collection', 'CollectionPage'
-      process_items @json['items']
-    when 'OrderedCollection', 'OrderedCollectionPage'
-      process_items @json['orderedItems']
-    end
   end
 
   private
 
+  def collection_items(collection)
+    collection = fetch_collection(collection['first']) if collection['first'].present?
+    return unless collection.is_a?(Hash)
+
+    case collection['type']
+    when 'Collection', 'CollectionPage'
+      collection['items']
+    when 'OrderedCollection', 'OrderedCollectionPage'
+      collection['orderedItems']
+    end
+  end
+
+  def fetch_collection(collection_or_uri)
+    return collection_or_uri if collection_or_uri.is_a?(Hash)
+    return if invalid_origin?(collection_or_uri)
+
+    fetch_resource_without_id_validation(collection_or_uri, local_follower, true)
+  end
+
   def process_items(items)
-    status_ids = items.map { |item| value_or_id(item) }
+    process_note_items(items) if @options[:note]
-                      .filter_map { |uri| ActivityPub::FetchRemoteStatusService.new.call(uri) unless ActivityPub::TagManager.instance.local_uri?(uri) }
+    process_hashtag_items(items) if @options[:hashtag]
-                      .filter_map { |status| status.id if status.account_id == @account.id }
+  end
+
+  def process_note_items(items)
+    status_ids = items.filter_map do |item|
+      next unless item.is_a?(String) || item['type'] == 'Note'
+
+      uri = value_or_id(item)
+      next if ActivityPub::TagManager.instance.local_uri?(uri)
+
+      status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower)
+      next unless status&.account_id == @account.id
+
+      status.id
+    rescue ActiveRecord::RecordInvalid => e
+      Rails.logger.debug "Invalid pinned status #{uri}: #{e.message}"
+      nil
+    end
+
     to_remove = []
     to_add    = status_ids
 
@@ -43,7 +75,29 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
     end
   end
 
-  def supported_context?
+  def process_hashtag_items(items)
-    super(@json)
+    names     = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') } #.map { |name| HashtagNormalizer.new.normalize(name) }
+    to_remove = []
+    to_add    = names
+
+    FeaturedTag.where(account: @account).map(&:name).each do |name|
+      if names.include?(name)
+        to_add.delete(name)
+      else
+        to_remove << name
+      end
+    end
+
+    FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty?
+
+    to_add.each do |name|
+      FeaturedTag.create!(account: @account, name: name)
+    end
+  end
+
+  def local_follower
+    return @local_follower if defined?(@local_follower)
+
+    @local_follower = @account.followers.local.without_suspended.first
   end
 end

app/services/activitypub/fetch_featured_tags_collection_service.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+class ActivityPub::FetchFeaturedTagsCollectionService < BaseService
+  include JsonLdHelper
+
+  def call(account, url)
+    return if account.suspended? || account.local?
+
+    @account = account
+
+    if url.present?
+      @json = fetch_resource(url, true, local_follower)
+
+      return unless supported_context?(@json)
+
+      process_items(collection_items(@json))
+    else
+      html  = fetch_public_html(account.url)
+      items = collection_items_from_html(html) unless html.nil?
+      process_items(items) unless items.nil?
+    end
+
+  end
+
+  private
+
+  def fetch_public_html(url)
+    return @html if defined?(@html)
+
+    Request.new(:get, url).add_headers('Accept' => 'text/html', 'User-Agent' => Mastodon::Version.user_agent + ' Bot').perform do |res|
+      if res.code == 200 && res.mime_type == 'text/html'
+        @html = res.body_with_limit
+      else
+        @html = nil
+      end
+    end
+
+    @html
+  end
+
+  def collection_items_from_html(html)
+    Nokogiri::HTML.parse(html).css('.directory__tag a').filter_map do |a|
+      name = /tagged\/(#{Tag::HASHTAG_NAME_RE})\z/.match(CGI.unescape(URI::parse(a['href']).path)).to_a[1]
+      { 'type' => 'Hashtag', 'name' => "##{name}" } if name.present?
+    end
+  end
+
+  def collection_items(collection)
+    all_items = []
+
+    collection = fetch_collection(collection['first']) if collection['first'].present?
+
+    while collection.is_a?(Hash)
+      items = begin
+        case collection['type']
+        when 'Collection', 'CollectionPage'
+          collection['items']
+        when 'OrderedCollection', 'OrderedCollectionPage'
+          collection['orderedItems']
+        end
+      end
+
+      break if items.blank?
+
+      all_items.concat(items)
+
+      break if all_items.size >= FeaturedTag::LIMIT
+
+      collection = collection['next'].present? ? fetch_collection(collection['next']) : nil
+    end
+
+    all_items
+  end
+
+  def fetch_collection(collection_or_uri)
+    return collection_or_uri if collection_or_uri.is_a?(Hash)
+    return if invalid_origin?(collection_or_uri)
+
+    fetch_resource_without_id_validation(collection_or_uri, local_follower, true)
+  end
+
+  def process_items(items)
+    names     = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') } #.map { |name| HashtagNormalizer.new.normalize(name) }
+    to_remove = []
+    to_add    = names
+
+    FeaturedTag.where(account: @account).map(&:name).each do |name|
+      if names.include?(name)
+        to_add.delete(name)
+      else
+        to_remove << name
+      end
+    end
+
+    FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty?
+
+    to_add.each do |name|
+      FeaturedTag.create!(account: @account, name: name)
+    end
+  end
+
+  def local_follower
+    return @local_follower if defined?(@local_follower)
+
+    @local_follower = @account.followers.local.without_suspended.first
+  end
+end

app/services/activitypub/process_account_service.rb

@@ -47,6 +47,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
     unless @options[:only_key] || @account.suspended?
       check_featured_collection! if @account.featured_collection_url.present?
+      check_featured_tags_collection!
       check_links! unless @account.fields.empty?
     end
 
@@ -94,16 +95,17 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def set_immediate_attributes!
-    @account.featured_collection_url = @json['featured'] || ''
+    @account.featured_collection_url      = @json['featured'] || ''
-    @account.devices_url             = @json['devices'] || ''
+    @account.featured_tags_collection_url = @json['featuredTags'] || ''
-    @account.display_name            = fix_emoji(@json['name']) || ''
+    @account.devices_url                  = @json['devices'] || ''
-    @account.note                    = @json['summary'] || ''
+    @account.display_name                 = fix_emoji(@json['name']) || ''
-    @account.locked                  = @json['manuallyApprovesFollowers'] || false
+    @account.note                         = @json['summary'] || ''
-    @account.fields                  = property_values || {}
+    @account.locked                       = @json['manuallyApprovesFollowers'] || false
-    @account.settings                = defer_settings.merge(other_settings, birthday, address, is_cat)
+    @account.fields                       = property_values || {}
-    @account.also_known_as           = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) }
+    @account.settings                     = defer_settings.merge(other_settings, birthday, address, is_cat)
-    @account.discoverable            = @json['discoverable'] || false
+    @account.also_known_as                = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) }
-    @account.searchability           = searchability_from_audience
+    @account.discoverable                 = @json['discoverable'] || false
+    @account.searchability                = searchability_from_audience
   end
 
   def set_fetchable_key!
@@ -161,7 +163,11 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def check_featured_collection!
-    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id)
+    ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'hashtag' => @json['featuredTags'].blank? && !@account.featured_tags.exists? })
+  end
+
+  def check_featured_tags_collection!
+    ActivityPub::SynchronizeFeaturedTagsCollectionWorker.perform_async(@account.id, @json['featuredTags'])
   end
 
   def check_links!

app/services/create_featured_tag_service.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateFeaturedTagService < BaseService
+  include Payloadable
+
+  def call(account, name, force: true)
+    @account = account
+
+    FeaturedTag.create!(account: account, name: name).tap do |featured_tag|
+      ActivityPub::AccountRawDistributionWorker.perform_async(build_json(featured_tag), account.id) if @account.local?
+    end
+  rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
+    if force && e.is_a(ActiveRecord::RecordNotUnique)
+      FeaturedTag.by_name(name).find_by!(account: account)
+    else
+      account.featured_tags.new(name: name)
+    end
+  end
+
+  private
+
+  def build_json(featured_tag)
+    Oj.dump(serialize_payload(featured_tag, ActivityPub::AddSerializer, signer: @account))
+  end
+end

app/services/remove_featured_tag_service.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class RemoveFeaturedTagService < BaseService
+  include Payloadable
+
+  def call(account, featured_tag)
+    @account = account
+
+    featured_tag.destroy!
+    ActivityPub::AccountRawDistributionWorker.perform_async(build_json(featured_tag), account.id) if @account.local?
+  end
+
+  private
+
+  def build_json(featured_tag)
+    Oj.dump(serialize_payload(featured_tag, ActivityPub::RemoveSerializer, signer: @account))
+  end
+end

app/views/settings/featured_tags/index.html.haml

@@ -21,6 +21,7 @@
     %div
       %h4
         = fa_icon 'hashtag'
+        -# = featured_tag.display_name
         = featured_tag.name
         %small
           - if featured_tag.last_status_at.nil?

app/workers/activitypub/account_raw_distribution_worker.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ActivityPub::AccountRawDistributionWorker < ActivityPub::RawDistributionWorker
+  protected
+
+  def inboxes
+    @inboxes ||= AccountReachFinder.new(@account).inboxes
+  end
+end

app/workers/activitypub/synchronize_featured_collection_worker.rb

@@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker
 
   sidekiq_options queue: 'pull', lock: :until_executed
 
-  def perform(account_id)
+  def perform(account_id, options = {})
-    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id))
+    options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys)
+
+    ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options)
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/activitypub/synchronize_featured_tags_collection_worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class ActivityPub::SynchronizeFeaturedTagsCollectionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull', lock: :until_executed
+
+  def perform(account_id, url)
+    ActivityPub::FetchFeaturedTagsCollectionService.new.call(Account.find(account_id), url)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/activitypub/update_distribution_worker.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
-class ActivityPub::UpdateDistributionWorker
+class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker
-  include Sidekiq::Worker
   include Payloadable
 
-  sidekiq_options queue: 'push'
+  sidekiq_options queue: 'push', lock: :until_executed
 
+  # Distribute an profile update to servers that might have a copy
+  # of the account in question
   def perform(account_id, options = {})
     @options = options.with_indifferent_access
     @account = Account.find(account_id)

app/workers/remove_featured_tag_worker.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RemoveFeaturedTagWorker
+  include Sidekiq::Worker
+
+  def perform(account_id, featured_tag_id)
+    RemoveFeaturedTagService.new.call(Account.find(account_id), FeaturedTag.find(featured_tag_id))
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

db/migrate/20221014111931_add_featured_tags_url_to_account.rb

@@ -0,0 +1,5 @@
+class AddFeaturedTagsURLToAccount < ActiveRecord::Migration[6.1]
+  def change
+    add_column :accounts, :featured_tags_collection_url, :string
+  end
+end

db/schema.rb

@@ -210,6 +210,7 @@ ActiveRecord::Schema.define(version: 2023_01_29_193248) do
     t.jsonb "settings", default: "{}", null: false
     t.integer "silence_mode", default: 0, null: false
     t.integer "searchability", default: 3, null: false
+    t.string "featured_tags_collection_url"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"