Extract counters from accounts table to account_stats table

app/controllers/admin/instances_controller.rb

@@ -31,7 +31,7 @@ def ordered_instances
     end
 
     def subscribeable_accounts
-      Account.with_followers.remote.where(domain: params[:by_domain])
+      Account.remote.where(protocol: :ostatus).where(domain: params[:by_domain])
     end
 
     def filter_params

app/controllers/api/v1/accounts/follower_accounts_controller.rb

@@ -25,7 +25,7 @@ def load_accounts
   end
 
   def default_accounts
-    Account.includes(:active_relationships).references(:active_relationships)
+    Account.includes(:active_relationships, :account_stat).references(:active_relationships)
   end
 
   def paginated_follows

app/controllers/api/v1/accounts/following_accounts_controller.rb

@@ -25,7 +25,7 @@ def load_accounts
   end
 
   def default_accounts
-    Account.includes(:passive_relationships).references(:passive_relationships)
+    Account.includes(:passive_relationships, :account_stat).references(:passive_relationships)
   end
 
   def paginated_follows

app/controllers/api/v1/blocks_controller.rb

@@ -19,7 +19,7 @@ def load_accounts
   end
 
   def paginated_blocks
-    @paginated_blocks ||= Block.eager_load(:target_account)
+    @paginated_blocks ||= Block.eager_load(target_account: :account_stat)
                                .where(account: current_account)
                                .paginate_by_max_id(
                                  limit_param(DEFAULT_ACCOUNTS_LIMIT),

app/controllers/api/v1/endorsements_controller.rb

@@ -27,7 +27,7 @@ def load_accounts
   end
 
   def endorsed_accounts
-    current_account.endorsed_accounts
+    current_account.endorsed_accounts.includes(:account_stat)
   end
 
   def insert_pagination_headers

app/controllers/api/v1/follow_requests_controller.rb

@@ -33,7 +33,7 @@ def load_accounts
   end
 
   def default_accounts
-    Account.includes(:follow_requests).references(:follow_requests)
+    Account.includes(:follow_requests, :account_stat).references(:follow_requests)
   end
 
   def paginated_follow_requests

app/controllers/api/v1/lists/accounts_controller.rb

@@ -37,9 +37,9 @@ def set_list
 
   def load_accounts
     if unlimited?
-      @list.accounts.all
+      @list.accounts.includes(:account_stat).all
     else
-      @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id])
+      @list.accounts.includes(:account_stat).paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id])
     end
   end
 

app/controllers/api/v1/statuses/favourited_by_accounts_controller.rb

@@ -22,7 +22,7 @@ def load_accounts
 
   def default_accounts
     Account
-      .includes(:favourites)
+      .includes(:favourites, :account_stat)
       .references(:favourites)
       .where(favourites: { status_id: @status.id })
   end

app/controllers/api/v1/statuses/reblogged_by_accounts_controller.rb

@@ -21,7 +21,7 @@ def load_accounts
   end
 
   def default_accounts
-    Account.includes(:statuses).references(:statuses)
+    Account.includes(:statuses, :account_stat).references(:statuses)
   end
 
   def paginated_statuses

app/models/account.rb

@@ -32,9 +32,6 @@
 #  suspended               :boolean          default(FALSE), not null
 #  locked                  :boolean          default(FALSE), not null
 #  header_remote_url       :string           default(""), not null
-#  statuses_count          :integer          default(0), not null
-#  followers_count         :integer          default(0), not null
-#  following_count         :integer          default(0), not null
 #  last_webfingered_at     :datetime
 #  inbox_url               :string           default(""), not null
 #  outbox_url              :string           default(""), not null
@@ -58,6 +55,7 @@ class Account < ApplicationRecord
   include AccountInteractions
   include Attachmentable
   include Paginable
+  include AccountCounters
 
   enum protocol: [:ostatus, :activitypub]
 
@@ -119,8 +117,6 @@ class Account < ApplicationRecord
 
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
-  scope :without_followers, -> { where(followers_count: 0) }
-  scope :with_followers, -> { where('followers_count > 0') }
   scope :expiring, ->(time) { remote.where.not(subscription_expires_at: nil).where('subscription_expires_at < ?', time) }
   scope :partitioned, -> { order(Arel.sql('row_number() over (partition by domain)')) }
   scope :silenced, -> { where(silenced: true) }
@@ -385,7 +381,9 @@ def search_for(terms, limit = 10)
         LIMIT ?
       SQL
 
-      find_by_sql([sql, limit])
+      records = find_by_sql([sql, limit])
+      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      records
     end
 
     def advanced_search_for(terms, account, limit = 10, following = false)
@@ -412,7 +410,7 @@ def advanced_search_for(terms, account, limit = 10, following = false)
           LIMIT ?
         SQL
 
-        find_by_sql([sql, account.id, account.id, account.id, limit])
+        records = find_by_sql([sql, account.id, account.id, account.id, limit])
       else
         sql = <<-SQL.squish
           SELECT
@@ -428,8 +426,11 @@ def advanced_search_for(terms, account, limit = 10, following = false)
           LIMIT ?
         SQL
 
-        find_by_sql([sql, account.id, account.id, limit])
+        records = find_by_sql([sql, account.id, account.id, limit])
       end
+
+      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      records
     end
 
     private

app/models/account_stat.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_stats
+#
+#  id              :bigint(8)        not null, primary key
+#  account_id      :bigint(8)        not null
+#  statuses_count  :bigint(8)        default(0), not null
+#  following_count :bigint(8)        default(0), not null
+#  followers_count :bigint(8)        default(0), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#
+
+class AccountStat < ApplicationRecord
+  belongs_to :account, inverse_of: :account_stat
+
+  def increment_count!(key)
+    update(key => public_send(key) + 1)
+  end
+
+  def decrement_count!(key)
+    update(key => [public_send(key) - 1, 0].max)
+  end
+end

app/models/concerns/account_counters.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module AccountCounters
+  extend ActiveSupport::Concern
+
+  included do
+    has_one :account_stat, inverse_of: :account
+    after_save :save_account_stat
+  end
+
+  delegate :statuses_count,
+           :statuses_count=,
+           :following_count,
+           :following_count=,
+           :followers_count,
+           :followers_count=,
+           :increment_count!,
+           :decrement_count!,
+           to: :account_stat
+
+  def account_stat
+    super || build_account_stat
+  end
+
+  private
+
+  def save_account_stat
+    return unless account_stat&.changed?
+    account_stat.save
+  end
+end

app/models/follow.rb

@@ -16,11 +16,8 @@ class Follow < ApplicationRecord
   include Paginable
   include RelationshipCacheable
 
-  belongs_to :account, counter_cache: :following_count
-
-  belongs_to :target_account,
-             class_name: 'Account',
-             counter_cache: :followers_count
+  belongs_to :account
+  belongs_to :target_account, class_name: 'Account'
 
   has_one :notification, as: :activity, dependent: :destroy
 
@@ -39,7 +36,9 @@ def revoke_request!
   end
 
   before_validation :set_uri, only: :create
+  after_create :increment_cache_counters
   after_destroy :remove_endorsements
+  after_destroy :decrement_cache_counters
 
   private
 
@@ -50,4 +49,14 @@ def set_uri
   def remove_endorsements
     AccountPin.where(target_account_id: target_account_id, account_id: account_id).delete_all
   end
+
+  def increment_cache_counters
+    account&.increment_count!(:following_count)
+    target_account&.increment_count!(:followers_count)
+  end
+
+  def decrement_cache_counters
+    account&.decrement_count!(:following_count)
+    target_account&.decrement_count!(:followers_count)
+  end
 end

app/models/notification.rb

@@ -75,7 +75,7 @@ def reload_stale_associations!(cached_items)
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).each_with_object({}) { |a, h| h[a.id] = a }
+      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
 
       cached_items.each do |item|
         item.from_account = accounts[item.from_account_id]

app/models/status.rb

@@ -94,27 +94,27 @@ class Status < ApplicationRecord
     end
   }
 
-  cache_associated :account,
-                   :application,
+  cache_associated :application,
                    :media_attachments,
                    :conversation,
                    :status_stat,
                    :tags,
                    :preview_cards,
                    :stream_entry,
-                   active_mentions: :account,
+                   account: :account_stat,
+                   active_mentions: { account: :account_stat },
                    reblog: [
-                     :account,
                      :application,
                      :stream_entry,
                      :tags,
                      :preview_cards,
                      :media_attachments,
                      :conversation,
                      :status_stat,
-                     active_mentions: :account,
+                     account: :account_stat,
+                     active_mentions: { account: :account_stat },
                    ],
-                   thread: :account
+                   thread: { account: :account_stat }
 
   delegate :domain, to: :account, prefix: true
 
@@ -348,7 +348,7 @@ def reload_stale_associations!(cached_items)
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).each_with_object({}) { |a, h| h[a.id] = a }
+      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
 
       cached_items.each do |item|
         item.account = accounts[item.account_id]
@@ -475,25 +475,15 @@ def update_statistics
   def increment_counter_caches
     return if direct_visibility?
 
-    if association(:account).loaded?
-      account.update_attribute(:statuses_count, account.statuses_count + 1)
-    else
-      Account.where(id: account_id).update_all('statuses_count = COALESCE(statuses_count, 0) + 1')
-    end
-
+    account&.increment_count!(:statuses_count)
     reblog&.increment_count!(:reblogs_count) if reblog?
     thread&.increment_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
   end
 
   def decrement_counter_caches
     return if direct_visibility? || marked_for_mass_destruction?
 
-    if association(:account).loaded?
-      account.update_attribute(:statuses_count, [account.statuses_count - 1, 0].max)
-    else
-      Account.where(id: account_id).update_all('statuses_count = GREATEST(COALESCE(statuses_count, 0) - 1, 0)')
-    end
-
+    account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
     thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
   end

app/presenters/instance_presenter.rb

@@ -22,7 +22,7 @@ def user_count
   end
 
   def status_count
-    Rails.cache.fetch('local_status_count') { Account.local.sum(:statuses_count) }
+    Rails.cache.fetch('local_status_count') { Account.local.joins(:account_stat).sum('account_stats.statuses_count') }
   end
 
   def domain_count

db/migrate/20181116165755_create_account_stats.rb

@@ -0,0 +1,12 @@
+class CreateAccountStats < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_stats do |t|
+      t.belongs_to :account, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true }
+      t.bigint :statuses_count, null: false, default: 0
+      t.bigint :following_count, null: false, default: 0
+      t.bigint :followers_count, null: false, default: 0
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20181116173541_copy_account_stats.rb

@@ -0,0 +1,54 @@
+class CopyAccountStats < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      if supports_upsert?
+        up_fast
+      else
+        up_slow
+      end
+    end
+  end
+
+  def down
+    # Nothing
+  end
+
+  private
+
+  def supports_upsert?
+    version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+    version >= 90500
+  end
+
+  def up_fast
+    say 'Upsert is available, importing counters using the fast method'
+
+    Account.unscoped.select('id').find_in_batches(batch_size: 5_000) do |accounts|
+      execute <<-SQL.squish
+        INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at)
+        SELECT id, statuses_count, following_count, followers_count, created_at, updated_at
+        FROM accounts
+        WHERE id IN (#{accounts.map(&:id).join(', ')})
+        ON CONFLICT (account_id) DO UPDATE
+        SET statuses_count = EXCLUDED.statuses_count, following_count = EXCLUDED.following_count, followers_count = EXCLUDED.followers_count
+      SQL
+    end
+  end
+
+  def up_slow
+    say 'Upsert is not available in PostgreSQL below 9.5, falling back to slow import of counters'
+
+    # We cannot use bulk INSERT or overarching transactions here because of possible
+    # uniqueness violations that we need to skip over
+    Account.unscoped.select('id, statuses_count, following_count, followers_count, created_at, updated_at').find_each do |account|
+      begin
+        params = [[nil, account.id], [nil, account.statuses_count], [nil, account.following_count], [nil, account.followers_count], [nil, account.created_at], [nil, account.updated_at]]
+        exec_insert('INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6)', nil, params)
+      rescue ActiveRecord::RecordNotUnique
+        next
+      end
+    end
+  end
+end