Merge branch 'main' into temp_for-ffxiv-mastodon

Gemfile

@@ -35,11 +35,14 @@ group :pam_authentication, optional: true do
 end
 
 gem 'net-ldap', '~> 0.18'
-gem 'omniauth-cas', '~> 2.0'
-gem 'omniauth-saml', '~> 1.10'
+
+# TODO: Point back at released omniauth-cas gem when PR merged
+# https://github.com/dlindahl/omniauth-cas/pull/68
+gem 'omniauth-cas', github: 'stanhu/omniauth-cas', ref: '4211e6d05941b4a981f9a36b49ec166cecd0e271'
+gem 'omniauth-saml', '~> 2.0'
 gem 'omniauth_openid_connect', '~> 0.6.1'
-gem 'omniauth', '~> 1.9'
-gem 'omniauth-rails_csrf_protection', '~> 0.1'
+gem 'omniauth', '~> 2.0'
+gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
 gem 'color_diff', '~> 0.1'
 gem 'discard', '~> 1.2'

Gemfile.lock

@@ -26,6 +26,16 @@ GIT
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
 
+GIT
+  remote: https://github.com/stanhu/omniauth-cas.git
+  revision: 4211e6d05941b4a981f9a36b49ec166cecd0e271
+  ref: 4211e6d05941b4a981f9a36b49ec166cecd0e271
+  specs:
+    omniauth-cas (2.0.0)
+      addressable (~> 2.3)
+      nokogiri (~> 1.5)
+      omniauth (>= 1.2, < 3)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -472,19 +482,16 @@ GEM
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     oj (3.15.0)
-    omniauth (1.9.2)
+    omniauth (2.1.1)
       hashie (>= 3.4.6)
-      rack (>= 1.6.2, < 3)
-    omniauth-cas (2.0.0)
-      addressable (~> 2.3)
-      nokogiri (~> 1.5)
-      omniauth (~> 1.2)
-    omniauth-rails_csrf_protection (0.1.2)
+      rack (>= 2.2.3)
+      rack-protection
+    omniauth-rails_csrf_protection (1.0.1)
       actionpack (>= 4.2)
-      omniauth (>= 1.3.1)
-    omniauth-saml (1.10.3)
-      omniauth (~> 1.3, >= 1.3.2)
-      ruby-saml (~> 1.9)
+      omniauth (~> 2.0)
+    omniauth-saml (2.1.0)
+      omniauth (~> 2.0)
+      ruby-saml (~> 1.12)
     omniauth_openid_connect (0.6.1)
       omniauth (>= 1.9, < 3)
       openid_connect (~> 1.1)
@@ -542,6 +549,8 @@ GEM
       httpclient
       json-jwt (>= 1.11.0)
       rack (>= 2.1.0)
+    rack-protection (3.0.5)
+      rack
     rack-proxy (0.7.6)
       rack
     rack-test (2.1.0)
@@ -873,10 +882,10 @@ DEPENDENCIES
   nokogiri (~> 1.15)
   nsa!
   oj (~> 3.14)
-  omniauth (~> 1.9)
-  omniauth-cas (~> 2.0)
-  omniauth-rails_csrf_protection (~> 0.1)
-  omniauth-saml (~> 1.10)
+  omniauth (~> 2.0)
+  omniauth-cas!
+  omniauth-rails_csrf_protection (~> 1.0)
+  omniauth-saml (~> 2.0)
   omniauth_openid_connect (~> 0.6.1)
   ox (~> 2.14)
   parslet

app/chewy/accounts_index.rb

@@ -33,7 +33,7 @@ class AccountsIndex < Chewy::Index
       },
 
       verbatim: {
-        tokenizer: 'whitespace',
+        tokenizer: 'standard',
         filter: %w(lowercase asciifolding cjk_width),
       },
 

app/controllers/concerns/captcha_concern.rb

@@ -42,7 +42,7 @@ def check_captcha!
   end
 
   def extend_csp_for_captcha!
-    policy = request.content_security_policy
+    policy = request.content_security_policy&.clone
 
     return unless captcha_required? && policy.present?
 
@@ -54,6 +54,8 @@ def extend_csp_for_captcha!
 
       policy.send(directive, *values)
     end
+
+    request.content_security_policy = policy
   end
 
   def render_captcha

app/models/application_record.rb

@@ -5,7 +5,7 @@ class ApplicationRecord < ActiveRecord::Base
 
   include Remotable
 
-  connects_to database: { writing: :primary, reading: ENV['DB_REPLICA_NAME'] || ENV['REPLICA_DATABASE_URL'] ? :replica : :primary }
+  connects_to database: { writing: :primary, reading: ENV['REPLICA_DB_NAME'] || ENV['REPLICA_DATABASE_URL'] ? :replica : :primary }
 
   class << self
     def update_index(_type_name, *_args, &_block)

app/services/account_search_service.rb

@@ -8,6 +8,143 @@ class AccountSearchService < BaseService
   # Min. number of characters to look for non-exact matches
   MIN_QUERY_LENGTH = 5
 
+  class QueryBuilder
+    def initialize(query, account, options = {})
+      @query = query
+      @account = account
+      @options = options
+    end
+
+    def build
+      AccountsIndex.query(
+        bool: {
+          must: {
+            function_score: {
+              query: {
+                bool: {
+                  must: must_clauses,
+                },
+              },
+
+              functions: [
+                reputation_score_function,
+                followers_score_function,
+                time_distance_function,
+              ],
+            },
+          },
+
+          should: should_clauses,
+        }
+      )
+    end
+
+    private
+
+    def must_clauses
+      if @account && @options[:following]
+        [core_query, only_following_query]
+      else
+        [core_query]
+      end
+    end
+
+    def should_clauses
+      if @account && !@options[:following]
+        [boost_following_query]
+      else
+        []
+      end
+    end
+
+    # This function limits results to only the accounts the user is following
+    def only_following_query
+      {
+        terms: {
+          id: following_ids,
+        },
+      }
+    end
+
+    # This function promotes accounts the user is following
+    def boost_following_query
+      {
+        terms: {
+          id: following_ids,
+          boost: 100,
+        },
+      }
+    end
+
+    # This function deranks accounts that follow more people than follow them
+    def reputation_score_function
+      {
+        script_score: {
+          script: {
+            source: "(Math.max(doc['followers_count'].value, 0) + 0.0) / (Math.max(doc['followers_count'].value, 0) + Math.max(doc['following_count'].value, 0) + 1)",
+          },
+        },
+      }
+    end
+
+    # This function promotes accounts that have more followers
+    def followers_score_function
+      {
+        script_score: {
+          script: {
+            source: "(Math.max(doc['followers_count'].value, 0) / (Math.max(doc['followers_count'].value, 0) + 1))",
+          },
+        },
+      }
+    end
+
+    # This function deranks accounts that haven't posted in a long time
+    def time_distance_function
+      {
+        gauss: {
+          last_status_at: {
+            scale: '30d',
+            offset: '30d',
+            decay: 0.3,
+          },
+        },
+      }
+    end
+
+    def following_ids
+      @following_ids ||= @account.active_relationships.pluck(:target_account_id) + [@account.id]
+    end
+  end
+
+  class AutocompleteQueryBuilder < QueryBuilder
+    private
+
+    def core_query
+      {
+        multi_match: {
+          query: @query,
+          type: 'bool_prefix',
+          fields: %w(username username.* display_name display_name.*),
+        },
+      }
+    end
+  end
+
+  class FullQueryBuilder < QueryBuilder
+    private
+
+    def core_query
+      {
+        multi_match: {
+          query: @query,
+          type: 'most_fields',
+          fields: %w(username^2 display_name^2 text text.*),
+          operator: 'and',
+        },
+      }
+    end
+  end
+
   def call(query, account = nil, options = {})
     @query   = query&.strip&.gsub(/\A@/, '')
     @limit   = options[:limit].to_i
@@ -71,27 +208,15 @@ def simple_search_results
   end
 
   def from_elasticsearch
-    must_clauses   = must_clause
-    should_clauses = should_clause
-
-    if account
-      return [] if options[:following] && following_ids.empty?
-
-      if options[:following]
-        must_clauses << { terms: { id: following_ids } }
-      elsif following_ids.any?
-        should_clauses << { terms: { id: following_ids, boost: 100 } }
+    query_builder = begin
+      if options[:use_searchable_text]
+        FullQueryBuilder.new(terms_for_query, account, options.slice(:following))
+      else
+        AutocompleteQueryBuilder.new(terms_for_query, account, options.slice(:following))
       end
     end
 
-    query     = { bool: { must: must_clauses, should: should_clauses } }
-    functions = [reputation_score_function, followers_score_function, time_distance_function]
-
-    records = AccountsIndex.query(function_score: { query: query, functions: functions })
-                           .limit(limit_for_non_exact_results)
-                           .offset(offset)
-                           .objects
-                           .compact
+    records = query_builder.build.limit(limit_for_non_exact_results).offset(offset).objects.compact
 
     ActiveRecord::Associations::Preloader.new(records: records, associations: :account_stat)
 
@@ -100,76 +225,6 @@ def from_elasticsearch
     nil
   end
 
-  def reputation_score_function
-    {
-      script_score: {
-        script: {
-          source: "(Math.max(doc['followers_count'].value, 0) + 0.0) / (Math.max(doc['followers_count'].value, 0) + Math.max(doc['following_count'].value, 0) + 1)",
-        },
-      },
-    }
-  end
-
-  def followers_score_function
-    {
-      script_score: {
-        script: {
-          source: "Math.log10(Math.max(doc['followers_count'].value, 0) + 2)",
-        },
-      },
-    }
-  end
-
-  def time_distance_function
-    {
-      gauss: {
-        last_status_at: {
-          scale: '30d',
-          offset: '30d',
-          decay: 0.3,
-        },
-      },
-    }
-  end
-
-  def must_clause
-    if options[:start_with_hashtag]
-      fields = %w(text text.*)
-    else
-      fields = %w(username username.* display_name display_name.*)
-      fields << 'text' << 'text.*' if options[:use_searchable_text]
-    end
-
-    [
-      {
-        multi_match: {
-          query: terms_for_query,
-          fields: fields,
-          type: 'best_fields',
-          operator: 'or',
-        },
-      },
-    ]
-  end
-
-  def should_clause
-    [
-      {
-        multi_match: {
-          query: terms_for_query,
-          fields: %w(username username.* display_name display_name.*),
-          type: 'best_fields',
-          operator: 'and',
-          boost: 10,
-        },
-      },
-    ]
-  end
-
-  def following_ids
-    @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id]
-  end
-
   def limit_for_non_exact_results
     return 0 if @account.nil? && query.size < MIN_QUERY_LENGTH
 

config/database.yml

@@ -55,10 +55,10 @@ production:
     prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %>
   replica:
     <<: *default
-    database: <%= ENV['DB_REPLICA_NAME'] ||ENV['DB_NAME'] || 'mastodon_production' %>
-    username: <%= ENV['DB_REPLICA_USER'] ||ENV['DB_USER'] || 'mastodon' %>
-    password: <%= (ENV['DB_REPLICA_PASS'] || ENV['DB_PASS'] || '').to_json %>
-    host: <%= ENV['DB_REPLICA_HOST'] ||ENV['DB_HOST'] || 'localhost' %>
-    port: <%= ENV['DB_REPLICA_PORT'] ||ENV['DB_PORT'] || 5432 %>
+    database: <%= ENV['REPLICA_DB_NAME'] ||ENV['DB_NAME'] || 'mastodon_production' %>
+    username: <%= ENV['REPLICA_DB_USER'] ||ENV['DB_USER'] || 'mastodon' %>
+    password: <%= (ENV['REPLICA_DB_PASS'] || ENV['DB_PASS'] || '').to_json %>
+    host: <%= ENV['REPLICA_DB_HOST'] ||ENV['DB_HOST'] || 'localhost' %>
+    port: <%= ENV['REPLICA_DB_PORT'] ||ENV['DB_PORT'] || 5432 %>
     prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %>
     replica: true