[F] Refactor rate-limiting

* Remove allow2ban banning / filtering for registration, just
  throttle instead.
* Add new settings section for rate limiting
* Add rake tasks to enable/disable/reset settings when necessary
* Standardize rate limiting with categories that can be optionally
  disabled while keeping the rest of the rate limiting active.

api/.rubocop.yml

@@ -183,9 +183,17 @@ Style/GlobalStdStream:
 Style/HashAsLastArrayItem:
   Enabled: true
 
+# False positive with non-hash objects that have a `values` method.
+Style/HashEachMethods:
+  Enabled: false
+
 Style/HashLikeCase:
   Enabled: true
 
+# Guard clauses do not always make more sense.
+Style/IfUnlessModifier:
+  Enabled: false
+
 Style/MultilineBlockChain:
   Enabled: false
 
@@ -270,9 +278,6 @@ Style/ClassAndModuleChildren:
 Style/ExponentialNotation:
   Enabled: true
 
-Style/HashEachMethods:
-  Enabled: true
-
 Style/HashTransformKeys:
   Enabled: true
 

api/Gemfile

@@ -117,6 +117,7 @@ gem "statesman", "~> 3.4"
 gem "statesman-events", "~> 0.0.1"
 gem "store_model", "~> 2.2.0"
 gem "strip_attributes", "~> 1.13.0"
+gem "terminal-table", "~> 3.0.2"
 gem "terrapin", "~> 0.6.0"
 gem "tus-server", "~> 2.0"
 gem "twitter", "~> 7.0"

api/Gemfile.lock

@@ -741,6 +741,8 @@ GEM
       msgpack
       redis
     systemu (2.6.5)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
     test-prof (1.2.3)
@@ -944,6 +946,7 @@ DEPENDENCIES
   statesman-events (~> 0.0.1)
   store_model (~> 2.2.0)
   strip_attributes (~> 1.13.0)
+  terminal-table (~> 3.0.2)
   terrapin (~> 0.6.0)
   test-prof (~> 1.0)
   timecop (~> 0.9)

api/app/models/settings.rb

@@ -15,6 +15,7 @@ class Settings < ApplicationRecord
   attribute :email, SettingSections::Email.to_type, default: {}
   attribute :ingestion, SettingSections::Ingestion.to_type, default: {}
   attribute :integrations, SettingSections::Integrations.to_type, default: {}
+  attribute :rate_limiting, SettingSections::RateLimiting.to_type, default: {}
   attribute :secrets, SettingSections::Secrets.to_type, default: {}
   attribute :theme, SettingSections::Theme.to_type, default: {}
 
@@ -56,6 +57,26 @@ def manifold_analytics_enabled
     include SettingSections::Accessors.new(section)
   end
 
+  # @param [:all, ManifoldEnv::Types::THROTTLED_CATEGORIES] category
+  # @return [void]
+  def disable_rate_limiting!(category)
+    rate_limiting.disable! category
+
+    rate_limiting_will_change!
+
+    save!
+  end
+
+  # @param [:all, ManifoldEnv::Types::THROTTLED_CATEGORIES] category
+  # @return [void]
+  def enable_rate_limiting!(category)
+    rate_limiting.enable! category
+
+    rate_limiting_will_change!
+
+    save!
+  end
+
   # @param [Symbol] section
   # @param [{Symbol => String}] new_values
   # @return [void]

api/app/services/setting_sections.rb

@@ -14,6 +14,7 @@ module SettingSections
     :general,
     :ingestion,
     :integrations,
+    :rate_limiting,
     :secrets,
     :theme,
   ].freeze

api/app/services/setting_sections/rate_limiting.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module SettingSections
+  # Settings related to the rate-limiting subsystem.
+  #
+  # @note The `enabled` option takes precedence over the more granular options
+  #   when set to `false`.
+  class RateLimiting < Base
+    STATUS_TABLE_OPTIONS = {
+      title: "Manifold Rate-limiting Settings",
+      headings: [
+        "Category",
+        { value: "On", alignment: :right },
+        { value: "On (Effective)", alignment: :right }
+      ],
+      style: {
+        width: 78,
+      },
+    }.freeze
+
+    attribute :enabled, :boolean, default: true
+
+    ManifoldEnv::Types::THROTTLED_CATEGORIES.each do |category|
+      attribute category, :boolean, default: true
+    end
+
+    # @param [ManifoldEnv::Types::ThrottledCategory] name
+    def category?(name)
+      name.in?(ManifoldEnv::Types::THROTTLED_CATEGORIES)
+    end
+
+    # @param [ManifoldEnv::Types::ThrottledCategory] name
+    def category_disabled?(name)
+      category?(name) && !__send__(name)
+    end
+
+    # @see #toggle!
+    # @param [ManifoldEnv::Types::ThrottledCategory] category
+    # @return [void]
+    def disable!(category)
+      toggle! category, false
+    end
+
+    # Boolean complement of `enabled`
+    def disabled?
+      !enabled
+    end
+
+    # @param [ManifoldEnv::Types::ThrottledCategory] name
+    def disabled_for?(name)
+      disabled? || category_disabled?(name)
+    end
+
+    # @see #toggle!
+    # @param [ManifoldEnv::Types::ThrottledCategory] category
+    # @return [void]
+    def enable!(category)
+      toggle! category, true
+    end
+
+    # @return [Terminal::Table]
+    def status_table
+      Terminal::Table.new(**STATUS_TABLE_OPTIONS) do |t|
+        status_table_add_global!(t)
+
+        status_table_add_categories!(t)
+      end.tap do |t|
+        status_table_align!(t)
+      end
+    end
+
+    private
+
+    # @param [Boolean] value
+    # @return [String]
+    def boolean_cell(value)
+      value.present? ? ?T : ?F
+    end
+
+    # @param [Terminal::Table] t
+    # @return [void]
+    def status_table_add_categories!(table)
+      ManifoldEnv::Types::THROTTLED_CATEGORIES.each do |category|
+        table << [category.to_s.humanize, boolean_cell(__send__(category)), boolean_cell(!disabled_for?(category))]
+      end
+    end
+
+    # @param [Terminal::Table] table
+    # @return [void]
+    def status_table_add_global!(table)
+      table << ["All", boolean_cell(enabled?), boolean_cell(enabled?)]
+
+      if disabled?
+        table << :separator
+        table << [{ value: "All Categories Disabled Globally", colspan: 3, alignment: :right }]
+      end
+
+      table << :separator
+    end
+
+    # @param [Terminal::Table] table
+    # @return [void]
+    def status_table_align!(table)
+      table.align_column 1, :right
+      table.align_column 2, :right
+    end
+
+    # @param [ManifoldEnv::Types::ThrottledCategory] category
+    # @param [Boolean] value
+    # @return [void]
+    def toggle!(category, value)
+      if category == :all
+        self.enabled = value
+
+        if value
+          ManifoldEnv::Types::THROTTLED_CATEGORIES.each do |name|
+            toggle!(name, value)
+          end
+        end
+      elsif category?(category)
+        self[category] = value
+      else
+        # :nocov:
+        raise ArgumentError, "cannot toggle rate-limiting category #{category.inspect}"
+        # :nocov:
+      end
+    end
+  end
+end

api/config/initializers/80_introspection.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.middleware.insert_before Rack::Attack, ManifoldEnv::Introspector
+end

api/config/initializers/rack_attack.rb

@@ -1,101 +1,60 @@
 # frozen_string_literal: true
 
-require "auth_token"
-
-EMPTY_PARAMS = {}.with_indifferent_access.freeze
-
-JSON_PARAMS_FROM = ->(request) do
-  params = JSON.parse(request.body)
-
-  params.try(:with_indifferent_access) || EMPTY_PARAMS
-rescue JSON::ParserError
-  EMPTY_PARAMS
-ensure
-  request.body.rewind
+# :nocov:
+# We want to ensure that the public IP used by the client is never
+# accidentally blocklisted or throttled.
+unless Rails.env.development? || Rails.env.test?
+  ManifoldEnv.rate_limiting.derive_public_ips! Rails.application.config.manifold.domain
 end
 
-IS_COMMENT_CREATE = ->(request) do
-  request.post? && request.path.include?("/relationships/comments")
+ManifoldEnv.rate_limiting.public_ips.each do |public_ip|
+  Rack::Attack.safelist_ip public_ip
 end
+# :nocov:
 
-IS_PUBLIC_ANNOTATION_CREATE = ->(request) do
-  return false unless request.post? && request.path.include?("/relationships/annotations")
-
-  params = JSON_PARAMS_FROM.(request)
-
-  params.dig("data", "attributes", "private").blank?
-end
-
-IS_PUBLIC_RG_CREATE = ->(request) do
-  return false unless request.post? && request.path.start_with?("/api/v1/reading_groups")
-
-  params = JSON_PARAMS_FROM.(request)
-
-  params.dig("data", "attributes", "privacy") != "private"
+Rack::Attack.safelist("allow all GET requests") do |request|
+  # We do not currently throttle GET requests.
+  request.get?
 end
 
 Rack::Attack.safelist("mark any admin access safe") do |request|
-  AuthToken.authorized_admin?(request.env["HTTP_AUTHORIZATION"])
-end
-
-ANN_LIMITS = { limit: 5, period: 300, }.freeze
-
-Rack::Attack.throttle("public annotation creation by email", **ANN_LIMITS) do |request|
-  AuthToken.real_email_for(request.env["HTTP_AUTHORIZATION"]) if IS_PUBLIC_ANNOTATION_CREATE.(request)
+  request.env["manifold_env.authorized_admin"]
 end
 
-Rack::Attack.throttle("public annotation creation by ip", **ANN_LIMITS) do |request|
-  request.ip if IS_PUBLIC_ANNOTATION_CREATE.(request)
+Rack::Attack.safelist("skip when disabled globally or per category") do |request|
+  request.env["manifold_env.rate_limiting_disabled"]
 end
 
-COMMENT_LIMITS = { limit: 10, period: 3600, }.freeze
-
-Rack::Attack.throttle("comment creation by email", **COMMENT_LIMITS) do |request|
-  AuthToken.real_email_for(request.env["HTTP_AUTHORIZATION"]) if IS_COMMENT_CREATE.(request)
-end
-
-Rack::Attack.throttle("comment creation by ip", **COMMENT_LIMITS) do |request|
-  request.ip if IS_COMMENT_CREATE.(request)
-end
-
-RG_LIMITS = { limit: 10, period: 3600, }.freeze
-
-Rack::Attack.throttle("public reading group creation by email", **RG_LIMITS) do |request|
-  AuthToken.real_email_for(request.env["HTTP_AUTHORIZATION"]) if IS_PUBLIC_RG_CREATE.(request)
-end
-
-Rack::Attack.throttle("public reading group creation by ip", **RG_LIMITS) do |request|
-  request.ip if IS_PUBLIC_RG_CREATE.(request)
-end
-
-Rack::Attack.blocklist("allow2ban registration by email") do |req|
-  params = JSON_PARAMS_FROM.(req)
-
-  real_email = AuthToken.real_email_from(params.dig("data", "attributes", "email"))
-
-  Rack::Attack::Allow2Ban.filter(real_email, maxretry: 5, findtime: 1.day, bantime: 1.month) do
-    req.path.start_with?("/api/v1/users") && req.post?
+ManifoldEnv.rate_limiting.each_throttled_category do |throttler|
+  Rack::Attack.throttle throttler.email_key, **throttler.options do |request|
+    request.env["manifold_env.real_email"] if request.env["manifold_env.throttled_category"] == throttler.category
   end
-end
 
-Rack::Attack.blocklist("allow2ban registration by ip") do |req|
-  Rack::Attack::Allow2Ban.filter(req.ip, maxretry: 5, findtime: 1.day, bantime: 1.month) do
-    req.path.start_with?("/api/v1/users") && req.post?
+  Rack::Attack.throttle throttler.ip_key, **throttler.options do |request|
+    request.ip if request.env["manifold_env.throttled_category"] == throttler.category
   end
 end
 
 ActiveSupport::Notifications.subscribe("blocklist.rack_attack") do |name, start, finish, request_id, payload|
+  # :nocov:
   ThrottledRequest.track! payload[:request]
+  # :nocov:
 end
 
 ActiveSupport::Notifications.subscribe("throttle.rack_attack") do |name, start, finish, request_id, payload|
+  # :nocov:
   ThrottledRequest.track! payload[:request]
+  # :nocov:
 end
 
 Rack::Attack.blocklisted_responder = lambda do |request|
+  # :nocov:
   [503, {}, ["Internal Server Error\n"]]
+  # :nocov:
 end
 
 Rack::Attack.throttled_responder = lambda do |request|
+  # :nocov:
   [503, {}, ["Internal Server Error\n"]]
+  # :nocov:
 end

api/db/migrate/20240327194259_add_rate_limiting_to_settings.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddRateLimitingToSettings < ActiveRecord::Migration[6.1]
+  def change
+    change_table :settings do |t|
+      t.jsonb :rate_limiting, null: false, default: {}
+    end
+  end
+end

api/db/structure.sql

@@ -2426,7 +2426,8 @@ CREATE TABLE public.settings (
     press_logo_mobile_data jsonb,
     favicon_data jsonb,
     fa_cache jsonb DEFAULT '{}'::jsonb NOT NULL,
-    ingestion jsonb DEFAULT '{}'::jsonb
+    ingestion jsonb DEFAULT '{}'::jsonb,
+    rate_limiting jsonb DEFAULT '{}'::jsonb NOT NULL
 );
 
 
@@ -7192,6 +7193,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20231010184158'),
 ('20231129172116'),
 ('20240220212417'),
-('20240223163849');
+('20240223163849'),
+('20240327194259');