Upstream Journaled::AuditLog feature to rubygems release (#26)

This upstreams Betterment's `Journaled::AuditLog` mixin (for ActiveRecord models) into the public rubygems release, making it easier for Betterment to consume it (and also making it available for others to use).

It's similar to other audit logging / papertrail-like gems, in that it will record changes to models (insert/update/delete). But instead of storing these changes in a local `versions` table (etc), it will emit them in the form of journaled events.

I've updated the README with some of the details (including how sensitive/encrypted fields are handled, etc). This pairs with the [5.0 release](https://github.com/Betterment/journaled/releases/tag/v5.0.0) that introduced transactionally-batched journaling. (So, if you're changing several records at once within the scope of a single transaction, you'll only end up enqueuing 1 journaled event job).

app/models/concerns/journaled/changes.rb

@@ -56,7 +56,7 @@ def update_columns(attributes, opts = { force: false })
   end
 
   class_methods do
-    def journal_changes_to(*attribute_names, as:, enqueue_with: {}) # rubocop:disable Naming/MethodParameterName
+    def journal_changes_to(*attribute_names, as:, enqueue_with: {})
       if attribute_names.empty? || attribute_names.any? { |n| !n.is_a?(Symbol) }
         raise "one or more symbol attribute_name arguments is required"
       end

app/models/journaled/audit_log/event.rb

@@ -0,0 +1,87 @@
+# FIXME: This cannot be included in lib/ because Journaled::Event is autoloaded via app/models
+#        Autoloading Journaled::Event isn't strictly necessary, and for compatibility it would
+#        make sense to move it to lib/.
+module Journaled
+  module AuditLog
+    Event = Struct.new(:record, :database_operation, :unfiltered_changes) do
+      include Journaled::Event
+
+      journal_attributes :class_name, :table_name, :record_id,
+                         :database_operation, :changes, :snapshot, :actor, tagged: true
+
+      def journaled_stream_name
+        AuditLog.default_stream_name || super
+      end
+
+      def created_at
+        case database_operation
+          when 'insert'
+            record_created_at
+          when 'update'
+            record_updated_at
+          when 'delete'
+            Time.zone.now
+          else
+            raise "Unhandled database operation type: #{database_operation}"
+        end
+      end
+
+      def record_created_at
+        record.try(:created_at) || Time.zone.now
+      end
+
+      def record_updated_at
+        record.try(:updated_at) || Time.zone.now
+      end
+
+      def class_name
+        record.class.name
+      end
+
+      def table_name
+        record.class.table_name
+      end
+
+      def record_id
+        record.id
+      end
+
+      def changes
+        filtered_changes = unfiltered_changes.deep_dup.deep_symbolize_keys
+        filtered_changes.each do |key, value|
+          filtered_changes[key] = value.map { |val| '[FILTERED]' if val } if filter_key?(key)
+        end
+      end
+
+      def snapshot
+        filtered_attributes if record._log_snapshot || AuditLog.snapshots_enabled
+      end
+
+      def actor
+        Journaled.actor_uri
+      end
+
+      private
+
+      def filter_key?(key)
+        filter_params.include?(key) || encrypted_column?(key)
+      end
+
+      def encrypted_column?(key)
+        key.to_s.end_with?('_crypt', '_hmac') ||
+          (Rails::VERSION::MAJOR >= 7 && record.encrypted_attribute?(key))
+      end
+
+      def filter_params
+        Rails.application.config.filter_parameters
+      end
+
+      def filtered_attributes
+        attrs = record.attributes.dup.symbolize_keys
+        attrs.each do |key, _value|
+          attrs[key] = '[FILTERED]' if filter_key?(key)
+        end
+      end
+    end
+  end
+end

journaled.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency "activejob"
   s.add_dependency "activerecord"
+  s.add_dependency "activesupport"
   s.add_dependency "aws-sdk-kinesis", "< 2"
   s.add_dependency "json-schema"
   s.add_dependency "railties", ">= 5.2"

journaled_schemas/journaled/audit_log/event.json

@@ -0,0 +1,31 @@
+{
+  "type": "object",
+  "title": "audit_log_event",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "event_type",
+    "created_at",
+    "class_name",
+    "table_name",
+    "record_id",
+    "database_operation",
+    "changes",
+    "snapshot",
+    "actor",
+    "tags"
+  ],
+  "properties": {
+    "id": { "type": "string" },
+    "event_type": { "type": "string" },
+    "created_at": { "type": "string" },
+    "class_name": { "type": "string" },
+    "table_name": { "type": "string" },
+    "record_id": { "type": ["string", "integer"] },
+    "database_operation": { "type": "string" },
+    "changes": { "type": "object", "additionalProperties": true },
+    "snapshot": { "type": ["object", "null"], "additionalProperties": true },
+    "actor": { "type": "string" },
+    "tags": { "type": "object", "additionalProperties": true }
+  }
+}

lib/journaled.rb

@@ -69,3 +69,5 @@ def self.tag!(**tags)
     Current.tags = Current.tags.merge(tags)
   end
 end
+
+require 'journaled/audit_log'

lib/journaled/audit_log.rb

@@ -0,0 +1,194 @@
+require 'active_support/core_ext/module/attribute_accessors_per_thread'
+
+module Journaled
+  module AuditLog
+    extend ActiveSupport::Concern
+
+    DEFAULT_EXCLUDED_CLASSES = %w(
+      Delayed::Job
+      PaperTrail::Version
+      ActiveStorage::Attachment
+      ActiveStorage::Blob
+      ActiveRecord::InternalMetadata
+      ActiveRecord::SchemaMigration
+    ).freeze
+
+    mattr_accessor(:default_ignored_columns) { %i(created_at updated_at) }
+    mattr_accessor(:default_stream_name) { Journaled.default_stream_name }
+    mattr_accessor(:excluded_classes) { DEFAULT_EXCLUDED_CLASSES.dup }
+    thread_mattr_accessor(:snapshots_enabled) { false }
+    thread_mattr_accessor(:_disabled) { false }
+    thread_mattr_accessor(:_force) { false }
+
+    class << self
+      def exclude_classes!
+        excluded_classes.each do |name|
+          if Rails::VERSION::MAJOR >= 6 && Rails.autoloaders.zeitwerk_enabled?
+            zeitwerk_exclude!(name)
+          else
+            classic_exclude!(name)
+          end
+        end
+      end
+
+      def with_snapshots
+        snapshots_enabled_was = snapshots_enabled
+        self.snapshots_enabled = true
+        yield
+      ensure
+        self.snapshots_enabled = snapshots_enabled_was
+      end
+
+      def without_audit_logging
+        disabled_was = _disabled
+        self._disabled = true
+        yield
+      ensure
+        self._disabled = disabled_was
+      end
+
+      private
+
+      def zeitwerk_exclude!(name)
+        if Object.const_defined?(name)
+          name.constantize.skip_audit_log
+        else
+          Rails.autoloaders.main.on_load(name) { |klass, _path| klass.skip_audit_log }
+        end
+      end
+
+      def classic_exclude!(name)
+        name.constantize.skip_audit_log
+      rescue NameError
+        nil
+      end
+    end
+
+    Config = Struct.new(:enabled, :ignored_columns) do
+      private :enabled
+      def enabled?
+        !AuditLog._disabled && self[:enabled].present?
+      end
+    end
+
+    included do
+      prepend BlockedMethods
+      singleton_class.prepend BlockedClassMethods
+
+      class_attribute :audit_log_config, default: Config.new(false, AuditLog.default_ignored_columns)
+      attr_accessor :_log_snapshot
+
+      after_create { _emit_audit_log!('insert') }
+      after_update { _emit_audit_log!('update') if _audit_log_changes.any? }
+      after_destroy { _emit_audit_log!('delete') }
+    end
+
+    class_methods do
+      def has_audit_log(ignore: [])
+        ignored_columns = _audit_log_inherited_ignored_columns + [ignore].flatten(1)
+        self.audit_log_config = Config.new(true, ignored_columns.uniq)
+      end
+
+      def skip_audit_log
+        self.audit_log_config = Config.new(false, _audit_log_inherited_ignored_columns.uniq)
+      end
+
+      private
+
+      def _audit_log_inherited_ignored_columns
+        (superclass.try(:audit_log_config)&.ignored_columns || []) + audit_log_config.ignored_columns
+      end
+    end
+
+    module BlockedMethods
+      BLOCKED_METHODS = {
+        delete: '#destroy',
+        update_column: '#update!',
+        update_columns: '#update!',
+      }.freeze
+
+      def delete(**kwargs)
+        _journaled_audit_log_check!(:delete, **kwargs) do
+          super()
+        end
+      end
+
+      def update_column(name, value, **kwargs)
+        _journaled_audit_log_check!(:update_column, **kwargs.merge(name => value)) do
+          super(name, value)
+        end
+      end
+
+      def update_columns(args = {}, **kwargs)
+        _journaled_audit_log_check!(:update_columns, **args.merge(kwargs)) do
+          super(args.merge(kwargs).except(:_force))
+        end
+      end
+
+      def _journaled_audit_log_check!(method, **kwargs) # rubocop:disable Metrics/AbcSize
+        force_was = AuditLog._force
+        AuditLog._force = kwargs.delete(:_force) if kwargs.key?(:_force)
+        audited_columns = kwargs.keys - audit_log_config.ignored_columns
+
+        if method == :delete || audited_columns.any?
+          column_message = <<~MSG if kwargs.any?
+            You are attempting to change the following audited columns:
+              #{audited_columns.inspect}
+
+          MSG
+          raise <<~MSG if audit_log_config.enabled? && !AuditLog._force
+            #{column_message}Using `#{method}` is blocked because it skips audit logging (and other Rails callbacks)!
+            Consider using `#{BLOCKED_METHODS[method]}` instead, or pass `_force: true` as an argument.
+          MSG
+        end
+
+        yield
+      ensure
+        AuditLog._force = force_was
+      end
+    end
+
+    module BlockedClassMethods
+      BLOCKED_METHODS = {
+        delete_all: '.destroy_all',
+        insert: '.create!',
+        insert_all: '.each { create!(...) }',
+        update_all: '.find_each { update!(...) }',
+        upsert: '.create_or_find_by!',
+        upsert_all: '.each { create_or_find_by!(...) }',
+      }.freeze
+
+      BLOCKED_METHODS.each do |method, alternative|
+        define_method(method) do |*args, **kwargs, &block|
+          force_was = AuditLog._force
+          AuditLog._force = kwargs.delete(:_force) if kwargs.key?(:_force)
+
+          raise <<~MSG if audit_log_config.enabled? && !AuditLog._force
+            `#{method}` is blocked because it skips callbacks and audit logs!
+            Consider using `#{alternative}` instead, or pass `_force: true` as an argument.
+          MSG
+
+          super(*args, **kwargs, &block)
+        ensure
+          AuditLog._force = force_was
+        end
+      end
+    end
+
+    def _emit_audit_log!(database_operation)
+      if audit_log_config.enabled?
+        event = Journaled::AuditLog::Event.new(self, database_operation, _audit_log_changes)
+        ActiveSupport::Notifications.instrument('journaled.audit_log.journal', event: event) do
+          event.journal!
+        end
+      end
+    end
+
+    def _audit_log_changes
+      previous_changes.except(*audit_log_config.ignored_columns)
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) { include Journaled::AuditLog }
+Journaled::Engine.config.after_initialize { Journaled::AuditLog.exclude_classes! }

lib/journaled/version.rb

@@ -1,3 +1,3 @@
 module Journaled
-  VERSION = "5.0.0".freeze
+  VERSION = "5.1.0".freeze
 end

spec/dummy/config/application.rb

@@ -10,16 +10,8 @@
 
 module Dummy
   class Application < Rails::Application
-    # Settings in config/environments/* take precedence over those specified here.
-    # Application configuration should go into files in config/initializers
-    # -- all .rb files in that directory are automatically loaded.
-
-    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
-    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
-    # config.time_zone = 'Central Time (US & Canada)'
-
-    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
-    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
-    # config.i18n.default_locale = :de
+    config.autoloader = Rails::VERSION::MAJOR >= 7 ? :zeitwerk : :classic
+    config.active_record.sqlite3.represent_boolean_as_integer = true if Rails::VERSION::MAJOR < 6
+    config.active_record.legacy_connection_handling = false if Rails::VERSION::MAJOR >= 7
   end
 end