From a2a6331451ae32c7936e97691e0f2ab52e8486fc Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Wed, 22 Mar 2023 16:37:46 -0500 Subject: [PATCH] Add ActiveSupport::MessagePack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ActiveSupport::MessagePack` is a serializer that integrates with the `msgpack` gem to serialize a variety of Ruby objects. `AS::MessagePack` supports several types beyond the base types that `msgpack` supports, including `Time` and `Range`, as well as Active Support types such as `AS::TimeWithZone` and `AS::HashWithIndifferentAccess`. Compared to `JSON` and `Marshal`, `AS::MessagePack` can provide a performance improvement and message size reduction. For example, when used with `MessageVerifier`: ```ruby # frozen_string_literal: true require "benchmark/ips" require "active_support/all" require "active_support/message_pack" marshal_verifier = ActiveSupport::MessageVerifier.new("secret", serializer: Marshal) json_verifier = ActiveSupport::MessageVerifier.new("secret", serializer: JSON) asjson_verifier = ActiveSupport::MessageVerifier.new("secret", serializer: ActiveSupport::JSON) msgpack_verifier = ActiveSupport::MessageVerifier.new("secret", serializer: ActiveSupport::MessagePack) ActiveSupport::Messages::Metadata.use_message_serializer_for_metadata = true expiry = 1.year.from_now data = { bool: true, num: 123456789, string: "x" * 50 } Benchmark.ips do |x| x.report("Marshal") do marshal_verifier.verify(marshal_verifier.generate(data, expires_at: expiry)) end x.report("JSON") do json_verifier.verify(json_verifier.generate(data, expires_at: expiry)) end x.report("AS::JSON") do asjson_verifier.verify(asjson_verifier.generate(data, expires_at: expiry)) end x.report("MessagePack") do msgpack_verifier.verify(msgpack_verifier.generate(data, expires_at: expiry)) end x.compare! end puts "Marshal size: #{marshal_verifier.generate(data, expires_at: expiry).bytesize}" puts "JSON size: #{json_verifier.generate(data, expires_at: expiry).bytesize}" puts "MessagePack size: #{msgpack_verifier.generate(data, expires_at: expiry).bytesize}" ``` ``` Warming up -------------------------------------- Marshal 1.206k i/100ms JSON 1.165k i/100ms AS::JSON 790.000 i/100ms MessagePack 1.798k i/100ms Calculating ------------------------------------- Marshal 11.748k (± 1.3%) i/s - 59.094k in 5.031071s JSON 11.498k (± 1.4%) i/s - 58.250k in 5.066957s AS::JSON 7.867k (± 2.4%) i/s - 39.500k in 5.024055s MessagePack 17.865k (± 0.8%) i/s - 89.900k in 5.032592s Comparison: MessagePack: 17864.9 i/s Marshal: 11747.8 i/s - 1.52x (± 0.00) slower JSON: 11498.4 i/s - 1.55x (± 0.00) slower AS::JSON: 7866.9 i/s - 2.27x (± 0.00) slower Marshal size: 254 JSON size: 234 MessagePack size: 194 ``` Additionally, `ActiveSupport::MessagePack::CacheSerializer` is a serializer that is suitable for use as an `ActiveSupport::Cache` coder. `AS::MessagePack::CacheSerializer` can serialize `ActiveRecord::Base` instances, including loaded associations. Like `AS::MessagePack`, it provides a performance improvement and payload size reduction: ```ruby # frozen_string_literal: true require "benchmark/ips" require "active_support/message_pack" ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:") ActiveRecord::Schema.define do create_table :posts, force: true do |t| t.string :body t.timestamps end create_table :comments, force: true do |t| t.integer :post_id t.string :body t.timestamps end end class Post < ActiveRecord::Base has_many :comments end class Comment < ActiveRecord::Base belongs_to :post end post = Post.create!(body: "x" * 100) 2.times { post.comments.create!(body: "x" * 100) } post.comments.load cache_entry = ActiveSupport::Cache::Entry.new(post) Rails70Coder = ActiveSupport::Cache::Coders::Rails70Coder CacheSerializer = ActiveSupport::MessagePack::CacheSerializer Benchmark.ips do |x| x.report("Rails70Coder") do Rails70Coder.load(Rails70Coder.dump(cache_entry)) end x.report("CacheSerializer") do CacheSerializer.load(CacheSerializer.dump(cache_entry)) end x.compare! end puts "Rails70Coder size: #{Rails70Coder.dump(cache_entry).bytesize}" puts "CacheSerializer size: #{CacheSerializer.dump(cache_entry).bytesize}" ``` ``` Warming up -------------------------------------- Rails70Coder 329.000 i/100ms CacheSerializer 492.000 i/100ms Calculating ------------------------------------- Rails70Coder 3.285k (± 1.7%) i/s - 16.450k in 5.008447s CacheSerializer 4.895k (± 2.4%) i/s - 24.600k in 5.028803s Comparison: CacheSerializer: 4894.7 i/s Rails70Coder: 3285.4 i/s - 1.49x slower Rails70Coder size: 808 CacheSerializer size: 593 ``` Co-authored-by: Jean Boussier --- Gemfile | 1 + Gemfile.lock | 3 +- .../lib/active_record/message_pack.rb | 124 ++++++++ activerecord/lib/active_record/railtie.rb | 9 + activerecord/test/cases/message_pack_test.rb | 89 ++++++ .../lib/active_support/message_pack.rb | 50 +++ .../message_pack/cache_serializer.rb | 44 +++ .../active_support/message_pack/extensions.rb | 292 ++++++++++++++++++ .../active_support/message_pack/serializer.rb | 63 ++++ .../lib/active_support/messages/metadata.rb | 19 +- .../message_pack/cache_serializer_test.rb | 122 ++++++++ .../test/message_pack/serializer_test.rb | 22 ++ .../message_pack/shared_serializer_tests.rb | 157 ++++++++++ .../test/messages/message_metadata_tests.rb | 2 + .../initializers/frameworks_test.rb | 17 + 15 files changed, 1010 insertions(+), 4 deletions(-) create mode 100644 activerecord/lib/active_record/message_pack.rb create mode 100644 activerecord/test/cases/message_pack_test.rb create mode 100644 activesupport/lib/active_support/message_pack.rb create mode 100644 activesupport/lib/active_support/message_pack/cache_serializer.rb create mode 100644 activesupport/lib/active_support/message_pack/extensions.rb create mode 100644 activesupport/lib/active_support/message_pack/serializer.rb create mode 100644 activesupport/test/message_pack/cache_serializer_test.rb create mode 100644 activesupport/test/message_pack/serializer_test.rb create mode 100644 activesupport/test/message_pack/shared_serializer_tests.rb diff --git a/Gemfile b/Gemfile index bd91a5a41366e..5d2b2c4a67b03 100644 --- a/Gemfile +++ b/Gemfile @@ -68,6 +68,7 @@ gem "listen", "~> 3.3", require: false gem "libxml-ruby", platforms: :ruby gem "connection_pool", require: false gem "rexml", require: false +gem "msgpack", ">= 1.7.0", require: false # for railties gem "bootsnap", ">= 1.4.4", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 9039bb8adb956..8bb8cdcfdeaeb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -320,7 +320,7 @@ GEM mixlib-shellout (3.2.7) chef-utils mono_logger (1.1.1) - msgpack (1.6.0) + msgpack (1.7.0) multi_json (1.15.0) multipart-post (2.2.3) mustermann (3.0.0) @@ -580,6 +580,7 @@ DEPENDENCIES minitest-bisect minitest-ci minitest-retry + msgpack (>= 1.7.0) mysql2 (~> 0.5) nokogiri (>= 1.8.1, != 1.11.0) pg (~> 1.3) diff --git a/activerecord/lib/active_record/message_pack.rb b/activerecord/lib/active_record/message_pack.rb new file mode 100644 index 0000000000000..b10f4ef6ea02a --- /dev/null +++ b/activerecord/lib/active_record/message_pack.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +module ActiveRecord + module MessagePack # :nodoc: + FORMAT_VERSION = 1 + + class << self + def dump(input) + encoder = Encoder.new + [FORMAT_VERSION, encoder.encode(input), encoder.entries] + end + + def load(dumped) + format_version, top_level, entries = dumped + unless format_version == FORMAT_VERSION + raise "Invalid format version: #{format_version.inspect}" + end + Decoder.new(entries).decode(top_level) + end + end + + module Extensions + extend self + + def install(registry) + registry.register_type 119, ActiveModel::Type::Binary::Data, + packer: :to_s, + unpacker: :new + + registry.register_type 120, ActiveRecord::Base, + packer: method(:write_record), + unpacker: method(:read_record), + recursive: true + end + + def write_record(record, packer) + packer.write(ActiveRecord::MessagePack.dump(record)) + end + + def read_record(unpacker) + ActiveRecord::MessagePack.load(unpacker.read) + end + end + + class Encoder + attr_reader :entries + + def initialize + @entries = [] + @refs = {}.compare_by_identity + end + + def encode(input) + if input.is_a?(Array) + input.map { |record| encode_record(record) } + elsif input + encode_record(input) + end + end + + def encode_record(record) + ref = @refs[record] + + if !ref + ref = @refs[record] = @entries.size + @entries << build_entry(record) + add_cached_associations(record, @entries.last) + end + + ref + end + + def build_entry(record) + [ + ActiveSupport::MessagePack::Extensions.dump_class(record.class), + record.attributes_for_database, + record.new_record? + ] + end + + def add_cached_associations(record, entry) + record.class.reflections.each_value do |reflection| + if record.association_cached?(reflection.name) + entry << reflection.name << encode(record.association(reflection.name).target) + end + end + end + end + + class Decoder + def initialize(entries) + @records = entries.map { |entry| build_record(entry) } + @records.zip(entries) { |record, entry| resolve_cached_associations(record, entry) } + end + + def decode(ref) + if ref.is_a?(Array) + ref.map { |r| @records[r] } + elsif ref + @records[ref] + end + end + + def build_record(entry) + class_name, attributes_hash, is_new_record, * = entry + klass = ActiveSupport::MessagePack::Extensions.load_class(class_name) + attributes = klass.attributes_builder.build_from_database(attributes_hash) + klass.allocate.init_with_attributes(attributes, is_new_record) + end + + def resolve_cached_associations(record, entry) + i = 3 # entry == [class_name, attributes_hash, is_new_record, *associations] + while i < entry.length + begin + record.association(entry[i]).target = decode(entry[i + 1]) + rescue ActiveRecord::AssociationNotFoundError + # The association no longer exists, so just skip it. + end + i += 2 + end + end + end + end +end diff --git a/activerecord/lib/active_record/railtie.rb b/activerecord/lib/active_record/railtie.rb index 5500da68e1c93..a10464274bf66 100644 --- a/activerecord/lib/active_record/railtie.rb +++ b/activerecord/lib/active_record/railtie.rb @@ -424,5 +424,14 @@ class Railtie < Rails::Railtie # :nodoc: end end end + + initializer "active_record.message_pack" do + ActiveSupport.on_load(:message_pack) do + ActiveSupport.on_load(:active_record) do + require "active_record/message_pack" + ActiveRecord::MessagePack::Extensions.install(ActiveSupport::MessagePack::CacheSerializer) + end + end + end end end diff --git a/activerecord/test/cases/message_pack_test.rb b/activerecord/test/cases/message_pack_test.rb new file mode 100644 index 0000000000000..1da01db060770 --- /dev/null +++ b/activerecord/test/cases/message_pack_test.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require "cases/helper" +require "models/author" +require "models/binary" +require "models/comment" +require "models/post" +require "active_support/message_pack" +require "active_record/message_pack" + +class ActiveRecordMessagePackTest < ActiveRecord::TestCase + test "enshrines type IDs" do + expected = { + 119 => ActiveModel::Type::Binary::Data, + 120 => ActiveRecord::Base, + } + + factory = ::MessagePack::Factory.new + ActiveRecord::MessagePack::Extensions.install(factory) + actual = factory.registered_types.to_h do |entry| + [entry[:type], entry[:class]] + end + + assert_equal expected, actual + end + + test "roundtrips record and cached associations" do + post = Post.create!(title: "A Title", body: "A body.") + post.create_author!(name: "An Author") + post.comments.create!(body: "A comment.") + post.comments.create!(body: "Another comment.", author: post.author) + post.comments.load + + assert_no_queries do + roundtripped_post = roundtrip(post) + + assert_equal post, roundtripped_post + assert_equal post.author, roundtripped_post.author + assert_equal post.comments.to_a, roundtripped_post.comments.to_a + assert_equal post.comments.map(&:author), roundtripped_post.comments.map(&:author) + + assert_same roundtripped_post, roundtripped_post.comments[0].post + assert_same roundtripped_post, roundtripped_post.comments[1].post + assert_same roundtripped_post.author, roundtripped_post.comments[1].author + end + end + + test "roundtrips new_record? status" do + post = Post.new(title: "A Title", body: "A body.") + post.create_author!(name: "An Author") + + assert_no_queries do + roundtripped_post = roundtrip(post) + + assert_equal post.attributes, roundtripped_post.attributes + assert_equal post.new_record?, roundtripped_post.new_record? + assert_equal post.author, roundtripped_post.author + assert_equal post.author.new_record?, roundtripped_post.author.new_record? + end + end + + test "roundtrips binary attribute" do + binary = Binary.new(data: Marshal.dump("data")) + assert_equal binary.attributes, roundtrip(binary).attributes + end + + test "raises ActiveSupport::MessagePack::MissingClassError if record class no longer exists" do + klass = Class.new(Post) + def klass.name; "SomeLegacyClass"; end + dumped = serializer.dump(klass.new(title: "A Title", body: "A body.")) + + assert_raises ActiveSupport::MessagePack::MissingClassError do + serializer.load(dumped) + end + end + + private + def serializer + @serializer ||= ::MessagePack::Factory.new.tap do |factory| + ActiveRecord::MessagePack::Extensions.install(factory) + ActiveSupport::MessagePack::Extensions.install(factory) + ActiveSupport::MessagePack::Extensions.install_unregistered_type_error(factory) + end + end + + def roundtrip(input) + serializer.load(serializer.dump(input)) + end +end diff --git a/activesupport/lib/active_support/message_pack.rb b/activesupport/lib/active_support/message_pack.rb new file mode 100644 index 0000000000000..3ce2c07b5caa0 --- /dev/null +++ b/activesupport/lib/active_support/message_pack.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +begin + gem "msgpack", ">= 1.7.0" + require "msgpack" +rescue LoadError => error + warn "ActiveSupport::MessagePack requires the msgpack gem, version 1.7.0 or later. " \ + "Please add it to your Gemfile: `gem \"msgpack\", \">= 1.7.0\"`" + raise error +end + +require_relative "message_pack/cache_serializer" +require_relative "message_pack/serializer" + +module ActiveSupport + module MessagePack + extend Serializer + + ## + # :singleton-method: dump + # :call-seq: dump(object) + # + # Dumps an object. Raises ActiveSupport::MessagePack::UnserializableObjectError + # if the object type is not supported. + # + #-- + # Implemented by Serializer#dump. + + ## + # :singleton-method: load + # :call-seq: load(dumped) + # + # Loads an object dump created by ::dump. + # + #-- + # Implemented by Serializer#load. + + ## + # :singleton-method: signature? + # :call-seq: signature?(dumped) + # + # Returns true if the given dump begins with an +ActiveSupport::MessagePack+ + # signature. + # + #-- + # Implemented by Serializer#signature?. + + ActiveSupport.run_load_hooks(:message_pack, self) + end +end diff --git a/activesupport/lib/active_support/message_pack/cache_serializer.rb b/activesupport/lib/active_support/message_pack/cache_serializer.rb new file mode 100644 index 0000000000000..abea5b1c8e72c --- /dev/null +++ b/activesupport/lib/active_support/message_pack/cache_serializer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "serializer" + +module ActiveSupport + module MessagePack + module CacheSerializer + include Serializer + extend self + + ZLIB_HEADER = "\x78" + + def dump(entry) + super(entry.pack) + end + + def dump_compressed(entry, threshold) # :nodoc: + dumped = dump(entry) + if dumped.bytesize >= threshold + compressed = Zlib::Deflate.deflate(dumped) + compressed.bytesize < dumped.bytesize ? compressed : dumped + else + dumped + end + end + + def load(dumped) + dumped = Zlib::Inflate.inflate(dumped) if compressed?(dumped) + ActiveSupport::Cache::Entry.unpack(super) + rescue ActiveSupport::MessagePack::MissingClassError + # Treat missing class as cache miss => return nil + end + + private + def compressed?(dumped) + dumped.start_with?(ZLIB_HEADER) + end + + def install_unregistered_type_handler + Extensions.install_unregistered_type_fallback(message_pack_factory) + end + end + end +end diff --git a/activesupport/lib/active_support/message_pack/extensions.rb b/activesupport/lib/active_support/message_pack/extensions.rb new file mode 100644 index 0000000000000..3cd7b880a6fd7 --- /dev/null +++ b/activesupport/lib/active_support/message_pack/extensions.rb @@ -0,0 +1,292 @@ +# frozen_string_literal: true + +require "bigdecimal" +require "date" +require "ipaddr" +require "pathname" +require "uri/generic" +require "msgpack/bigint" +require "active_support/hash_with_indifferent_access" +require "active_support/time" + +module ActiveSupport + module MessagePack + class UnserializableObjectError < StandardError; end + class MissingClassError < StandardError; end # :nodoc: + + module Extensions # :nodoc: + extend self + + def install(registry) + registry.register_type 0, Symbol, + packer: :to_msgpack_ext, + unpacker: :from_msgpack_ext, + optimized_symbols_parsing: true + + registry.register_type 1, Integer, + packer: ::MessagePack::Bigint.method(:to_msgpack_ext), + unpacker: ::MessagePack::Bigint.method(:from_msgpack_ext), + oversized_integer_extension: true + + registry.register_type 2, BigDecimal, + packer: :_dump, + unpacker: :_load + + registry.register_type 3, Rational, + packer: method(:write_rational), + unpacker: method(:read_rational), + recursive: true + + registry.register_type 4, Complex, + packer: method(:write_complex), + unpacker: method(:read_complex), + recursive: true + + registry.register_type 5, DateTime, + packer: method(:write_datetime), + unpacker: method(:read_datetime), + recursive: true + + registry.register_type 6, Date, + packer: method(:write_date), + unpacker: method(:read_date), + recursive: true + + registry.register_type 7, Time, + packer: method(:write_time), + unpacker: method(:read_time), + recursive: true + + registry.register_type 8, ActiveSupport::TimeWithZone, + packer: method(:write_time_with_zone), + unpacker: method(:read_time_with_zone), + recursive: true + + registry.register_type 9, ActiveSupport::TimeZone, + packer: method(:dump_time_zone), + unpacker: method(:load_time_zone) + + registry.register_type 10, ActiveSupport::Duration, + packer: method(:write_duration), + unpacker: method(:read_duration), + recursive: true + + registry.register_type 11, Range, + packer: method(:write_range), + unpacker: method(:read_range), + recursive: true + + registry.register_type 12, Set, + packer: method(:write_set), + unpacker: method(:read_set), + recursive: true + + registry.register_type 13, URI::Generic, + packer: :to_s, + unpacker: URI.method(:parse) + + registry.register_type 14, IPAddr, + packer: :to_s, + unpacker: :new + + registry.register_type 15, Pathname, + packer: :to_s, + unpacker: :new + + registry.register_type 16, Regexp, + packer: :to_s, + unpacker: :new + + registry.register_type 17, ActiveSupport::HashWithIndifferentAccess, + packer: method(:write_hash_with_indifferent_access), + unpacker: method(:read_hash_with_indifferent_access), + recursive: true + end + + def install_unregistered_type_error(registry) + registry.register_type 127, Object, + packer: method(:raise_unserializable), + unpacker: method(:raise_invalid_format) + end + + def install_unregistered_type_fallback(registry) + registry.register_type 127, Object, + packer: method(:write_object), + unpacker: method(:read_object), + recursive: true + end + + def write_rational(rational, packer) + packer.write(rational.numerator) + packer.write(rational.denominator) unless rational.numerator.zero? + end + + def read_rational(unpacker) + numerator = unpacker.read + Rational(numerator, numerator.zero? ? 1 : unpacker.read) + end + + def write_complex(complex, packer) + packer.write(complex.real) + packer.write(complex.imaginary) + end + + def read_complex(unpacker) + Complex(unpacker.read, unpacker.read) + end + + def write_datetime(datetime, packer) + packer.write(datetime.jd) + packer.write(datetime.hour) + packer.write(datetime.min) + packer.write(datetime.sec) + write_rational(datetime.sec_fraction, packer) + write_rational(datetime.offset, packer) + end + + def read_datetime(unpacker) + DateTime.jd(unpacker.read, unpacker.read, unpacker.read, unpacker.read + read_rational(unpacker), read_rational(unpacker)) + end + + def write_date(date, packer) + packer.write(date.jd) + end + + def read_date(unpacker) + Date.jd(unpacker.read) + end + + def write_time(time, packer) + packer.write(time.tv_sec) + packer.write(time.tv_nsec) + packer.write(time.utc_offset) + end + + def read_time(unpacker) + Time.at_without_coercion(unpacker.read, unpacker.read, :nanosecond, in: unpacker.read) + end + + def write_time_with_zone(twz, packer) + write_time(twz.utc, packer) + write_time_zone(twz.time_zone, packer) + end + + def read_time_with_zone(unpacker) + ActiveSupport::TimeWithZone.new(read_time(unpacker), read_time_zone(unpacker)) + end + + def dump_time_zone(time_zone) + time_zone.name + end + + def load_time_zone(name) + ActiveSupport::TimeZone[name] + end + + def write_time_zone(time_zone, packer) + packer.write(dump_time_zone(time_zone)) + end + + def read_time_zone(unpacker) + load_time_zone(unpacker.read) + end + + def write_duration(duration, packer) + packer.write(duration.value) + packer.write(duration._parts.values_at(*ActiveSupport::Duration::PARTS)) + end + + def read_duration(unpacker) + value = unpacker.read + parts = ActiveSupport::Duration::PARTS.zip(unpacker.read).to_h + parts.compact! + ActiveSupport::Duration.new(value, parts) + end + + def write_range(range, packer) + packer.write(range.begin) + packer.write(range.end) + packer.write(range.exclude_end?) + end + + def read_range(unpacker) + Range.new(unpacker.read, unpacker.read, unpacker.read) + end + + def write_set(set, packer) + packer.write(set.to_a) + end + + def read_set(unpacker) + Set.new(unpacker.read) + end + + def write_hash_with_indifferent_access(hwia, packer) + packer.write(hwia.to_h) + end + + def read_hash_with_indifferent_access(unpacker) + ActiveSupport::HashWithIndifferentAccess.new(unpacker.read) + end + + def raise_unserializable(object, *) + raise UnserializableObjectError, "Unsupported type #{object.class} for object #{object.inspect}" + end + + def raise_invalid_format(*) + raise "Invalid format" + end + + def dump_class(klass) + raise UnserializableObjectError, "Cannot serialize anonymous class" unless klass.name + klass.name + end + + def load_class(name) + Object.const_get(name) + rescue NameError => error + if error.name.to_s == name + raise MissingClassError, "Missing class: #{name}" + else + raise + end + end + + def write_class(klass, packer) + packer.write(dump_class(klass)) + end + + def read_class(unpacker) + load_class(unpacker.read) + end + + LOAD_WITH_MSGPACK_EXT = 0 + LOAD_WITH_JSON_CREATE = 1 + + def write_object(object, packer) + if object.class.respond_to?(:from_msgpack_ext) + packer.write(LOAD_WITH_MSGPACK_EXT) + write_class(object.class, packer) + packer.write(object.to_msgpack_ext) + elsif object.class.respond_to?(:json_create) + packer.write(LOAD_WITH_JSON_CREATE) + write_class(object.class, packer) + packer.write(object.as_json) + else + raise_unserializable(object) + end + end + + def read_object(unpacker) + case unpacker.read + when LOAD_WITH_MSGPACK_EXT + read_class(unpacker).from_msgpack_ext(unpacker.read) + when LOAD_WITH_JSON_CREATE + read_class(unpacker).json_create(unpacker.read) + else + raise_invalid_format + end + end + end + end +end diff --git a/activesupport/lib/active_support/message_pack/serializer.rb b/activesupport/lib/active_support/message_pack/serializer.rb new file mode 100644 index 0000000000000..093f723a64471 --- /dev/null +++ b/activesupport/lib/active_support/message_pack/serializer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require_relative "extensions" + +module ActiveSupport + module MessagePack + module Serializer # :nodoc: + SIGNATURE = (+"\xCC\x80").force_encoding("ASCII-8BIT").freeze # == 128.to_msgpack + SIGNATURE_INT = 128 + + def dump(object) + message_pack_pool.packer do |packer| + packer.write(SIGNATURE_INT) + packer.write(object) + packer.full_pack + end + end + + def load(dumped) + message_pack_pool.unpacker do |unpacker| + unpacker.feed_reference(dumped) + raise "Invalid serialization format" unless unpacker.read == SIGNATURE_INT + unpacker.full_unpack + end + end + + def signature?(dumped) + dumped.start_with?(SIGNATURE) + end + + def message_pack_factory + @message_pack_factory ||= ::MessagePack::Factory.new + end + + def message_pack_factory=(factory) + @message_pack_pool = nil + @message_pack_factory = factory + end + + delegate :register_type, to: :message_pack_factory + + def warmup + message_pack_pool # eagerly compute + end + + private + def message_pack_pool + @message_pack_pool ||= begin + unless message_pack_factory.frozen? + Extensions.install(message_pack_factory) + install_unregistered_type_handler + message_pack_factory.freeze + end + message_pack_factory.pool(ENV.fetch("RAILS_MAX_THREADS") { 5 }) + end + end + + def install_unregistered_type_handler + Extensions.install_unregistered_type_error(message_pack_factory) + end + end + end +end diff --git a/activesupport/lib/active_support/messages/metadata.rb b/activesupport/lib/active_support/messages/metadata.rb index 5b8ccf68705bd..6688f2575f22c 100644 --- a/activesupport/lib/active_support/messages/metadata.rb +++ b/activesupport/lib/active_support/messages/metadata.rb @@ -15,6 +15,13 @@ module Metadata # :nodoc: Marshal, ] + TIMESTAMP_SERIALIZERS = [] + + ActiveSupport.on_load(:message_pack) do + ENVELOPE_SERIALIZERS.unshift ActiveSupport::MessagePack + TIMESTAMP_SERIALIZERS.unshift ActiveSupport::MessagePack + end + private def serialize_with_metadata(data, **metadata) has_metadata = metadata.any? { |k, v| v } @@ -80,11 +87,17 @@ def dual_serialized_metadata_envelope_json?(string) end def pick_expiry(expires_at, expires_in) - if expires_at - expires_at.utc.iso8601(3) + expiry = if expires_at + expires_at.utc elsif expires_in - Time.now.utc.advance(seconds: expires_in).iso8601(3) + Time.now.utc.advance(seconds: expires_in) end + + unless Metadata::TIMESTAMP_SERIALIZERS.include?(serializer) + expiry = expiry&.iso8601(3) + end + + expiry end def parse_expiry(expires_at) diff --git a/activesupport/test/message_pack/cache_serializer_test.rb b/activesupport/test/message_pack/cache_serializer_test.rb new file mode 100644 index 0000000000000..c1a5739cfbb01 --- /dev/null +++ b/activesupport/test/message_pack/cache_serializer_test.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require_relative "../abstract_unit" +require "active_support/message_pack" +require_relative "shared_serializer_tests" + +class MessagePackCacheSerializerTest < ActiveSupport::TestCase + include MessagePackSharedSerializerTests + + test "uses #to_msgpack_ext and ::from_msgpack_ext to roundtrip unregistered objects" do + assert_roundtrip DefinesFromMsgpackExt.new("foo") + end + + test "uses #as_json and ::json_create to roundtrip unregistered objects" do + assert_roundtrip DefinesJsonCreate.new("foo") + end + + test "raises error when unable to serialize an unregistered object" do + assert_raises ActiveSupport::MessagePack::UnserializableObjectError do + dump(Unserializable.new("foo")) + end + end + + test "raises error when serializing an unregistered object with an anonymous class" do + assert_raises ActiveSupport::MessagePack::UnserializableObjectError do + dump(Class.new(DefinesFromMsgpackExt).new("foo")) + end + end + + test "integrates with ActiveSupport::Cache" do + with_cache do |cache| + value = DefinesFromMsgpackExt.new("foo") + cache.write("key", value) + assert_equal value, cache.read("key") + end + end + + test "treats missing class as a cache miss" do + klass = Class.new(DefinesFromMsgpackExt) + def klass.name; "DoesNotActuallyExist"; end + + with_cache do |cache| + value = klass.new("foo") + cache.write("key", value) + assert_nil cache.read("key") + end + end + + test "supports compression" do + entry = ActiveSupport::Cache::Entry.new(["foo"] * 100) + uncompressed = serializer.dump(entry) + compressed = serializer.dump_compressed(entry, 1) + + assert_operator compressed.bytesize, :<, uncompressed.bytesize + assert_equal serializer.load(uncompressed).value, serializer.load(compressed).value + + with_cache(compress_threshold: 1) do |cache| + assert_equal compressed, cache.send(:serialize_entry, entry) + end + end + + private + def serializer + ActiveSupport::MessagePack::CacheSerializer + end + + def dump(object) + super(ActiveSupport::Cache::Entry.new(object)) + end + + def load(dumped) + super.value + end + + def with_cache(**options, &block) + Dir.mktmpdir do |dir| + block.call(ActiveSupport::Cache::FileStore.new(dir, coder: serializer, **options)) + end + end + + class HasValue + attr_reader :value + + def initialize(value) + @value = value + end + + def ==(other) + self.class == other.class && value == other.value + end + end + + class DefinesJsonCreate < HasValue + def self.json_create(hash) + DefinesJsonCreate.new(hash["as_json"]) + end + + def as_json + { "as_json" => value } + end + end + + class DefinesFromMsgpackExt < DefinesJsonCreate + def self.from_msgpack_ext(string) + DefinesFromMsgpackExt.new(string.chomp!("msgpack_ext")) + end + + def to_msgpack_ext + value + "msgpack_ext" + end + end + + class Unserializable < HasValue + def as_json + {} + end + + def to_msgpack_ext + "" + end + end +end diff --git a/activesupport/test/message_pack/serializer_test.rb b/activesupport/test/message_pack/serializer_test.rb new file mode 100644 index 0000000000000..12644a3d6105b --- /dev/null +++ b/activesupport/test/message_pack/serializer_test.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "../abstract_unit" +require "active_support/message_pack" +require_relative "shared_serializer_tests" + +class MessagePackSerializerTest < ActiveSupport::TestCase + include MessagePackSharedSerializerTests + + test "raises friendly error when dumping an unsupported object" do + assert_raises ActiveSupport::MessagePack::UnserializableObjectError do + dump(UnsupportedObject.new) + end + end + + private + def serializer + ActiveSupport::MessagePack + end + + class UnsupportedObject; end +end diff --git a/activesupport/test/message_pack/shared_serializer_tests.rb b/activesupport/test/message_pack/shared_serializer_tests.rb new file mode 100644 index 0000000000000..20dc094cb479c --- /dev/null +++ b/activesupport/test/message_pack/shared_serializer_tests.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module MessagePackSharedSerializerTests + extend ActiveSupport::Concern + + included do + test "enshrines type IDs" do + expected = { + 0 => Symbol, + 1 => Integer, + 2 => BigDecimal, + 3 => Rational, + 4 => Complex, + 5 => DateTime, + 6 => Date, + 7 => Time, + 8 => ActiveSupport::TimeWithZone, + 9 => ActiveSupport::TimeZone, + 10 => ActiveSupport::Duration, + 11 => Range, + 12 => Set, + 13 => URI::Generic, + 14 => IPAddr, + 15 => Pathname, + 16 => Regexp, + 17 => ActiveSupport::HashWithIndifferentAccess, + 127 => Object, + } + + serializer.warmup + actual = serializer.message_pack_factory.registered_types.to_h do |entry| + [entry[:type], entry[:class]] + end + + assert_equal expected, actual + end + + test "includes signature in message" do + assert serializer.signature?(dump("")) + assert_not serializer.signature?(Marshal.dump("")) + assert_not serializer.signature?("{}") + end + + test "roundtrips Symbol" do + assert_roundtrip :some_symbol + end + + test "roundtrips very large Integer" do + assert_roundtrip 2**512 + end + + test "roundtrips BigDecimal" do + assert_roundtrip BigDecimal("9876543210.0123456789") + end + + test "roundtrips Rational" do + assert_roundtrip Rational(1, 3) + end + + test "optimizes Rational zero encoding" do + assert_roundtrip Rational(0, 1) + + serialized_zero = dump(Rational(0, 1)) + serialized_nonzero = dump(Rational(1, 1)) + assert_operator serialized_zero.size, :<, serialized_nonzero.size + end + + test "roundtrips Complex" do + assert_roundtrip Complex(1, -1) + end + + test "roundtrips DateTime" do + assert_roundtrip DateTime.new(1999, 12, 31, 12, 34, 56 + Rational(789, 1000), Rational(-1, 2)) + assert_roundtrip DateTime.now + end + + test "roundtrips Date" do + assert_roundtrip Date.new(1999, 12, 31) + assert_roundtrip Date.today + end + + test "roundtrips Time" do + assert_roundtrip Time.new(1999, 12, 31, 12, 34, 56 + Rational(789, 1000), "-12:00") + assert_roundtrip Time.now + end + + test "roundtrips ActiveSupport::TimeWithZone" do + assert_roundtrip ActiveSupport::TimeWithZone.new( + Time.new(1999, 12, 31, 12, 34, 56 + Rational(789, 1000), "UTC"), + ActiveSupport::TimeZone["Australia/Lord_Howe"] + ) + assert_roundtrip Time.current + end + + test "roundtrips ActiveSupport::TimeZone" do + assert_roundtrip ActiveSupport::TimeZone["Eastern Time (US & Canada)"] + end + + test "roundtrips ActiveSupport::Duration" do + assert_roundtrip 1.year + 2.months + 3.weeks + 4.days + 5.hours + 6.minutes + 7.seconds + assert_roundtrip 1.month + 1.day + end + + test "roundtrips Range" do + assert_roundtrip 1..2 + assert_roundtrip 1...2 + assert_roundtrip 1..nil + assert_roundtrip 1...nil + assert_roundtrip nil..2 + assert_roundtrip nil...2 + assert_roundtrip "1".."2" + assert_roundtrip "1"..."2" + end + + test "roundtrips Set" do + assert_roundtrip Set.new([nil, true, 2, "three"]) + end + + test "roundtrips URI::Generic" do + assert_roundtrip URI("https://example.com/#test") + end + + test "roundtrips IPAddr" do + assert_roundtrip IPAddr.new("127.0.0.1") + end + + test "roundtrips Pathname" do + assert_roundtrip Pathname(__FILE__) + end + + test "roundtrips Regexp" do + assert_roundtrip %r/(?m-ix:.*)/ + end + + test "roundtrips ActiveSupport::HashWithIndifferentAccess" do + assert_roundtrip ActiveSupport::HashWithIndifferentAccess.new(a: true, b: 2, c: "three") + end + end + + private + def dump(object) + serializer.dump(object) + end + + def load(dumped) + serializer.load(dumped) + end + + def assert_roundtrip(object) + serialized = dump(object) + assert_kind_of String, serialized + + deserialized = load(serialized) + assert_instance_of object.class, deserialized + assert_equal object, deserialized + end +end diff --git a/activesupport/test/messages/message_metadata_tests.rb b/activesupport/test/messages/message_metadata_tests.rb index 38bee2085b324..a915a23ba1989 100644 --- a/activesupport/test/messages/message_metadata_tests.rb +++ b/activesupport/test/messages/message_metadata_tests.rb @@ -3,6 +3,7 @@ require "active_support/json" require "active_support/time" require "active_support/messages/metadata" +require "active_support/message_pack" module MessageMetadataTests extend ActiveSupport::Concern @@ -119,6 +120,7 @@ def self.load(value) JSON, ActiveSupport::JSON, ActiveSupport::JsonWithMarshalFallback, + ActiveSupport::MessagePack, CustomSerializer, ] diff --git a/railties/test/application/initializers/frameworks_test.rb b/railties/test/application/initializers/frameworks_test.rb index 9f5e9855536a3..974606896fa7a 100644 --- a/railties/test/application/initializers/frameworks_test.rb +++ b/railties/test/application/initializers/frameworks_test.rb @@ -376,5 +376,22 @@ def self.<(_) assert_nil ActiveRecord::Scoping::ScopeRegistry.current_scope(Post) end + + test "ActiveRecord::MessagePack extensions are installed when using ActiveSupport::MessagePack::CacheSerializer" do + rails %w(generate model post title:string) + rails %w(db:migrate) + + add_to_config <<~RUBY + require "active_support/message_pack" + config.cache_store = :file_store, #{app_path("tmp/cache").inspect}, + { coder: ActiveSupport::MessagePack::CacheSerializer } + RUBY + + require "#{app_path}/config/environment" + + post = Post.create!(title: "Hello World") + Rails.cache.write("hello", post) + assert_equal post, Rails.cache.read("hello") + end end end