From 5b2bcdca42d582c617e93618611693425794ec49 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Thu, 18 Jan 2024 16:25:22 -0700 Subject: [PATCH 01/11] refactoring/reimplementation of driver benchmarks --- .gitignore | 1 + lib/mongo/grid/fs_bucket.rb | 6 + profile/benchmark_21.rb | 62 -------- profile/driver_bench/base.rb | 149 ++++++++++++++++++ profile/driver_bench/bson.rb | 5 + profile/driver_bench/bson/base.rb | 24 +++ profile/driver_bench/bson/decodable.rb | 27 ++++ profile/driver_bench/bson/deep.rb | 4 + profile/driver_bench/bson/deep/base.rb | 19 +++ profile/driver_bench/bson/deep/decoding.rb | 16 ++ profile/driver_bench/bson/deep/encoding.rb | 16 ++ profile/driver_bench/bson/encodable.rb | 24 +++ profile/driver_bench/bson/flat.rb | 4 + profile/driver_bench/bson/flat/base.rb | 19 +++ profile/driver_bench/bson/flat/decoding.rb | 16 ++ profile/driver_bench/bson/flat/encoding.rb | 16 ++ profile/driver_bench/bson/full.rb | 4 + profile/driver_bench/bson/full/base.rb | 19 +++ profile/driver_bench/bson/full/decoding.rb | 16 ++ profile/driver_bench/bson/full/encoding.rb | 16 ++ profile/driver_bench/multi_doc.rb | 5 + profile/driver_bench/multi_doc/base.rb | 40 +++++ profile/driver_bench/multi_doc/bulk_insert.rb | 4 + .../multi_doc/bulk_insert/base.rb | 30 ++++ .../multi_doc/bulk_insert/large_doc.rb | 22 +++ .../multi_doc/bulk_insert/small_doc.rb | 22 +++ profile/driver_bench/multi_doc/find_many.rb | 29 ++++ profile/driver_bench/multi_doc/grid_fs.rb | 4 + .../driver_bench/multi_doc/grid_fs/base.rb | 23 +++ .../multi_doc/grid_fs/download.rb | 36 +++++ .../driver_bench/multi_doc/grid_fs/upload.rb | 30 ++++ profile/driver_bench/parallel.rb | 4 + profile/driver_bench/parallel/base.rb | 32 ++++ profile/driver_bench/parallel/counter.rb | 42 +++++ profile/driver_bench/parallel/dispatcher.rb | 45 ++++++ profile/driver_bench/parallel/gridfs.rb | 4 + profile/driver_bench/parallel/gridfs/base.rb | 33 ++++ .../driver_bench/parallel/gridfs/download.rb | 65 ++++++++ .../driver_bench/parallel/gridfs/upload.rb | 40 +++++ profile/driver_bench/parallel/ldjson.rb | 4 + profile/driver_bench/parallel/ldjson/base.rb | 42 +++++ .../driver_bench/parallel/ldjson/export.rb | 67 ++++++++ .../driver_bench/parallel/ldjson/import.rb | 41 +++++ profile/driver_bench/single_doc.rb | 5 + profile/driver_bench/single_doc/base.rb | 46 ++++++ .../driver_bench/single_doc/find_one_by_id.rb | 30 ++++ profile/driver_bench/single_doc/insert_one.rb | 4 + .../single_doc/insert_one/base.rb | 26 +++ .../single_doc/insert_one/large_doc.rb | 22 +++ .../single_doc/insert_one/small_doc.rb | 22 +++ .../driver_bench/single_doc/run_command.rb | 30 ++++ profile/profile.rb | 47 ------ 52 files changed, 1250 insertions(+), 109 deletions(-) delete mode 100644 profile/benchmark_21.rb create mode 100644 profile/driver_bench/base.rb create mode 100644 profile/driver_bench/bson.rb create mode 100644 profile/driver_bench/bson/base.rb create mode 100644 profile/driver_bench/bson/decodable.rb create mode 100644 profile/driver_bench/bson/deep.rb create mode 100644 profile/driver_bench/bson/deep/base.rb create mode 100644 profile/driver_bench/bson/deep/decoding.rb create mode 100644 profile/driver_bench/bson/deep/encoding.rb create mode 100644 profile/driver_bench/bson/encodable.rb create mode 100644 profile/driver_bench/bson/flat.rb create mode 100644 profile/driver_bench/bson/flat/base.rb create mode 100644 profile/driver_bench/bson/flat/decoding.rb create mode 100644 profile/driver_bench/bson/flat/encoding.rb create mode 100644 profile/driver_bench/bson/full.rb create mode 100644 profile/driver_bench/bson/full/base.rb create mode 100644 profile/driver_bench/bson/full/decoding.rb create mode 100644 profile/driver_bench/bson/full/encoding.rb create mode 100644 profile/driver_bench/multi_doc.rb create mode 100644 profile/driver_bench/multi_doc/base.rb create mode 100644 profile/driver_bench/multi_doc/bulk_insert.rb create mode 100644 profile/driver_bench/multi_doc/bulk_insert/base.rb create mode 100644 profile/driver_bench/multi_doc/bulk_insert/large_doc.rb create mode 100644 profile/driver_bench/multi_doc/bulk_insert/small_doc.rb create mode 100644 profile/driver_bench/multi_doc/find_many.rb create mode 100644 profile/driver_bench/multi_doc/grid_fs.rb create mode 100644 profile/driver_bench/multi_doc/grid_fs/base.rb create mode 100644 profile/driver_bench/multi_doc/grid_fs/download.rb create mode 100644 profile/driver_bench/multi_doc/grid_fs/upload.rb create mode 100644 profile/driver_bench/parallel.rb create mode 100644 profile/driver_bench/parallel/base.rb create mode 100644 profile/driver_bench/parallel/counter.rb create mode 100644 profile/driver_bench/parallel/dispatcher.rb create mode 100644 profile/driver_bench/parallel/gridfs.rb create mode 100644 profile/driver_bench/parallel/gridfs/base.rb create mode 100644 profile/driver_bench/parallel/gridfs/download.rb create mode 100644 profile/driver_bench/parallel/gridfs/upload.rb create mode 100644 profile/driver_bench/parallel/ldjson.rb create mode 100644 profile/driver_bench/parallel/ldjson/base.rb create mode 100644 profile/driver_bench/parallel/ldjson/export.rb create mode 100644 profile/driver_bench/parallel/ldjson/import.rb create mode 100644 profile/driver_bench/single_doc.rb create mode 100644 profile/driver_bench/single_doc/base.rb create mode 100644 profile/driver_bench/single_doc/find_one_by_id.rb create mode 100644 profile/driver_bench/single_doc/insert_one.rb create mode 100644 profile/driver_bench/single_doc/insert_one/base.rb create mode 100644 profile/driver_bench/single_doc/insert_one/large_doc.rb create mode 100644 profile/driver_bench/single_doc/insert_one/small_doc.rb create mode 100644 profile/driver_bench/single_doc/run_command.rb delete mode 100644 profile/profile.rb diff --git a/.gitignore b/.gitignore index f1acca9738..43e98f22a1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ gemfiles/*.gemfile.lock .env build profile/benchmarking/data +profile/data secrets-export.sh secrets-expansion.yml atlas-expansion.yml diff --git a/lib/mongo/grid/fs_bucket.rb b/lib/mongo/grid/fs_bucket.rb index 6cb732efc2..8d723cdf55 100644 --- a/lib/mongo/grid/fs_bucket.rb +++ b/lib/mongo/grid/fs_bucket.rb @@ -484,6 +484,12 @@ def write_concern end end + # Drop the collections that implement this bucket. + def drop + files_collection.drop + chunks_collection.drop + end + private # @param [ Hash ] opts The options. diff --git a/profile/benchmark_21.rb b/profile/benchmark_21.rb deleted file mode 100644 index e03c16d565..0000000000 --- a/profile/benchmark_21.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -require 'mongo' -require 'benchmark' - -Mongo::Logger.level = Logger::INFO - -client = Mongo::Client.new( - [ '127.0.0.1:27017' ], - database: 'ruby-driver', - user: 'root-user', - password: 'password', - auth_source: 'admin' -) - -collection = client[:test] - -Benchmark.bm do |bm| - - small_documents = 50000.times.map do |i| - { name: 'user', index: i, embedded: [{ n: i }] } - end - - bm.report('Mongo::Collection#insert_many') do - collection.insert_many(small_documents) - end - - bm.report('Mongo::Cursor#each') do - collection.find.each do |document| - end - end - - bm.report('Mongo::Collection::View#update_many') do - collection.find(name: 'user').update_many({ '$set' => { name: 'user_modified' }}) - end - - bm.report('Mongo::Collection::View#delete_many') do - collection.find(name: 'user_modified').delete_many - end - - large_documents = 50000.times.map do |i| - { name: 'user', index: i, embedded: 100.times.map{ |x| { n: x } }} - end - - bm.report('Mongo::Collection#insert_many') do - collection.insert_many(large_documents) - end - - bm.report('Mongo::Cursor#each') do - collection.find.each do |document| - end - end - - bm.report('Mongo::Collection::View#update_many') do - collection.find(name: 'user').update_many({ '$set' => { name: 'user_modified' }}) - end - - bm.report('Mongo::Collection::View#delete_many') do - collection.find(name: 'user_modified').delete_many - end -end diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb new file mode 100644 index 0000000000..c1b499f2ed --- /dev/null +++ b/profile/driver_bench/base.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'benchmark' +require 'mongo' + +module Mongo + module DriverBench + # Base class for DriverBench profile benchmarking classes. + # + # @api private + class Base + # Where to look for the data files + DATA_PATH = File.expand_path('../data/driver_bench', __dir__) + + # The maximum number of iterations to perform when executing the + # micro-benchmark. + attr_reader :max_iterations + + # The minimum number of seconds that the micro-benchmark must run, + # regardless of how many iterations it takes. + attr_reader :min_time + + # The maximum number of seconds that the micro-benchmark must run, + # regardless of how many iterations it takes. + attr_reader :max_time + + # The dataset to be used by the micro-benchmark. + attr_reader :dataset + + # The size of the dataset, computed per the spec, to be + # used for scoring the results. + attr_reader :dataset_size + + # Instantiate a new micro-benchmark class. + def initialize + @max_iterations = 100 + @min_time = 60 + @max_time = 300 # 5 minutes + end + + # Runs the micro-benchmark, and returns an array of timings, with one + # entry for each iteration of the benchmark. It may have fewer than + # max_iterations entries if it takes longer than max_time seconds, or + # more than max_iterations entries if it would take less than min_time + # seconds to run. + # + # @return [ Array ] the array of timings (in seconds) for + # each iteration. + def run + [].tap do |timings| + iteration_count = 0 + cumulative_time = 0 + + setup + + loop do + before_task + timing = without_gc { Benchmark.realtime { do_task } } + after_task + + iteration_count += 1 + cumulative_time += timing + timings.push timing + + # always stop after the maximum time has elapsed, regardless of + # iteration count. + break if cumulative_time > max_time + + # otherwise, break if the minimum time has elapsed, and the maximum + # number of iterations have been reached. + break if cumulative_time >= min_time && iteration_count >= max_iterations + end + + teardown + end + end + + private + + # Instantiate a new client. + def new_client(uri = ENV['MONGODB_URI']) + Mongo::Client.new(uri) + end + + # Runs a given block with GC disabled. + def without_gc + GC.disable + yield + ensure + GC.enable + end + + # By default, the file name is assumed to be relative to the + # DATA_PATH, unless the file name is an absolute path. + def path_to_file(file_name) + return file_name if file_name.start_with?('/') + File.join(DATA_PATH, file_name) + end + + # Load a json file and represent each document as a Hash. + # + # @param [ String ] file_name The file name. + # + # @return [ Array ] A list of extended-json documents. + def load_file(file_name) + File.readlines(path_to_file(file_name)).map { |line| BSON::Document.new(parse_line(line)) } + end + + # Returns the size (in bytes) of the given file. + def size_of_file(file_name) + File.size(path_to_file(file_name)) + end + + # Load a json document as a Hash and convert BSON-specific types. + # Replace the _id field as an BSON::ObjectId if it's represented as '$oid'. + # + # @param [ String ] document The json document. + # + # @return [ Hash ] An extended-json document. + def parse_line(document) + JSON.parse(document).tap do |doc| + doc['_id'] = ::BSON::ObjectId.from_string(doc['_id']['$oid']) if doc['_id'] && doc['_id']['$oid'] + end + end + + # Executed at the start of the micro-benchmark. + def setup + end + + # Executed before each iteration of the benchmark. + def before_task + end + + # Smallest amount of code necessary to do the task, + # invoked once per iteration. + def do_task + raise NotImplementedError + end + + # Executed after each iteration of the benchmark. + def after_task + end + + # Executed at the end of the micro-benchmark. + def teardown + end + end + end +end diff --git a/profile/driver_bench/bson.rb b/profile/driver_bench/bson.rb new file mode 100644 index 0000000000..0dba37bdfa --- /dev/null +++ b/profile/driver_bench/bson.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'bson/deep' +require_relative 'bson/flat' +require_relative 'bson/full' diff --git a/profile/driver_bench/bson/base.rb b/profile/driver_bench/bson/base.rb new file mode 100644 index 0000000000..0fd99af990 --- /dev/null +++ b/profile/driver_bench/bson/base.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + class Base < Mongo::DriverBench::Base + private + + def setup + @dataset ||= load_file(file_name).first + @dataset_size ||= size_of_file(file_name) + end + + # Returns the name of the file name that contains + # the dataset to use. + def file_name + raise NotImplementedError + end + end + end + end +end diff --git a/profile/driver_bench/bson/decodable.rb b/profile/driver_bench/bson/decodable.rb new file mode 100644 index 0000000000..bf18fd8555 --- /dev/null +++ b/profile/driver_bench/bson/decodable.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + module Decodable + private + + # The buffer to decode for the test + attr_reader :buffer + + def before_task + @buffer = ::BSON::Document.new(dataset).to_bson + end + + def do_task + 10_000.times do + ::BSON::Document.from_bson(buffer) + buffer.rewind! + end + end + end + end + end +end diff --git a/profile/driver_bench/bson/deep.rb b/profile/driver_bench/bson/deep.rb new file mode 100644 index 0000000000..9472bee0aa --- /dev/null +++ b/profile/driver_bench/bson/deep.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'deep/encoding' +require_relative 'deep/decoding' diff --git a/profile/driver_bench/bson/deep/base.rb b/profile/driver_bench/bson/deep/base.rb new file mode 100644 index 0000000000..2e304fdab0 --- /dev/null +++ b/profile/driver_bench/bson/deep/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + module Deep + class Base < Mongo::DriverBench::BSON::Base + private + + def file_name + "extended_bson/deep_bson.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/bson/deep/decoding.rb b/profile/driver_bench/bson/deep/decoding.rb new file mode 100644 index 0000000000..b7e2611417 --- /dev/null +++ b/profile/driver_bench/bson/deep/decoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../decodable' + +module Mongo + module DriverBench + module BSON + module Deep + class Decoding < Mongo::DriverBench::BSON::Deep::Base + include Decodable + end + end + end + end +end diff --git a/profile/driver_bench/bson/deep/encoding.rb b/profile/driver_bench/bson/deep/encoding.rb new file mode 100644 index 0000000000..07bfa74ad1 --- /dev/null +++ b/profile/driver_bench/bson/deep/encoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../encodable' + +module Mongo + module DriverBench + module BSON + module Deep + class Encoding < Mongo::DriverBench::BSON::Deep::Base + include Encodable + end + end + end + end +end diff --git a/profile/driver_bench/bson/encodable.rb b/profile/driver_bench/bson/encodable.rb new file mode 100644 index 0000000000..985c397e28 --- /dev/null +++ b/profile/driver_bench/bson/encodable.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + module Encodable + private + + # The document to encode for the test + attr_reader :document + + def before_task + @document = ::BSON::Document.new(dataset) + end + + def do_task + 10_000.times { document.to_bson } + end + end + end + end +end diff --git a/profile/driver_bench/bson/flat.rb b/profile/driver_bench/bson/flat.rb new file mode 100644 index 0000000000..994dffc2df --- /dev/null +++ b/profile/driver_bench/bson/flat.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'flat/encoding' +require_relative 'flat/decoding' diff --git a/profile/driver_bench/bson/flat/base.rb b/profile/driver_bench/bson/flat/base.rb new file mode 100644 index 0000000000..69ad20d89d --- /dev/null +++ b/profile/driver_bench/bson/flat/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + module Flat + class Base < Mongo::DriverBench::BSON::Base + private + + def file_name + "extended_bson/flat_bson.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/bson/flat/decoding.rb b/profile/driver_bench/bson/flat/decoding.rb new file mode 100644 index 0000000000..993fe09189 --- /dev/null +++ b/profile/driver_bench/bson/flat/decoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../decodable' + +module Mongo + module DriverBench + module BSON + module Flat + class Decoding < Mongo::DriverBench::BSON::Flat::Base + include Decodable + end + end + end + end +end diff --git a/profile/driver_bench/bson/flat/encoding.rb b/profile/driver_bench/bson/flat/encoding.rb new file mode 100644 index 0000000000..d486de6cbf --- /dev/null +++ b/profile/driver_bench/bson/flat/encoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../encodable' + +module Mongo + module DriverBench + module BSON + module Flat + class Encoding < Mongo::DriverBench::BSON::Flat::Base + include Encodable + end + end + end + end +end diff --git a/profile/driver_bench/bson/full.rb b/profile/driver_bench/bson/full.rb new file mode 100644 index 0000000000..ee75b9016b --- /dev/null +++ b/profile/driver_bench/bson/full.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'full/encoding' +require_relative 'full/decoding' diff --git a/profile/driver_bench/bson/full/base.rb b/profile/driver_bench/bson/full/base.rb new file mode 100644 index 0000000000..f124c2cbf7 --- /dev/null +++ b/profile/driver_bench/bson/full/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module BSON + module Full + class Base < Mongo::DriverBench::BSON::Base + private + + def file_name + "extended_bson/full_bson.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/bson/full/decoding.rb b/profile/driver_bench/bson/full/decoding.rb new file mode 100644 index 0000000000..b598ffc37d --- /dev/null +++ b/profile/driver_bench/bson/full/decoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../decodable' + +module Mongo + module DriverBench + module BSON + module Full + class Decoding < Mongo::DriverBench::BSON::Full::Base + include Decodable + end + end + end + end +end diff --git a/profile/driver_bench/bson/full/encoding.rb b/profile/driver_bench/bson/full/encoding.rb new file mode 100644 index 0000000000..f7f9283d87 --- /dev/null +++ b/profile/driver_bench/bson/full/encoding.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../encodable' + +module Mongo + module DriverBench + module BSON + module Full + class Encoding < Mongo::DriverBench::BSON::Full::Base + include Encodable + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc.rb b/profile/driver_bench/multi_doc.rb new file mode 100644 index 0000000000..35b92eaf1b --- /dev/null +++ b/profile/driver_bench/multi_doc.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'multi_doc/bulk_insert' +require_relative 'multi_doc/find_many' +require_relative 'multi_doc/grid_fs' diff --git a/profile/driver_bench/multi_doc/base.rb b/profile/driver_bench/multi_doc/base.rb new file mode 100644 index 0000000000..fc2f09c537 --- /dev/null +++ b/profile/driver_bench/multi_doc/base.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module MultiDoc + class Base < Mongo::DriverBench::Base + private + + attr_reader :client + attr_reader :collection + + def setup + if file_name + @dataset ||= load_file(file_name) + @dataset_size ||= size_of_file(file_name) + end + + prepare_client + end + + def teardown + cleanup_client + end + + def prepare_client + @client = new_client.use('perftest') + @client.database.drop + + @collection = @client.database[:corpus].tap(&:create) + end + + def cleanup_client + @client.database.drop + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/bulk_insert.rb b/profile/driver_bench/multi_doc/bulk_insert.rb new file mode 100644 index 0000000000..e2cb7d0e06 --- /dev/null +++ b/profile/driver_bench/multi_doc/bulk_insert.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'bulk_insert/large_doc' +require_relative 'bulk_insert/small_doc' diff --git a/profile/driver_bench/multi_doc/bulk_insert/base.rb b/profile/driver_bench/multi_doc/bulk_insert/base.rb new file mode 100644 index 0000000000..7df057b1bc --- /dev/null +++ b/profile/driver_bench/multi_doc/bulk_insert/base.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module MultiDoc + module BulkInsert + class Base < Mongo::DriverBench::MultiDoc::Base + attr_reader :repetitions + attr_reader :bulk_dataset + + def setup + super + @bulk_dataset = dataset * repetitions + end + + def before_task + collection.drop + collection.create + end + + def do_task + collection.insert_many(bulk_dataset, ordered: true) + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb b/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb new file mode 100644 index 0000000000..697948bb9a --- /dev/null +++ b/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module MultiDoc + module BulkInsert + class LargeDoc < Mongo::DriverBench::MultiDoc::BulkInsert::Base + def initialize + super + @repetitions = 10 + end + + def file_name + "single_and_multi_document/large_doc.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb b/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb new file mode 100644 index 0000000000..10aeb19b21 --- /dev/null +++ b/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module MultiDoc + module BulkInsert + class SmallDoc < Mongo::DriverBench::MultiDoc::BulkInsert::Base + def initialize + super + @repetitions = 10_000 + end + + def file_name + "single_and_multi_document/small_doc.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb new file mode 100644 index 0000000000..e55d1027f2 --- /dev/null +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module MultiDoc + class FindMany < Mongo::DriverBench::MultiDoc::Base + def file_name + "single_and_multi_document/tweet.json" + end + + def setup + super + + 10_000.times do |i| + @collection.insert_one(dataset.first) + end + end + + def do_task + collection.find.each do |result| + # discard the result + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/grid_fs.rb b/profile/driver_bench/multi_doc/grid_fs.rb new file mode 100644 index 0000000000..b5edbdedd1 --- /dev/null +++ b/profile/driver_bench/multi_doc/grid_fs.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'grid_fs/download' +require_relative 'grid_fs/upload' diff --git a/profile/driver_bench/multi_doc/grid_fs/base.rb b/profile/driver_bench/multi_doc/grid_fs/base.rb new file mode 100644 index 0000000000..2b37cbb49c --- /dev/null +++ b/profile/driver_bench/multi_doc/grid_fs/base.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module MultiDoc + module GridFS + class Base < Mongo::DriverBench::MultiDoc::Base + private + + def file_name + 'single_and_multi_document/gridfs_large.bin' + end + + def load_file(file_name) + File.read(path_to_file(file_name), encoding: 'BINARY') + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/grid_fs/download.rb b/profile/driver_bench/multi_doc/grid_fs/download.rb new file mode 100644 index 0000000000..ce02cb7694 --- /dev/null +++ b/profile/driver_bench/multi_doc/grid_fs/download.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'stringio' +require_relative 'base' + +module Mongo + module DriverBench + module MultiDoc + module GridFS + class Download < Mongo::DriverBench::MultiDoc::GridFS::Base + private + + attr_reader :fs_bucket + attr_reader :file_id + + def setup + super + + @file_id = client.database.fs + .upload_from_stream "gridfstest", dataset + end + + def before_task + super + + @fs_bucket = client.database.fs + end + + def do_task + fs_bucket.download_to_stream(file_id, StringIO.new) + end + end + end + end + end +end diff --git a/profile/driver_bench/multi_doc/grid_fs/upload.rb b/profile/driver_bench/multi_doc/grid_fs/upload.rb new file mode 100644 index 0000000000..2a598579e3 --- /dev/null +++ b/profile/driver_bench/multi_doc/grid_fs/upload.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module MultiDoc + module GridFS + class Upload < Mongo::DriverBench::MultiDoc::GridFS::Base + private + + attr_reader :fs_bucket + + def before_task + super + + @fs_bucket = client.database.fs + @fs_bucket.drop + + @fs_bucket.upload_from_stream "one-byte-file", "\n" + end + + def do_task + fs_bucket.upload_from_stream file_name, dataset + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel.rb b/profile/driver_bench/parallel.rb new file mode 100644 index 0000000000..27bdba3d9e --- /dev/null +++ b/profile/driver_bench/parallel.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'parallel/gridfs' +require_relative 'parallel/ldjson' diff --git a/profile/driver_bench/parallel/base.rb b/profile/driver_bench/parallel/base.rb new file mode 100644 index 0000000000..cd7b1c0276 --- /dev/null +++ b/profile/driver_bench/parallel/base.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module Parallel + class Base < Mongo::DriverBench::Base + private + + attr_reader :client + + def setup + prepare_client + end + + def teardown + cleanup_client + end + + def prepare_client + @client = new_client.use('perftest') + @client.database.drop + end + + def cleanup_client + client.database.drop + end + end + end + end +end diff --git a/profile/driver_bench/parallel/counter.rb b/profile/driver_bench/parallel/counter.rb new file mode 100644 index 0000000000..7574978069 --- /dev/null +++ b/profile/driver_bench/parallel/counter.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'thread' + +module Mongo + module DriverBench + module Parallel + class Counter + def initialize(value = 0) + @mutex = Thread::Mutex.new + @condition = Thread::ConditionVariable.new + @counter = value + end + + def enter + inc + yield + ensure + dec + end + + def wait + @mutex.synchronize do + return if @counter.zero? + @condition.wait(@mutex) + end + end + + def inc + @mutex.synchronize { @counter += 1 } + end + + def dec + @mutex.synchronize do + @counter -= 1 if @counter > 0 + @condition.signal if @counter.zero? + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/dispatcher.rb b/profile/driver_bench/parallel/dispatcher.rb new file mode 100644 index 0000000000..11f6bf078d --- /dev/null +++ b/profile/driver_bench/parallel/dispatcher.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'etc' +require 'thread' + +require_relative 'counter' + +module Mongo + module DriverBench + module Parallel + class Dispatcher + attr_reader :source + + def initialize(source, workers: Etc.nprocessors, &block) + @source = source + @counter = Counter.new + @source_mutex = Thread::Mutex.new + + @threads = Array.new(workers).map { Thread.new { @counter.enter { Thread.stop; worker_loop(&block) } } } + loop until @threads.all? { |t| t.status == 'sleep' } + end + + def run + @threads.each(&:wakeup) + @counter.wait + end + + private + + def next_batch + @source_mutex.synchronize do + @source.next + end + end + + def worker_loop(&block) + loop do + batch = next_batch or return + block.call(batch) + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/gridfs.rb b/profile/driver_bench/parallel/gridfs.rb new file mode 100644 index 0000000000..5312304561 --- /dev/null +++ b/profile/driver_bench/parallel/gridfs.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'gridfs/download' +require_relative 'gridfs/upload' diff --git a/profile/driver_bench/parallel/gridfs/base.rb b/profile/driver_bench/parallel/gridfs/base.rb new file mode 100644 index 0000000000..3e1046e5c1 --- /dev/null +++ b/profile/driver_bench/parallel/gridfs/base.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module Parallel + module GridFS + class Base < Mongo::DriverBench::Parallel::Base + def file_name_at(index) + format("parallel/gridfs_multi/file%02d.txt", index) + end + + private + + attr_reader :bucket + + def prepare_bucket(initialize: true) + @bucket = client.database.fs + @bucket.drop + @bucket.upload_from_stream "one-byte-file", "\n" if initialize + end + + def upload_file(file_name) + File.open(path_to_file(file_name), 'r') do |file| + bucket.upload_from_stream file_name, file + end + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/gridfs/download.rb b/profile/driver_bench/parallel/gridfs/download.rb new file mode 100644 index 0000000000..3d64ea35cc --- /dev/null +++ b/profile/driver_bench/parallel/gridfs/download.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative 'upload' +require_relative '../dispatcher' + +module Mongo + module DriverBench + module Parallel + module GridFS + class Download < Mongo::DriverBench::Parallel::GridFS::Base + private + + class Source + def initialize(list) + @list = list + @n = 0 + end + + def next + id = @list.pop or return nil + [ @n, id ].tap { @n += 1 } + end + end + + def setup + super + prepare_bucket(initialize: false) + + dispatcher = Dispatcher.new(Upload::Source.new(self)) do |file_name| + upload_file(file_name) + end + dispatcher.run + + @destination = File.join(Dir.tmpdir, 'parallel') + end + + def before_task + super + FileUtils.rm_rf(@destination) + FileUtils.mkdir_p(@destination) + + ids = bucket.files_collection.find.map { |doc| doc['_id'] } + @dispatcher = Dispatcher.new(Source.new(ids)) do |(n, id)| + download_file(n, id) + end + end + + def do_task + @dispatcher.run + end + + def download_file(n, id) + path = File.join(@destination, file_name_at(n)) + FileUtils.mkdir_p(File.dirname(path)) + + File.open(path, 'w') do |file| + bucket.download_to_stream(id, file) + end + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/gridfs/upload.rb b/profile/driver_bench/parallel/gridfs/upload.rb new file mode 100644 index 0000000000..24974bd669 --- /dev/null +++ b/profile/driver_bench/parallel/gridfs/upload.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../dispatcher' + +module Mongo + module DriverBench + module Parallel + module GridFS + class Upload < Mongo::DriverBench::Parallel::GridFS::Base + class Source + def initialize(bench) + @n = 0 + @bench = bench + end + + def next + return nil if @n >= 50 + @bench.file_name_at(@n).tap { @n += 1 } + end + end + + private + + def before_task + super + prepare_bucket + @dispatcher = Dispatcher.new(Source.new(self)) do |file_name| + upload_file(file_name) + end + end + + def do_task + @dispatcher.run + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/ldjson.rb b/profile/driver_bench/parallel/ldjson.rb new file mode 100644 index 0000000000..7d6b4b6fa6 --- /dev/null +++ b/profile/driver_bench/parallel/ldjson.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'ldjson/export' +require_relative 'ldjson/import' diff --git a/profile/driver_bench/parallel/ldjson/base.rb b/profile/driver_bench/parallel/ldjson/base.rb new file mode 100644 index 0000000000..a449671038 --- /dev/null +++ b/profile/driver_bench/parallel/ldjson/base.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module Parallel + module LDJSON + class Base < Mongo::DriverBench::Parallel::Base + def file_name_at(index) + format("parallel/ldjson_multi/ldjson%03d.txt", index) + end + + private + + attr_reader :collection + + def insert_docs_from_file(file_name, ids_relative_to: nil) + next_id = ids_relative_to + docs = File.readlines(path_to_file(file_name)).map do |line| + JSON.parse(line).tap do |doc| + if ids_relative_to + doc['_id'] = next_id + next_id += 1 + end + end + end + + collection.insert_many(docs) + end + + def prepare_collection + @collection = @client.database[:corpus].tap do |corpus| + corpus.drop + corpus.create + end + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/ldjson/export.rb b/profile/driver_bench/parallel/ldjson/export.rb new file mode 100644 index 0000000000..4ff6f00f9b --- /dev/null +++ b/profile/driver_bench/parallel/ldjson/export.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'tmpdir' +require 'fileutils' + +require_relative 'base' +require_relative '../dispatcher' + +module Mongo + module DriverBench + module Parallel + module LDJSON + class Export < Mongo::DriverBench::Parallel::LDJSON::Base + private + + class DataSource + def initialize(collection) + @n = 0 + @collection = collection + end + + def next + return nil if @n >= 100 + + batch = @collection.find(_id: { '$gte' => @n * 5000, '$lt' => (@n + 1) * 5000 }).to_a + [ @n, batch ].tap { @n += 1 } + end + end + + def setup + super + @destination = File.join(Dir.tmpdir, 'parallel') + FileUtils.mkdir_p(@destination) + + prepare_collection + + 100.times do |n| + insert_docs_from_file(file_name_at(n), ids_relative_to: n * 5000) + end + end + + def before_task + super + @dispatcher = Dispatcher.new(DataSource.new(collection)) do |(n, batch)| + worker_task(n, batch) + end + end + + def do_task + @dispatcher.run + end + + def teardown + super + FileUtils.rm_rf(@destination) + end + + def worker_task(n, batch) + path = File.join(@destination, file_name_at(n)) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, batch.map(&:to_json).join("\n")) + end + end + end + end + end +end diff --git a/profile/driver_bench/parallel/ldjson/import.rb b/profile/driver_bench/parallel/ldjson/import.rb new file mode 100644 index 0000000000..cc26fc2f20 --- /dev/null +++ b/profile/driver_bench/parallel/ldjson/import.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../dispatcher' + +module Mongo + module DriverBench + module Parallel + module LDJSON + class Import < Mongo::DriverBench::Parallel::LDJSON::Base + private + + class DataSource + def initialize(bench) + @n = 0 + @bench = bench + end + + def next + return nil if @n >= 100 + + @bench.file_name_at(@n).tap { @n += 1 } + end + end + + def before_task + super + prepare_collection + @dispatcher = Dispatcher.new(DataSource.new(self)) do |file_name| + insert_docs_from_file(file_name) + end + end + + def do_task + @dispatcher.run + end + end + end + end + end +end diff --git a/profile/driver_bench/single_doc.rb b/profile/driver_bench/single_doc.rb new file mode 100644 index 0000000000..c20395dec7 --- /dev/null +++ b/profile/driver_bench/single_doc.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'single_doc/find_one_by_id' +require_relative 'single_doc/insert_one' +require_relative 'single_doc/run_command' diff --git a/profile/driver_bench/single_doc/base.rb b/profile/driver_bench/single_doc/base.rb new file mode 100644 index 0000000000..958b97551f --- /dev/null +++ b/profile/driver_bench/single_doc/base.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module SingleDoc + class Base < Mongo::DriverBench::Base + private + + attr_reader :client + attr_reader :collection + + def setup + if file_name + @dataset ||= load_file(file_name).first + @dataset_size ||= size_of_file(file_name) + end + + prepare_client + end + + def teardown + cleanup_client + end + + def prepare_client + @client = new_client.use('perftest') + @client.database.drop + + @collection = @client.database[:corpus].tap(&:create) + end + + def cleanup_client + @client.database.drop + end + + # Returns the name of the file that contains + # the dataset to use. + def file_name + nil + end + end + end + end +end diff --git a/profile/driver_bench/single_doc/find_one_by_id.rb b/profile/driver_bench/single_doc/find_one_by_id.rb new file mode 100644 index 0000000000..c896d6dc13 --- /dev/null +++ b/profile/driver_bench/single_doc/find_one_by_id.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module SingleDoc + class FindOneByID < Mongo::DriverBench::SingleDoc::Base + def file_name + "single_and_multi_document/tweet.json" + end + + def setup + super + + doc = dataset + 10_000.times do |i| + @collection.insert_one(doc.merge(_id: i + 1)) + end + end + + def do_task + 10_000.times do |i| + collection.find(_id: i + 1).to_a + end + end + end + end + end +end diff --git a/profile/driver_bench/single_doc/insert_one.rb b/profile/driver_bench/single_doc/insert_one.rb new file mode 100644 index 0000000000..3a231267f2 --- /dev/null +++ b/profile/driver_bench/single_doc/insert_one.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'insert_one/large_doc' +require_relative 'insert_one/small_doc' diff --git a/profile/driver_bench/single_doc/insert_one/base.rb b/profile/driver_bench/single_doc/insert_one/base.rb new file mode 100644 index 0000000000..4f4262a929 --- /dev/null +++ b/profile/driver_bench/single_doc/insert_one/base.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative '../base' + +module Mongo + module DriverBench + module SingleDoc + module InsertOne + class Base < Mongo::DriverBench::SingleDoc::Base + attr_reader :repetitions + + def before_task + collection.drop + collection.create + end + + def do_task + repetitions.times do |i| + collection.insert_one(dataset) + end + end + end + end + end + end +end diff --git a/profile/driver_bench/single_doc/insert_one/large_doc.rb b/profile/driver_bench/single_doc/insert_one/large_doc.rb new file mode 100644 index 0000000000..e37a021767 --- /dev/null +++ b/profile/driver_bench/single_doc/insert_one/large_doc.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module SingleDoc + module InsertOne + class LargeDoc < Mongo::DriverBench::SingleDoc::InsertOne::Base + def initialize + super + @repetitions = 10 + end + + def file_name + "single_and_multi_document/large_doc.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/single_doc/insert_one/small_doc.rb b/profile/driver_bench/single_doc/insert_one/small_doc.rb new file mode 100644 index 0000000000..f36e042b40 --- /dev/null +++ b/profile/driver_bench/single_doc/insert_one/small_doc.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module SingleDoc + module InsertOne + class SmallDoc < Mongo::DriverBench::SingleDoc::InsertOne::Base + def initialize + super + @repetitions = 10_000 + end + + def file_name + "single_and_multi_document/small_doc.json" + end + end + end + end + end +end diff --git a/profile/driver_bench/single_doc/run_command.rb b/profile/driver_bench/single_doc/run_command.rb new file mode 100644 index 0000000000..4382bf688f --- /dev/null +++ b/profile/driver_bench/single_doc/run_command.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative 'base' + +module Mongo + module DriverBench + module SingleDoc + class RunCommand < Mongo::DriverBench::SingleDoc::Base + def setup + super + @dataset_size = { hello: true }.to_bson.length + end + + def prepare_client + @client = new_client + end + + def cleanup_client + # do nothing + end + + def do_task + 10_000.times do + client.database.command(hello: true) + end + end + end + end + end +end diff --git a/profile/profile.rb b/profile/profile.rb deleted file mode 100644 index 26bf770752..0000000000 --- a/profile/profile.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -require 'ruby-prof' -require 'mongo' - -Mongo::Logger.level = Logger::INFO - -client = Mongo::Client.new( - [ '127.0.0.1:27017' ], - database: 'ruby-driver', - user: 'root-user', - password: 'password', - auth_source: 'admin' -) - -collection = client[:test] - -documents = 50000.times.map do |i| - { name: 'user', index: i } -end - -inserts = RubyProf.profile do - collection.insert_many(documents) -end - -iteration = RubyProf.profile do - collection.find.each do |document| - end -end - -updates = RubyProf.profile do - collection.find(name: 'user').update_many({ '$set' => { name: 'user_modified' }}) -end - -deletes = RubyProf.profile do - collection.find(name: 'user_modified').delete_many -end - -p 'Inserts:' -RubyProf::FlatPrinter.new(inserts).print(STDOUT, min_percent: 2) -p 'Iteration:' -RubyProf::FlatPrinter.new(iteration).print(STDOUT, min_percent: 2) -p 'Updates:' -RubyProf::FlatPrinter.new(updates).print(STDOUT, min_percent: 2) -p 'Deletes:' -RubyProf::FlatPrinter.new(deletes).print(STDOUT, min_percent: 2) From e603b2e914dd60f1f36a7d019c198b5f0b5e3e1c Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Thu, 18 Jan 2024 19:42:55 -0700 Subject: [PATCH 02/11] computing scores --- profile/driver_bench/base.rb | 23 +++++++++++--- profile/driver_bench/bson/base.rb | 2 +- profile/driver_bench/multi_doc/base.rb | 6 +++- .../multi_doc/bulk_insert/base.rb | 4 +++ .../driver_bench/multi_doc/grid_fs/base.rb | 4 +++ profile/driver_bench/parallel/dispatcher.rb | 4 +-- profile/driver_bench/parallel/gridfs/base.rb | 5 +++ profile/driver_bench/parallel/ldjson/base.rb | 5 +++ profile/driver_bench/percentiles.rb | 31 +++++++++++++++++++ profile/driver_bench/single_doc/base.rb | 6 +++- .../single_doc/insert_one/large_doc.rb | 4 +++ .../single_doc/insert_one/small_doc.rb | 4 +++ .../driver_bench/single_doc/run_command.rb | 4 +-- 13 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 profile/driver_bench/percentiles.rb diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index c1b499f2ed..6357c9af7c 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -3,6 +3,8 @@ require 'benchmark' require 'mongo' +require_relative 'percentiles' + module Mongo module DriverBench # Base class for DriverBench profile benchmarking classes. @@ -38,6 +40,20 @@ def initialize @max_time = 300 # 5 minutes end + def run + timings = run_benchmark + percentiles = Percentiles.new(timings) + score = dataset_size / percentiles[50] / 1_000_000.0 + + puts "score: #{score}" + puts "percentiles:" + [ 10, 25, 50, 75, 90, 95, 98, 99 ].each do |pct| + puts " #{pct}: #{percentiles[pct]}" + end + end + + private + # Runs the micro-benchmark, and returns an array of timings, with one # entry for each iteration of the benchmark. It may have fewer than # max_iterations entries if it takes longer than max_time seconds, or @@ -46,7 +62,7 @@ def initialize # # @return [ Array ] the array of timings (in seconds) for # each iteration. - def run + def run_benchmark [].tap do |timings| iteration_count = 0 cumulative_time = 0 @@ -54,6 +70,7 @@ def run setup loop do +#puts "elapsed: #{cumulative_time}s" before_task timing = without_gc { Benchmark.realtime { do_task } } after_task @@ -75,8 +92,6 @@ def run end end - private - # Instantiate a new client. def new_client(uri = ENV['MONGODB_URI']) Mongo::Client.new(uri) @@ -103,7 +118,7 @@ def path_to_file(file_name) # # @return [ Array ] A list of extended-json documents. def load_file(file_name) - File.readlines(path_to_file(file_name)).map { |line| BSON::Document.new(parse_line(line)) } + File.readlines(path_to_file(file_name)).map { |line| ::BSON::Document.new(parse_line(line)) } end # Returns the size (in bytes) of the given file. diff --git a/profile/driver_bench/bson/base.rb b/profile/driver_bench/bson/base.rb index 0fd99af990..69ef26f193 100644 --- a/profile/driver_bench/bson/base.rb +++ b/profile/driver_bench/bson/base.rb @@ -10,7 +10,7 @@ class Base < Mongo::DriverBench::Base def setup @dataset ||= load_file(file_name).first - @dataset_size ||= size_of_file(file_name) + @dataset_size ||= size_of_file(file_name) * 10_000 end # Returns the name of the file name that contains diff --git a/profile/driver_bench/multi_doc/base.rb b/profile/driver_bench/multi_doc/base.rb index fc2f09c537..fa5bddecc6 100644 --- a/profile/driver_bench/multi_doc/base.rb +++ b/profile/driver_bench/multi_doc/base.rb @@ -14,12 +14,16 @@ class Base < Mongo::DriverBench::Base def setup if file_name @dataset ||= load_file(file_name) - @dataset_size ||= size_of_file(file_name) + @dataset_size ||= size_of_file(file_name) * scale end prepare_client end + def scale + 10_000 + end + def teardown cleanup_client end diff --git a/profile/driver_bench/multi_doc/bulk_insert/base.rb b/profile/driver_bench/multi_doc/bulk_insert/base.rb index 7df057b1bc..06c5625e70 100644 --- a/profile/driver_bench/multi_doc/bulk_insert/base.rb +++ b/profile/driver_bench/multi_doc/bulk_insert/base.rb @@ -15,6 +15,10 @@ def setup @bulk_dataset = dataset * repetitions end + def scale + @repetitions + end + def before_task collection.drop collection.create diff --git a/profile/driver_bench/multi_doc/grid_fs/base.rb b/profile/driver_bench/multi_doc/grid_fs/base.rb index 2b37cbb49c..0d3b439ddc 100644 --- a/profile/driver_bench/multi_doc/grid_fs/base.rb +++ b/profile/driver_bench/multi_doc/grid_fs/base.rb @@ -9,6 +9,10 @@ module GridFS class Base < Mongo::DriverBench::MultiDoc::Base private + def scale + 1 + end + def file_name 'single_and_multi_document/gridfs_large.bin' end diff --git a/profile/driver_bench/parallel/dispatcher.rb b/profile/driver_bench/parallel/dispatcher.rb index 11f6bf078d..c6c8c6c51b 100644 --- a/profile/driver_bench/parallel/dispatcher.rb +++ b/profile/driver_bench/parallel/dispatcher.rb @@ -11,13 +11,13 @@ module Parallel class Dispatcher attr_reader :source - def initialize(source, workers: Etc.nprocessors, &block) + def initialize(source, workers: (ENV['WORKERS'] || (Etc.nprocessors * 0.4)).to_i, &block) @source = source @counter = Counter.new @source_mutex = Thread::Mutex.new @threads = Array.new(workers).map { Thread.new { @counter.enter { Thread.stop; worker_loop(&block) } } } - loop until @threads.all? { |t| t.status == 'sleep' } + sleep 0.1 until @threads.all? { |t| t.status == 'sleep' } end def run diff --git a/profile/driver_bench/parallel/gridfs/base.rb b/profile/driver_bench/parallel/gridfs/base.rb index 3e1046e5c1..638f3cdb4d 100644 --- a/profile/driver_bench/parallel/gridfs/base.rb +++ b/profile/driver_bench/parallel/gridfs/base.rb @@ -15,6 +15,11 @@ def file_name_at(index) attr_reader :bucket + def setup + super + @dataset_size = 50.times.sum { |i| File.size(path_to_file(file_name_at(i))) } + end + def prepare_bucket(initialize: true) @bucket = client.database.fs @bucket.drop diff --git a/profile/driver_bench/parallel/ldjson/base.rb b/profile/driver_bench/parallel/ldjson/base.rb index a449671038..682c860121 100644 --- a/profile/driver_bench/parallel/ldjson/base.rb +++ b/profile/driver_bench/parallel/ldjson/base.rb @@ -29,6 +29,11 @@ def insert_docs_from_file(file_name, ids_relative_to: nil) collection.insert_many(docs) end + def setup + super + @dataset_size = 100.times.sum { |i| File.size(path_to_file(file_name_at(i))) } + end + def prepare_collection @collection = @client.database[:corpus].tap do |corpus| corpus.drop diff --git a/profile/driver_bench/percentiles.rb b/profile/driver_bench/percentiles.rb new file mode 100644 index 0000000000..3ec9eb7e5f --- /dev/null +++ b/profile/driver_bench/percentiles.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Mongo + module DriverBench + # A utility class for returning the list item at a given percentile + # value. + class Percentiles + # @return [ Array ] the sorted list of numbers to consider + attr_reader :list + + # Create a new Percentiles object that encapsulates the given list of + # numbers. + # + # @param [ Array ] list the list of numbers to considier + def initialize(list) + @list = list.sort + end + + # Finds and returns the element in the list that represents the given + # percentile value. + # + # @param [ Number ] percentile a number in the range [1,100] + # + # @return [ Number ] the element of the list for the given percentile. + def [](percentile) + i = (list.size * percentile / 100.0).ceil - 1 + list[i] + end + end + end +end diff --git a/profile/driver_bench/single_doc/base.rb b/profile/driver_bench/single_doc/base.rb index 958b97551f..4e379afba2 100644 --- a/profile/driver_bench/single_doc/base.rb +++ b/profile/driver_bench/single_doc/base.rb @@ -14,12 +14,16 @@ class Base < Mongo::DriverBench::Base def setup if file_name @dataset ||= load_file(file_name).first - @dataset_size ||= size_of_file(file_name) + @dataset_size ||= size_of_file(file_name) * scale end prepare_client end + def scale + 10_000 + end + def teardown cleanup_client end diff --git a/profile/driver_bench/single_doc/insert_one/large_doc.rb b/profile/driver_bench/single_doc/insert_one/large_doc.rb index e37a021767..c97c5680ce 100644 --- a/profile/driver_bench/single_doc/insert_one/large_doc.rb +++ b/profile/driver_bench/single_doc/insert_one/large_doc.rb @@ -12,6 +12,10 @@ def initialize @repetitions = 10 end + def scale + @repetitions + end + def file_name "single_and_multi_document/large_doc.json" end diff --git a/profile/driver_bench/single_doc/insert_one/small_doc.rb b/profile/driver_bench/single_doc/insert_one/small_doc.rb index f36e042b40..b8c2a8c7d7 100644 --- a/profile/driver_bench/single_doc/insert_one/small_doc.rb +++ b/profile/driver_bench/single_doc/insert_one/small_doc.rb @@ -12,6 +12,10 @@ def initialize @repetitions = 10_000 end + def scale + @repetitions + end + def file_name "single_and_multi_document/small_doc.json" end diff --git a/profile/driver_bench/single_doc/run_command.rb b/profile/driver_bench/single_doc/run_command.rb index 4382bf688f..b267e9c9e0 100644 --- a/profile/driver_bench/single_doc/run_command.rb +++ b/profile/driver_bench/single_doc/run_command.rb @@ -8,7 +8,7 @@ module SingleDoc class RunCommand < Mongo::DriverBench::SingleDoc::Base def setup super - @dataset_size = { hello: true }.to_bson.length + @dataset_size = { hello: true }.to_bson.length * scale end def prepare_client @@ -20,7 +20,7 @@ def cleanup_client end def do_task - 10_000.times do + 10_000.times do |i| client.database.command(hello: true) end end From 2b29988e95478e889787ae8d88fe9402637b0061 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 19 Jan 2024 14:42:25 -0700 Subject: [PATCH 03/11] supplant profile/benchmarking with profile/driver_bench --- .gitignore | 1 - Rakefile | 2 +- profile/benchmarking.rb | 82 ------ profile/benchmarking/bson.rb | 151 ----------- profile/benchmarking/helper.rb | 164 ------------ profile/benchmarking/multi_doc.rb | 182 ------------- profile/benchmarking/parallel.rb | 244 ------------------ profile/benchmarking/percentiles.rb | 31 --- profile/benchmarking/rake/bson.rake | 122 --------- profile/benchmarking/rake/multi_doc.rake | 34 --- profile/benchmarking/rake/parallel.rake | 36 --- profile/benchmarking/rake/single_doc.rake | 27 -- profile/benchmarking/rake/tasks.rake | 11 - profile/benchmarking/single_doc.rb | 149 ----------- profile/benchmarking/summary.rb | 56 ---- profile/driver_bench.rb | 3 + profile/driver_bench/base.rb | 23 +- profile/driver_bench/bson.rb | 11 + profile/driver_bench/bson/base.rb | 4 + profile/driver_bench/bson/decodable.rb | 5 + profile/driver_bench/bson/deep.rb | 10 + profile/driver_bench/bson/deep/base.rb | 7 +- profile/driver_bench/bson/deep/decoding.rb | 7 + profile/driver_bench/bson/deep/encoding.rb | 7 + profile/driver_bench/bson/encodable.rb | 5 + profile/driver_bench/bson/flat.rb | 10 + profile/driver_bench/bson/flat/base.rb | 7 +- profile/driver_bench/bson/flat/decoding.rb | 6 + profile/driver_bench/bson/flat/encoding.rb | 6 + profile/driver_bench/bson/full.rb | 10 + profile/driver_bench/bson/full/base.rb | 7 +- profile/driver_bench/bson/full/decoding.rb | 6 + profile/driver_bench/bson/full/encoding.rb | 6 + profile/driver_bench/multi_doc.rb | 11 + profile/driver_bench/multi_doc/bulk_insert.rb | 10 + .../multi_doc/bulk_insert/base.rb | 8 +- .../multi_doc/bulk_insert/large_doc.rb | 8 +- .../multi_doc/bulk_insert/small_doc.rb | 8 +- profile/driver_bench/multi_doc/find_many.rb | 10 +- profile/driver_bench/multi_doc/grid_fs.rb | 10 + .../driver_bench/multi_doc/grid_fs/base.rb | 5 + .../multi_doc/grid_fs/download.rb | 11 +- .../driver_bench/multi_doc/grid_fs/upload.rb | 8 +- profile/driver_bench/parallel.rb | 11 + profile/driver_bench/parallel/base.rb | 3 + profile/driver_bench/parallel/counter.rb | 18 +- profile/driver_bench/parallel/dispatcher.rb | 24 +- profile/driver_bench/parallel/gridfs.rb | 10 + profile/driver_bench/parallel/gridfs/base.rb | 7 +- .../driver_bench/parallel/gridfs/download.rb | 11 + .../driver_bench/parallel/gridfs/upload.rb | 11 + profile/driver_bench/parallel/ldjson.rb | 10 + profile/driver_bench/parallel/ldjson/base.rb | 5 +- .../driver_bench/parallel/ldjson/export.rb | 8 + .../driver_bench/parallel/ldjson/import.rb | 8 + profile/driver_bench/rake/tasks.rake | 8 + profile/driver_bench/single_doc.rb | 12 + profile/driver_bench/single_doc/base.rb | 8 +- .../driver_bench/single_doc/find_one_by_id.rb | 8 +- profile/driver_bench/single_doc/insert_one.rb | 10 + .../single_doc/insert_one/base.rb | 4 + .../single_doc/insert_one/large_doc.rb | 12 +- .../single_doc/insert_one/small_doc.rb | 12 +- .../driver_bench/single_doc/run_command.rb | 6 + profile/driver_bench/suite.rb | 128 +++++++++ 65 files changed, 525 insertions(+), 1330 deletions(-) delete mode 100644 profile/benchmarking.rb delete mode 100644 profile/benchmarking/bson.rb delete mode 100644 profile/benchmarking/helper.rb delete mode 100644 profile/benchmarking/multi_doc.rb delete mode 100644 profile/benchmarking/parallel.rb delete mode 100644 profile/benchmarking/percentiles.rb delete mode 100644 profile/benchmarking/rake/bson.rake delete mode 100644 profile/benchmarking/rake/multi_doc.rake delete mode 100644 profile/benchmarking/rake/parallel.rake delete mode 100644 profile/benchmarking/rake/single_doc.rake delete mode 100644 profile/benchmarking/rake/tasks.rake delete mode 100644 profile/benchmarking/single_doc.rb delete mode 100644 profile/benchmarking/summary.rb create mode 100644 profile/driver_bench.rb create mode 100644 profile/driver_bench/rake/tasks.rake create mode 100644 profile/driver_bench/suite.rb diff --git a/.gitignore b/.gitignore index 43e98f22a1..309742c33c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ gemfiles/*.gemfile.lock .env.private* .env build -profile/benchmarking/data profile/data secrets-export.sh secrets-expansion.yml diff --git a/Rakefile b/Rakefile index 4a4458070d..93578ae1b7 100644 --- a/Rakefile +++ b/Rakefile @@ -131,4 +131,4 @@ namespace :docs do end end -load 'profile/benchmarking/rake/tasks.rake' +load 'profile/driver_bench/rake/tasks.rake' diff --git a/profile/benchmarking.rb b/profile/benchmarking.rb deleted file mode 100644 index 08f6b20423..0000000000 --- a/profile/benchmarking.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -# Copyright (C) 2015-2020 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'benchmark' -require_relative 'benchmarking/helper' -require_relative 'benchmarking/bson' -require_relative 'benchmarking/single_doc' -require_relative 'benchmarking/multi_doc' -require_relative 'benchmarking/parallel' - -module Mongo - # Module with all functionality for running driver benchmark tests. - # - # @since 2.2.3 - module Benchmarking - extend self - - # @return [ String ] Path to Benchmarking test files. - DATA_PATH = [ __dir__, 'benchmarking', 'data' ].join('/').freeze - - # @return [ String ] The file containing the single tweet document. - TWEET_DOCUMENT_FILE = [ DATA_PATH, 'TWEET.json' ].join('/').freeze - - # @return [ String ] The file containing the single small document. - SMALL_DOCUMENT_FILE = [ DATA_PATH, 'SMALL_DOC.json' ].join('/').freeze - - # @return [ String ] The file containing the single large document. - LARGE_DOCUMENT_FILE = [ DATA_PATH, 'LARGE_DOC.json' ].join('/').freeze - - # @return [ String ] The file to upload when testing GridFS. - GRIDFS_FILE = [ DATA_PATH, 'GRIDFS_LARGE' ].join('/').freeze - - # @return [ String ] The file path and base name for the LDJSON files. - LDJSON_FILE_BASE = [ DATA_PATH, 'LDJSON_MULTI', 'LDJSON' ].join('/').freeze - - # @return [ String ] The file path and base name for the emitted LDJSON files. - LDJSON_FILE_OUTPUT_BASE = [ DATA_PATH, 'LDJSON_MULTI', 'output', 'LDJSON' ].join('/').freeze - - # @return [ String ] The file path and base name for the GRIDFS files to upload. - GRIDFS_MULTI_BASE = [ DATA_PATH, 'GRIDFS_MULTI', 'file' ].join('/').freeze - - # @return [ String ] The file path and base name for the emitted GRIDFS downloaded files. - GRIDFS_MULTI_OUTPUT_BASE = [ DATA_PATH, 'GRIDFS_MULTI', 'output', 'file-output' ].join('/').freeze - - # @return [ Integer ] The number of test repetitions. - TEST_REPETITIONS = 100 - - # Convenience helper for loading the single tweet document. - # - # @return [ Hash ] a single parsed JSON document - def tweet_document - Benchmarking.load_file(TWEET_DOCUMENT_FILE).first - end - - # Convenience helper for loading the single small document. - # - # @return [ Hash ] a single parsed JSON document - def small_document - Benchmarking.load_file(SMALL_DOCUMENT_FILE).first - end - - # Convenience helper for loading the single large document. - # - # @return [ Hash ] a single parsed JSON document - def large_document - Benchmarking.load_file(LARGE_DOCUMENT_FILE).first - end - end -end diff --git a/profile/benchmarking/bson.rb b/profile/benchmarking/bson.rb deleted file mode 100644 index 415e58114e..0000000000 --- a/profile/benchmarking/bson.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -# Copyright (C) 2015-2020 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require_relative 'percentiles' -require_relative 'summary' - -module Mongo - module Benchmarking - # These tests focus on BSON encoding and decoding; they are client-side only and - # do not involve any transmission of data to or from the server. - module BSON - extend self - - # Runs all of the benchmarks specified by the given mapping. - # - # @example Run a collection of benchmarks. - # Benchmarking::BSON.run_all( - # flat: %i[ encode decode ], - # deep: %i[ encode decode ], - # full: %i[ encode decode ] - # ) - # - # @return [ Hash ] a hash of the results for each benchmark - def run_all(map) - {}.tap do |results| - map.each do |type, actions| - results[type] = {} - - actions.each do |action| - results[type][action] = run(type, action) - end - end - end - end - - # As defined by the spec, the score for a given benchmark is the - # size of the task (in MB) divided by the median wall clock time. - # - # @param [ Symbol ] type the type of the task - # @param [ Mongo::Benchmarking::Percentiles ] percentiles the Percentiles - # object to query for the median time. - # @param [ Numeric ] scale the number of times the operation is performed - # per iteration, used to scale the task size. - # - # @return [ Numeric ] the score for the given task. - def score_for(type, percentiles, scale: 10_000) - task_size(type, scale) / percentiles[50] - end - - # Run a BSON benchmark test. - # - # @example Run a test. - # Benchmarking::BSON.run(:flat) - # - # @param [ Symbol ] type The type of test to run. - # @param [ :encode | :decode ] action The action to perform. - # - # @return [ Hash<:timings,:percentiles,:score> ] The test results for - # the requested benchmark. - def run(type, action) - timings = send(action, file_for(type)) - percentiles = Percentiles.new(timings) - score = score_for(type, percentiles) - - Summary.new(timings, percentiles, score) - end - - # Run an encoding BSON benchmark test. - # - # @example Run an encoding test. - # Benchmarking::BSON.encode(file_name) - # - # @param [ String ] file_name The name of the file with data for the test. - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Array ] The list of the results for each iteration - def encode(file_name) - data = Benchmarking.load_file(file_name) - document = ::BSON::Document.new(data.first) - - Benchmarking.benchmark do - 10_000.times { document.to_bson } - end - end - - # Run a decoding BSON benchmark test. - # - # @example Run an decoding test. - # Benchmarking::BSON.decode(file_name) - # - # @param [ String ] file_name The name of the file with data for the test. - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Array ] The list of the results for each iteration - def decode(file_name) - data = Benchmarking.load_file(file_name) - buffer = ::BSON::Document.new(data.first).to_bson - - Benchmarking.benchmark do - 10_000.times do - ::BSON::Document.from_bson(buffer) - buffer.rewind! - end - end - end - - private - - # The path to the source file for the given task type. - # - # @param [ Symbol ] type the task type - # - # @return [ String ] the path to the source file. - def file_for(type) - File.join(Benchmarking::DATA_PATH, "#{type}_bson.json") - end - - # As defined by the spec, the size of a BSON task is the size of the - # file, multipled by the scale (the number of times the file is processed - # per iteration), divided by a million. - # - # "the dataset size for a task is the size of the single-document source - # file...times 10,000 operations" - # - # "Each task will have defined for it an associated size in - # megabytes (MB)" - # - # @param [ Symbol ] type the type of the task - # @param [ Numeric ] scale the number of times the operation is performed - # per iteration (e.g. 10,000) - # - # @return [ Numeric ] the score for the task, reported in MB - def task_size(type, scale) - File.size(file_for(type)) * scale / 1_000_000.0 - end - end - end -end diff --git a/profile/benchmarking/helper.rb b/profile/benchmarking/helper.rb deleted file mode 100644 index 70d4e34312..0000000000 --- a/profile/benchmarking/helper.rb +++ /dev/null @@ -1,164 +0,0 @@ -# frozen_string_literal: true - -module Mongo - # Helper functions used by benchmarking tasks - module Benchmarking - extend self - - # Load a json file and represent each document as a Hash. - # - # @example Load a file. - # Benchmarking.load_file(file_name) - # - # @param [ String ] The file name. - # - # @return [ Array ] A list of extended-json documents. - # - # @since 2.2.3 - def load_file(file_name) - File.open(file_name, 'r') do |f| - f.each_line.collect do |line| - parse_json(line) - end - end - end - - # Load a json document as a Hash and convert BSON-specific types. - # Replace the _id field as an BSON::ObjectId if it's represented as '$oid'. - # - # @example Parse a json document. - # Benchmarking.parse_json(document) - # - # @param [ Hash ] The json document. - # - # @return [ Hash ] An extended-json document. - # - # @since 2.2.3 - def parse_json(document) - JSON.parse(document).tap do |doc| - doc['_id'] = ::BSON::ObjectId.from_string(doc['_id']['$oid']) if doc['_id'] && doc['_id']['$oid'] - end - end - - # The spec requires that most benchmarks use a variable number of - # iterations, defined as follows: - # - # * iterations should loop for at least 1 minute cumulative execution - # time - # * iterations should stop after 100 iterations or 5 minutes cumulative - # execution time, whichever is shorter - # - # This method will yield once for each iteration. - # - # @param [ Integer ] max_iterations the maximum number of iterations to - # attempt (default: 100) - # @param [ Integer ] min_time the minimum number of seconds to spend - # iterating - # @param [ Integer ] max_time the maximum number of seconds to spend - # iterating. - # - # @return [ Array ] the timings for each iteration - def benchmark(max_iterations: Benchmarking::TEST_REPETITIONS, - min_time: 60, - max_time: 5 * 60, - progress: default_progress_callback, - &block) - progress ||= ->(state) {} # fallback to a no-op callback - progress[:start] - - [].tap do |results| - iteration_count = 0 - cumulative_time = 0 - - loop do - timing = without_gc { Benchmark.realtime(&block) } - progress[:step] - - iteration_count += 1 - cumulative_time += timing - results.push timing - - # always stop after the maximum time has elapsed, regardless of - # iteration count. - break if cumulative_time > max_time - - # otherwise, break if the minimum time has elapsed, and the maximum - # number of iterations have been reached. - break if cumulative_time >= min_time && iteration_count >= max_iterations - end - - progress[:end] - end - end - - # Formats and displays a report of the given results. - # - # @param [ Hash ] results the results of a benchmarking run. - # @param [ Integer ] indent how much the report should be indented. - # @param [ Array ] percentiles the percentile values to report - def report(results, indent: 0, percentiles: [ 10, 25, 50, 75, 90, 95, 98, 99 ]) - results.each do |key, value| - puts format('%*s%s:', indent, '', key) - - if value.respond_to?(:summary) - puts value.summary(indent + 2, percentiles) - else - report(value, indent: indent + 2, percentiles: percentiles) - end - end - end - - # Get the median of values in a list. - # - # @example Get the median. - # Benchmarking.median(values) - # - # @param [ Array ] values The values to get the median of. - # - # @return [ Numeric ] The median of the list. - def median(values) - i = (values.size / 2) - 1 - values.sort[i] - end - - # Runs a given block with GC disabled. - def without_gc - GC.disable - yield - ensure - GC.enable - end - - private - - # Returns the proc object (or nil) corresponding to the "PROGRESS" - # environment variable. - # - # @return [ Proc | nil ] the callback proc to use (or nil if none should - # be used) - def default_progress_callback - case ENV['PROGRESS'] - when '0', 'false', 'none' - nil - when nil, '1', 'true', 'minimal' - method(:minimal_progress_callback).to_proc - else - raise ArgumentError, "unsupported progress callback #{ENV['PROGRESS'].inspect}" - end - end - - # A minimal progress callback implementation, printing '|' when a benchmark - # starts and '.' for each iteration. - # - # @param [ :start | :step | :end ] state the current progress state - def minimal_progress_callback(state) - case state - when :start then print '|' - when :step then print '.' - when :end then puts - end - - $stdout.flush - end - end -end diff --git a/profile/benchmarking/multi_doc.rb b/profile/benchmarking/multi_doc.rb deleted file mode 100644 index 545d5691d1..0000000000 --- a/profile/benchmarking/multi_doc.rb +++ /dev/null @@ -1,182 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -# Copyright (C) 2015-2020 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module Mongo - module Benchmarking - - # Multi-doc benchmarks focus on multiple-document read and write operations. - # They are designed to give insight into the efficiency of the driver's implementation - # of bulk/batch operations such as bulk writes and cursor reads. - # - # @since 2.2.3 - module MultiDoc - - extend self - - # Run a multi-document benchmark test. - # - # @example Run a test. - # Benchmarking::MultiDoc.run(:find_many) - # - # @param [ Symbol ] type The type of test to run. - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The test results. - # - # @since 2.2.3 - def run(type, repetitions = Benchmarking::TEST_REPETITIONS) - Mongo::Logger.logger.level = ::Logger::WARN - puts "#{type} : #{send(type, repetitions)}" - end - - # Test finding many documents. - # - # @example Test sending a find and exhausting the cursor. - # Benchmarking::MultiDoc.find_many(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def find_many(repetitions) - client.database.drop - doc = Benchmarking.tweet_document - - 10_000.times do |i| - collection.insert_one(doc) - end - - results = repetitions.times.collect do - Benchmark.realtime do - collection.find.to_a - end - end - client.database.drop - Benchmarking.median(results) - end - - # Test doing a bulk insert of small documents. - # - # @example Test bulk insert of small documents. - # Benchmarking::MultiDoc.bulk_insert_small(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def bulk_insert_small(repetitions) - bulk_insert(repetitions, [Benchmarking.small_document] * 10_000) - end - - # Test doing a bulk insert of large documents. - # - # @example Test bulk insert of large documents. - # Benchmarking::MultiDoc.bulk_insert_large(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def bulk_insert_large(repetitions) - bulk_insert(repetitions, [Benchmarking.large_document] * 10) - end - - # Test uploading to GridFS. - # - # @example Test uploading to GridFS. - # Benchmarking::MultiDoc.gridfs_upload(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def gridfs_upload(repetitions) - client.database.drop - create_collection - fs = client.with(write_concern: { w: 1 }).database.fs(write_concern: { w: 1}) - - s = StringIO.new('a') - fs.upload_from_stream('create-indices.test', s) - - file = File.open(GRIDFS_FILE) - - results = repetitions.times.collect do - file.rewind - Benchmark.realtime do - fs.upload_from_stream('GRIDFS_LARGE', file) - end - end - Benchmarking.median(results) - end - - # Test downloading from GridFS. - # - # @example Test downloading from GridFS. - # Benchmarking::MultiDoc.gridfs_download(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def gridfs_download(repetitions = Benchmarking::TEST_REPETITIONS) - client.database.drop - create_collection - fs = client.with(write_concern: { w: 1 }).database.fs(write_concern: { w: 1}) - - file_id = fs.upload_from_stream('gridfstest', File.open(GRIDFS_FILE)) - io = StringIO.new - - results = repetitions.times.collect do - io.rewind - Benchmark.realtime do - fs.download_to_stream(file_id, io) - end - end - Benchmarking.median(results) - end - - private - - def bulk_insert(repetitions, docs) - client.database.drop - create_collection - - results = repetitions.times.collect do - Benchmark.realtime do - collection.insert_many(docs) - end - end - client.database.drop - Benchmarking.median(results) - end - - def client - @client ||= Mongo::Client.new(["localhost:27017"], database: 'perftest', monitoring: false) - end - - def collection - @collection ||= begin; client[:corpus].tap { |coll| coll.create }; rescue Error::OperationFailure; client[:corpus]; end - end - alias :create_collection :collection - end - end -end diff --git a/profile/benchmarking/parallel.rb b/profile/benchmarking/parallel.rb deleted file mode 100644 index 92a92132ae..0000000000 --- a/profile/benchmarking/parallel.rb +++ /dev/null @@ -1,244 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -# Copyright (C) 2015-2020 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#require 'ruby-prof' - -module Mongo - module Benchmarking - - # Parallel tests simulate ETL operations from disk to database or vice-versa. - # They are designed to be implemented using a language's preferred approach to - # concurrency and thus stress how drivers handle concurrency. - # These intentionally involve overhead above and beyond the driver itself to - # simulate the sort of "real-world" pressures that a drivers would be under - # during concurrent operation. - # - # @since 2.2.3 - module Parallel - - extend self - - # Run a parallel benchmark test. - # - # @example Run a test. - # Benchmarking::Parallel.run(:import) - # - # @param [ Symbol ] type The type of test to run. - # - # @return [ Numeric ] The test results. - # - # @since 2.2.3 - def run(type) - Mongo::Logger.logger.level = ::Logger::WARN - type = type.to_s + '_jruby' if BSON::Environment.jruby? - puts "#{type} : #{send(type)}" - end - - # Test concurrently importing documents from a set of files. - # Using JRuby. - # - # @example Testing concurrently importing files using JRuby. - # Benchmarking::Parallel.import_jruby - # - # @return [ Numeric ] The test result. - # - # @since 2.2.3 - def import_jruby - #require 'jrjackson' - client.database.drop - create_collection - files = [*1..100].collect { |i| "#{LDJSON_FILE_BASE}#{i.to_s.rjust(3, "0")}.txt" } - - threads = [] - result = Benchmark.realtime do - 4.times do |i| - threads << Thread.new do - 25.times do |j| - docs = File.open(files[10 * i + j]).collect { |document| JSON.parse(document) } - #docs = File.open(files[10 * i + j]).collect { |document| JrJackson::Json.load(document) } - collection.insert_many(docs) - end - end - end - threads.collect { |t| t.join } - end - client.database.drop - result - end - - # Test concurrently importing documents from a set of files. - # - # @example Testing concurrently importing files. - # Benchmarking::Parallel.import - # - # @return [ Numeric ] The test result. - # - # @since 2.2.3 - def import - require 'yajl/json_gem' - require 'celluloid' - - Mongo::Collection.send(:include, Celluloid) - Celluloid.boot - - client.database.drop - create_collection - files = [*0..99].collect { |i| "#{LDJSON_FILE_BASE}#{i.to_s.rjust(3, "0")}.txt" } - - result = Benchmark.realtime do - Benchmarking::TEST_REPETITIONS.times do |i| - docs = File.open(files[i]).map{ |document| JSON.parse(document) } - collection.async.insert_many(docs) - end - end - client.database.drop - result - end - - # Test concurrently exporting documents from a collection to a set of files. - # - # @example Testing concurrently importing files. - # Benchmarking::Parallel.export - # - # @return [ Numeric ] The test result. - # - # @since 2.2.3 - def export - #require 'ruby-prof' - insert_files - files = [*1..Benchmarking::TEST_REPETITIONS].collect do |i| - name = "#{LDJSON_FILE_OUTPUT_BASE}#{i.to_s.rjust(3, "0")}.txt" - File.new(name, 'w') - end - #prof = nil - result = Benchmark.realtime do - Benchmarking::TEST_REPETITIONS.times do |i| - #prof = RubyProf.profile do - files[i].write(collection.find(_id: { '$gte' => (i * 5000), - '$lt' => (i+1) * 5000 }).to_a) - end - #end - end - client.database.drop - result - end - - # This benchmark tests driver performance uploading files from disk to GridFS. - # - # @example Test uploading files from disk to GridFS. - # Benchmarking::Parallel.gridfs_upload - # - # @return [ Numeric ] The test result. - # - # @since 2.2.3 - def gridfs_upload - n = 50 - client.database.drop - fs = client.database.fs - - files = [*0...n].collect do |i| - name = "#{GRIDFS_MULTI_BASE}#{i}.txt" - { - file: File.open(name, 'r'), - name: File.basename(name) - } - end - - s = StringIO.new('a') - fs.upload_from_stream('create-indices.test', s) - - Benchmark.realtime do - n.times do |i| - fs.upload_from_stream(files[i][:name], files[i][:file]) - end - end - end - alias :gridfs_upload_jruby :gridfs_upload - - - # This benchmark tests driver performance downloading files from GridFS to disk. - # - # @example Test downloading files from GridFS to disk. - # Benchmarking::Parallel.gridfs_download - # - # @return [ Numeric ] The test result. - # - # @since 2.2.3 - def gridfs_download - n_files = 50 - n_threads = BSON::Environment.jruby? ? 4 : 2 - threads = [] - client.database.drop - fs = client.database.fs - - file_info = [*0...n_files].collect do |i| - name = "#{GRIDFS_MULTI_BASE}#{i}.txt" - { - _id: fs.upload_from_stream(name, File.open(name)), - output_name: "#{GRIDFS_MULTI_OUTPUT_BASE}#{i}.txt" - } - end.freeze - - reps = n_files/n_threads.freeze - Benchmark.realtime do - n_threads.times do |i| - threads << Thread.new do - reps.times do |j| - index = i * reps + j - fs.download_to_stream(file_info[index][:_id], File.open(file_info[index][:output_name], "w")) - end - end - end - threads.collect(&:value) - end - end - alias :gridfs_download_jruby :gridfs_download - - private - - def insert_files - require 'yajl/json_gem' - require 'celluloid' - - Mongo::Collection.send(:include, Celluloid) - - client.database.drop - create_collection - files = [*1..Benchmarking::TEST_REPETITIONS].collect do |i| - File.open("#{LDJSON_FILE_BASE}#{i.to_s.rjust(3, "0")}.txt") - end - - Benchmarking::TEST_REPETITIONS.times do |i| - docs = files[i].each_with_index.collect do |document, offset| - JSON.parse(document).merge(_id: i * 5000 + offset) - end - collection.async.insert_many(docs) - end - puts "Imported #{Benchmarking::TEST_REPETITIONS} files, #{collection.count} documents." - end - - def client - @client ||= Mongo::Client.new(["localhost:27017"], database: 'perftest', monitoring: false) - end - - def collection - @collection ||= begin; client[:corpus].tap { |coll| coll.create }; rescue Error::OperationFailure; client[:corpus]; end - end - alias :create_collection :collection - end - end -end diff --git a/profile/benchmarking/percentiles.rb b/profile/benchmarking/percentiles.rb deleted file mode 100644 index aeebe9d1d9..0000000000 --- a/profile/benchmarking/percentiles.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Mongo - module Benchmarking - # A utility class for returning the list item at a given percentile - # value. - class Percentiles - # @return [ Array ] the sorted list of numbers to consider - attr_reader :list - - # Create a new Percentiles object that encapsulates the given list of - # numbers. - # - # @param [ Array ] list the list of numbers to considier - def initialize(list) - @list = list.sort - end - - # Finds and returns the element in the list that represents the given - # percentile value. - # - # @param [ Number ] percentile a number in the range [1,100] - # - # @return [ Number ] the element of the list for the given percentile. - def [](percentile) - i = (list.size * percentile / 100.0).ceil - 1 - list[i] - end - end - end -end diff --git a/profile/benchmarking/rake/bson.rake b/profile/benchmarking/rake/bson.rake deleted file mode 100644 index 4d5bdb1c04..0000000000 --- a/profile/benchmarking/rake/bson.rake +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -# rubocop:disable Layout/FirstHashElementIndentation - -desc 'Run the full BSON benchmarking suite' -task :bson do - puts 'BSON BENCHMARK SUITE' - Mongo::Benchmarking.report({ - bson: Mongo::Benchmarking::BSON.run_all( - flat: %i[ encode decode ], - deep: %i[ encode decode ], - full: %i[ encode decode ] - ) - }) -end - -namespace :bson do # rubocop:disable Metrics/BlockLength - # a convenience task for running all of the bson benchmark tasks; this is - # only useful for testing that they all work. - task test: %w[ - bson - bson:flat bson:flat:encode bson:flat:decode - bson:deep bson:deep:encode bson:deep:decode - bson:full bson:full:encode bson:full:decode - ] - - desc 'Learn how to run the BSON benchmarks' - task :help do - puts <<~HELP - The BSON micro benchmarks require a set of data files that are stored in - the specifications repository, here: - - https://github.com/mongodb/specifications/tree/master/source/benchmarking/data - - Download the `extended_bson.tgz` file and extract its contents. It should - contain a single folder (`extended_bson`) with several files in it. Move - those files to: - - #{Mongo::Benchmarking::DATA_PATH} - - Once there, you may run any of the BSON benchmarking tasks: - - $ rake benchmark:bson:flat:encode - - Tasks may be run in aggregate, as well, by specifying the namespace - directly: - - $ rake benchmark:bson:flat # runs all flat BSON benchmarks - $ rake benchmark:bson:deep # runs all deep BSON benchmarks - $ rake benchmark:bson:full # runs all full BSON benchmarks - # rake benchmark:bson # runs all BSON benchmarks - HELP - end - - desc 'Run the `flat` BSON benchmarking suite' - task :flat do - puts 'BSON BENCHMARK :: FLAT' - Mongo::Benchmarking.report({ - bson: Mongo::Benchmarking::BSON.run_all(flat: %i[ encode decode ]) - }) - end - - namespace :flat do - desc 'Run the `flat` encoding BSON benchmark' - task :encode do - puts 'BSON BENCHMARK :: FLAT :: ENCODE' - Mongo::Benchmarking.report({ bson: { flat: { encode: Mongo::Benchmarking::BSON.run(:flat, :encode) } } }) - end - - desc 'Run the `flat` decoding BSON benchmark' - task :decode do - puts 'BSON BENCHMARK :: FLAT :: DECODE' - Mongo::Benchmarking.report({ bson: { flat: { decode: Mongo::Benchmarking::BSON.run(:flat, :decode) } } }) - end - end - - desc 'Run the `deep` BSON benchmarking suite' - task :deep do - puts 'BSON BENCHMARK :: DEEP' - Mongo::Benchmarking.report({ - bson: Mongo::Benchmarking::BSON.run_all(deep: %i[ encode decode ]) - }) - end - - namespace :deep do - desc 'Run the `deep` encoding BSON benchmark' - task :encode do - puts 'BSON BENCHMARK :: DEEP :: ENCODE' - Mongo::Benchmarking.report({ bson: { deep: { encode: Mongo::Benchmarking::BSON.run(:deep, :encode) } } }) - end - - desc 'Run the `deep` decoding BSON benchmark' - task :decode do - puts 'BSON BENCHMARK :: DEEP :: DECODE' - Mongo::Benchmarking.report({ bson: { deep: { decode: Mongo::Benchmarking::BSON.run(:deep, :decode) } } }) - end - end - - desc 'Run the `full` BSON benchmarking suite' - task :full do - puts 'BSON BENCHMARK :: FULL' - Mongo::Benchmarking.report({ - bson: Mongo::Benchmarking::BSON.run_all({ full: %i[ encode decode ] }) - }) - end - - namespace :full do - desc 'Run the `full` encoding BSON benchmark' - task :encode do - puts 'BSON BENCHMARK :: FULL :: ENCODE' - Mongo::Benchmarking.report({ bson: { full: { encode: Mongo::Benchmarking::BSON.run(:full, :encode) } } }) - end - - desc 'Run the `full` decoding BSON benchmark' - task :decode do - puts 'BSON BENCHMARK :: FULL :: DECODE' - Mongo::Benchmarking.report({ bson: { full: { decode: Mongo::Benchmarking::BSON.run(:full, :decode) } } }) - end - end -end - -# rubocop:enable Layout/FirstHashElementIndentation diff --git a/profile/benchmarking/rake/multi_doc.rake b/profile/benchmarking/rake/multi_doc.rake deleted file mode 100644 index 86c190ef1f..0000000000 --- a/profile/benchmarking/rake/multi_doc.rake +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -namespace :multi_doc do - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called TWEET.json. - task :find_many do - puts 'MULTI DOCUMENT BENCHMARK :: FIND MANY' - Mongo::Benchmarking::MultiDoc.run(:find_many) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called SMALL_DOC.json. - task :bulk_insert_small do - puts 'MULTI DOCUMENT BENCHMARK :: BULK INSERT SMALL' - Mongo::Benchmarking::MultiDoc.run(:bulk_insert_small) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called LARGE_DOC.json. - task :bulk_insert_large do - puts 'MULTI DOCUMENT BENCHMARK :: BULK INSERT LARGE' - Mongo::Benchmarking::MultiDoc.run(:bulk_insert_large) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called GRIDFS_LARGE. - task :gridfs_upload do - puts 'MULTI DOCUMENT BENCHMARK :: GRIDFS UPLOAD' - Mongo::Benchmarking::MultiDoc.run(:gridfs_upload) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called GRIDFS_LARGE. - task :gridfs_download do - puts 'MULTI DOCUMENT BENCHMARK :: GRIDFS DOWNLOAD' - Mongo::Benchmarking::MultiDoc.run(:gridfs_download) - end -end diff --git a/profile/benchmarking/rake/parallel.rake b/profile/benchmarking/rake/parallel.rake deleted file mode 100644 index 98752e231e..0000000000 --- a/profile/benchmarking/rake/parallel.rake +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -namespace :parallel do - # Requirement: A directory in Mongo::Benchmarking::DATA_PATH, called LDJSON_MULTI, - # with the files used in this task. - task :import do - puts 'PARALLEL ETL BENCHMARK :: IMPORT' - Mongo::Benchmarking::Parallel.run(:import) - end - - # Requirement: A directory in Mongo::Benchmarking::DATA_PATH, called LDJSON_MULTI, - # with the files used in this task. - # Requirement: Another directory in '#{Mongo::Benchmarking::DATA_PATH}/LDJSON_MULTI' - # called 'output'. - task :export do - puts 'PARALLEL ETL BENCHMARK :: EXPORT' - Mongo::Benchmarking::Parallel.run(:export) - end - - # Requirement: A directory in Mongo::Benchmarking::DATA_PATH, called GRIDFS_MULTI, - # with the files used in this task. - task :gridfs_upload do - puts 'PARALLEL ETL BENCHMARK :: GRIDFS UPLOAD' - Mongo::Benchmarking::Parallel.run(:gridfs_upload) - end - - # Requirement: A directory in Mongo::Benchmarking::DATA_PATH, called GRIDFS_MULTI, - # with the files used in this task. - # Requirement: Another directory in '#{Mongo::Benchmarking::DATA_PATH}/GRIDFS_MULTI' - # called 'output'. - task :gridfs_download do - puts 'PARALLEL ETL BENCHMARK :: GRIDFS DOWNLOAD' - Mongo::Benchmarking::Parallel.run(:gridfs_download) - end -end diff --git a/profile/benchmarking/rake/single_doc.rake b/profile/benchmarking/rake/single_doc.rake deleted file mode 100644 index 803e28d593..0000000000 --- a/profile/benchmarking/rake/single_doc.rake +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -namespace :single_doc do - task :command do - puts 'SINGLE DOC BENCHMARK :: COMMAND' - Mongo::Benchmarking::SingleDoc.run(:command) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called TWEET.json. - task :find_one do - puts 'SINGLE DOC BENCHMARK :: FIND ONE BY ID' - Mongo::Benchmarking::SingleDoc.run(:find_one) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called SMALL_DOC.json. - task :insert_one_small do - puts 'SINGLE DOC BENCHMARK :: INSERT ONE SMALL DOCUMENT' - Mongo::Benchmarking::SingleDoc.run(:insert_one_small) - end - - # Requirement: A file in Mongo::Benchmarking::DATA_PATH, called LARGE_DOC.json. - task :insert_one_large do - puts 'SINGLE DOC BENCHMARK :: INSERT ONE LARGE DOCUMENT' - Mongo::Benchmarking::SingleDoc.run(:insert_one_large) - end -end diff --git a/profile/benchmarking/rake/tasks.rake b/profile/benchmarking/rake/tasks.rake deleted file mode 100644 index 7feae8d2ab..0000000000 --- a/profile/benchmarking/rake/tasks.rake +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -require_relative '../../benchmarking' - -# Some require data files, available from the drivers team. -# See the comments above each task for details. -namespace :benchmark do - %w[ bson single_doc multi_doc parallel ].each do |group| - load File.join(__dir__, "#{group}.rake") - end -end diff --git a/profile/benchmarking/single_doc.rb b/profile/benchmarking/single_doc.rb deleted file mode 100644 index a66635c542..0000000000 --- a/profile/benchmarking/single_doc.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -# Copyright (C) 2015-2020 MongoDB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module Mongo - module Benchmarking - - # Single-doc tests focus on single-document read and write operations. - # They are designed to give insights into the efficiency of the driver's - # implementation of the basic wire protocol. - # - # @since 2.2.3 - module SingleDoc - - extend self - - # Run a Single Document benchmark test. - # - # @example Run a test. - # Benchmarking::SingleDoc.run(:command) - # - # @param [ Symbol ] type The type of test to run. - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numberic ] The test results. - # - # @since 2.2.3 - def run(type, repetitions = Benchmarking::TEST_REPETITIONS) - Mongo::Logger.logger.level = ::Logger::WARN - puts "#{type} : #{send(type, repetitions)}" - end - - # Test sending a command to the server. - # - # @example Test sending an ismaster command. - # Benchmarking::SingleDoc.command(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @since 2.2.3 - def command(repetitions) - monitor = client.cluster.servers.first.monitor - results = repetitions.times.collect do - Benchmark.realtime do - 10_000.times do - client.database.command(hello: true) - end - end - end - Benchmarking.median(results) - end - - # Test sending find one by id. - # - # @example Test sending a find. - # Benchmarking::SingleDoc.find_one(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def find_one(repetitions) - client.database.drop - doc = Benchmarking.tweet_document - - 10_000.times do |i| - doc[:_id] = i - collection.insert_one(doc) - end - - results = repetitions.times.collect do - Benchmark.realtime do - 10_000.times do |i| - collection.find({ _id: i }, limit: -1).first - end - end - end - Benchmarking.median(results) - end - - # Test inserting a large document. - # - # @example Test inserting a large document. - # Benchmarking::SingleDoc.insert_one_large(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def insert_one_large(repetitions) - insert_one(repetitions, 10, Benchmarking.large_document) - end - - # Test inserting a small document. - # - # @example Test inserting a small document. - # Benchmarking::SingleDoc.insert_one_small(10) - # - # @param [ Integer ] repetitions The number of test repetitions. - # - # @return [ Numeric ] The median of the results. - # - # @since 2.2.3 - def insert_one_small(repetitions) - insert_one(repetitions, 10_000, Benchmarking.small_document) - end - - private - - def insert_one(repetitions, do_repetitions, doc) - client.database.drop - create_collection - - results = repetitions.times.collect do - Benchmark.realtime do - do_repetitions.times do - collection.insert_one(doc) - end - end - end - Benchmarking.median(results) - end - - def client - @client ||= Mongo::Client.new(["localhost:27017"], database: 'perftest', monitoring: false) - end - - def collection - @collection ||= begin; client[:corpus].tap { |coll| coll.create }; rescue Error::OperationFailure; client[:corpus]; end - end - alias :create_collection :collection - end - end -end diff --git a/profile/benchmarking/summary.rb b/profile/benchmarking/summary.rb deleted file mode 100644 index 93fddf5435..0000000000 --- a/profile/benchmarking/summary.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Mongo - module Benchmarking - # A utility class for encapsulating the summary information for a - # benchmark, including behaviors for reporting on the summary. - class Summary - # @return [ Array ] the timings of each iteration in the - # benchmark - attr_reader :timings - - # @return [ Percentiles ] the percentiles object for querying the - # timing at a given percentile value. - attr_reader :percentiles - - # @return [ Numeric ] the composite score for the benchmark - attr_reader :score - - # Construct a new Summary object with the given timings, percentiles, - # and score. - # - # @param [ Array ] timings the timings of each iteration in the - # benchmark - # @param [ Percentiles ] percentiles the percentiles object for querying - # the timing at a given percentile value - # @param [ Numeric ] score the composite score for the benchmark - def initialize(timings, percentiles, score) - @timings = timings - @percentiles = percentiles - @score = score - end - - # @return [ Numeric ] the median timing for the benchmark. - def median - percentiles[50] - end - - # Formats and displays the results of a single benchmark run. - # - # @param [ Integer ] indent how much the report should be indented - # @param [ Array ] points the percentile points to report - # - # @return [ String ] a YAML-formatted summary - def summary(indent, points) - [].tap do |lines| - lines << format('%*sscore: %g', indent, '', score) - lines << format('%*smedian: %g', indent, '', median) - lines << format('%*spercentiles:', indent, '') - points.each do |pct| - lines << format('%*s%g: %g', indent + 2, '', pct, percentiles[pct]) - end - end.join("\n") - end - end - end -end diff --git a/profile/driver_bench.rb b/profile/driver_bench.rb new file mode 100644 index 0000000000..f2ab14393d --- /dev/null +++ b/profile/driver_bench.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require 'driver_bench/suite' diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index 6357c9af7c..072fa5ee67 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -11,6 +11,12 @@ module DriverBench # # @api private class Base + # A convenience for setting and querying the benchmark's name + def self.bench_name(benchmark_name = nil) + @bench_name = benchmark_name if benchmark_name + @bench_name + end + # Where to look for the data files DATA_PATH = File.expand_path('../data/driver_bench', __dir__) @@ -36,20 +42,22 @@ class Base # Instantiate a new micro-benchmark class. def initialize @max_iterations = 100 - @min_time = 60 + @min_time = ENV['CHEAT'] ? 10 : 60 @max_time = 300 # 5 minutes end + # Runs the benchmark and returns the score. + # + # @return [ Hash ] the score and other + # attributes of the benchmark. def run timings = run_benchmark percentiles = Percentiles.new(timings) score = dataset_size / percentiles[50] / 1_000_000.0 - puts "score: #{score}" - puts "percentiles:" - [ 10, 25, 50, 75, 90, 95, 98, 99 ].each do |pct| - puts " #{pct}: #{percentiles[pct]}" - end + { name: self.class.bench_name, + score: score, + percentiles: percentiles } end private @@ -70,9 +78,8 @@ def run_benchmark setup loop do -#puts "elapsed: #{cumulative_time}s" before_task - timing = without_gc { Benchmark.realtime { do_task } } + timing = without_gc { Benchmark.realtime { ENV['CHEAT'] ? sleep(0.1) : do_task } } after_task iteration_count += 1 diff --git a/profile/driver_bench/bson.rb b/profile/driver_bench/bson.rb index 0dba37bdfa..6da18affc4 100644 --- a/profile/driver_bench/bson.rb +++ b/profile/driver_bench/bson.rb @@ -3,3 +3,14 @@ require_relative 'bson/deep' require_relative 'bson/flat' require_relative 'bson/full' + +module Mongo + module DriverBench + module BSON + ALL = [ *Deep::ALL, *Flat::ALL, *Full::ALL ].freeze + + # BSONBench consists of all BSON micro-benchmarks + BENCH = ALL + end + end +end diff --git a/profile/driver_bench/bson/base.rb b/profile/driver_bench/bson/base.rb index 69ef26f193..59aa69520a 100644 --- a/profile/driver_bench/bson/base.rb +++ b/profile/driver_bench/bson/base.rb @@ -5,9 +5,13 @@ module Mongo module DriverBench module BSON + # Abstract superclass for all BSON benchmarks. + # + # @api private class Base < Mongo::DriverBench::Base private + # Common setup for these benchmarks. def setup @dataset ||= load_file(file_name).first @dataset_size ||= size_of_file(file_name) * 10_000 diff --git a/profile/driver_bench/bson/decodable.rb b/profile/driver_bench/bson/decodable.rb index bf18fd8555..84c1506acf 100644 --- a/profile/driver_bench/bson/decodable.rb +++ b/profile/driver_bench/bson/decodable.rb @@ -5,16 +5,21 @@ module Mongo module DriverBench module BSON + # Common behavior for "decode" benchmarks. + # + # @api private module Decodable private # The buffer to decode for the test attr_reader :buffer + # Before executing the task itself. def before_task @buffer = ::BSON::Document.new(dataset).to_bson end + # The decode operation, performed 10k times. def do_task 10_000.times do ::BSON::Document.from_bson(buffer) diff --git a/profile/driver_bench/bson/deep.rb b/profile/driver_bench/bson/deep.rb index 9472bee0aa..6beff6dbe7 100644 --- a/profile/driver_bench/bson/deep.rb +++ b/profile/driver_bench/bson/deep.rb @@ -2,3 +2,13 @@ require_relative 'deep/encoding' require_relative 'deep/decoding' + +module Mongo + module DriverBench + module BSON + module Deep + ALL = [ Encoding, Decoding ].freeze + end + end + end +end diff --git a/profile/driver_bench/bson/deep/base.rb b/profile/driver_bench/bson/deep/base.rb index 2e304fdab0..350741da03 100644 --- a/profile/driver_bench/bson/deep/base.rb +++ b/profile/driver_bench/bson/deep/base.rb @@ -6,11 +6,16 @@ module Mongo module DriverBench module BSON module Deep + # Abstract superclass for deep BSON benchmarks. + # + # @api private class Base < Mongo::DriverBench::BSON::Base private + # @return [ String ] the name of the file to use as the + # dataset for these benchmarks. def file_name - "extended_bson/deep_bson.json" + 'extended_bson/deep_bson.json' end end end diff --git a/profile/driver_bench/bson/deep/decoding.rb b/profile/driver_bench/bson/deep/decoding.rb index b7e2611417..e830af1ed5 100644 --- a/profile/driver_bench/bson/deep/decoding.rb +++ b/profile/driver_bench/bson/deep/decoding.rb @@ -7,8 +7,15 @@ module Mongo module DriverBench module BSON module Deep + # "This benchmark tests driver performance decoding documents with + # deeply nested key/value pairs involving subdocuments, strings, + # integers, doubles and booleans." + # + # @api private class Decoding < Mongo::DriverBench::BSON::Deep::Base include Decodable + + bench_name 'Deep BSON Decoding' end end end diff --git a/profile/driver_bench/bson/deep/encoding.rb b/profile/driver_bench/bson/deep/encoding.rb index 07bfa74ad1..86ceb3466a 100644 --- a/profile/driver_bench/bson/deep/encoding.rb +++ b/profile/driver_bench/bson/deep/encoding.rb @@ -7,8 +7,15 @@ module Mongo module DriverBench module BSON module Deep + # "This benchmark tests driver performance encoding documents with + # deeply nested key/value pairs involving subdocuments, strings, + # integers, doubles and booleans." + # + # @api private class Encoding < Mongo::DriverBench::BSON::Deep::Base include Encodable + + bench_name 'Deep BSON Encoding' end end end diff --git a/profile/driver_bench/bson/encodable.rb b/profile/driver_bench/bson/encodable.rb index 985c397e28..55138fc868 100644 --- a/profile/driver_bench/bson/encodable.rb +++ b/profile/driver_bench/bson/encodable.rb @@ -5,16 +5,21 @@ module Mongo module DriverBench module BSON + # Common behavior for the "encode" benchmarks. + # + # @api private module Encodable private # The document to encode for the test attr_reader :document + # Before each task. def before_task @document = ::BSON::Document.new(dataset) end + # The encode operation itself, executed 10k times. def do_task 10_000.times { document.to_bson } end diff --git a/profile/driver_bench/bson/flat.rb b/profile/driver_bench/bson/flat.rb index 994dffc2df..40ee68d4bd 100644 --- a/profile/driver_bench/bson/flat.rb +++ b/profile/driver_bench/bson/flat.rb @@ -2,3 +2,13 @@ require_relative 'flat/encoding' require_relative 'flat/decoding' + +module Mongo + module DriverBench + module BSON + module Flat + ALL = [ Encoding, Decoding ].freeze + end + end + end +end diff --git a/profile/driver_bench/bson/flat/base.rb b/profile/driver_bench/bson/flat/base.rb index 69ad20d89d..64e7caecc8 100644 --- a/profile/driver_bench/bson/flat/base.rb +++ b/profile/driver_bench/bson/flat/base.rb @@ -6,11 +6,16 @@ module Mongo module DriverBench module BSON module Flat + # Abstract superclass of flat BSON benchmarks. + # + # @api private class Base < Mongo::DriverBench::BSON::Base private + # @return [ String ] the name of the file to use as the + # dataset for these benchmarks. def file_name - "extended_bson/flat_bson.json" + 'extended_bson/flat_bson.json' end end end diff --git a/profile/driver_bench/bson/flat/decoding.rb b/profile/driver_bench/bson/flat/decoding.rb index 993fe09189..177e189c32 100644 --- a/profile/driver_bench/bson/flat/decoding.rb +++ b/profile/driver_bench/bson/flat/decoding.rb @@ -7,8 +7,14 @@ module Mongo module DriverBench module BSON module Flat + # "This benchmark tests driver performance decoding documents with top + # level key/value pairs involving the most commonly-used BSON types." + # + # @api private class Decoding < Mongo::DriverBench::BSON::Flat::Base include Decodable + + bench_name 'Flat BSON Decoding' end end end diff --git a/profile/driver_bench/bson/flat/encoding.rb b/profile/driver_bench/bson/flat/encoding.rb index d486de6cbf..5f12d5c053 100644 --- a/profile/driver_bench/bson/flat/encoding.rb +++ b/profile/driver_bench/bson/flat/encoding.rb @@ -7,8 +7,14 @@ module Mongo module DriverBench module BSON module Flat + # "This benchmark tests driver performance encoding documents with top + # level key/value pairs involving the most commonly-used BSON types." + # + # @api private class Encoding < Mongo::DriverBench::BSON::Flat::Base include Encodable + + bench_name 'Flat BSON Encoding' end end end diff --git a/profile/driver_bench/bson/full.rb b/profile/driver_bench/bson/full.rb index ee75b9016b..ee0259fcf6 100644 --- a/profile/driver_bench/bson/full.rb +++ b/profile/driver_bench/bson/full.rb @@ -2,3 +2,13 @@ require_relative 'full/encoding' require_relative 'full/decoding' + +module Mongo + module DriverBench + module BSON + module Full + ALL = [ Encoding, Decoding ].freeze + end + end + end +end diff --git a/profile/driver_bench/bson/full/base.rb b/profile/driver_bench/bson/full/base.rb index f124c2cbf7..0d67d48e46 100644 --- a/profile/driver_bench/bson/full/base.rb +++ b/profile/driver_bench/bson/full/base.rb @@ -6,11 +6,16 @@ module Mongo module DriverBench module BSON module Full + # Abstract superclass for full BSON benchmarks. + # + # @api private class Base < Mongo::DriverBench::BSON::Base private + # @return [ String ] the name of the file to use as the + # dataset for these benchmarks. def file_name - "extended_bson/full_bson.json" + 'extended_bson/full_bson.json' end end end diff --git a/profile/driver_bench/bson/full/decoding.rb b/profile/driver_bench/bson/full/decoding.rb index b598ffc37d..199f505b96 100644 --- a/profile/driver_bench/bson/full/decoding.rb +++ b/profile/driver_bench/bson/full/decoding.rb @@ -7,8 +7,14 @@ module Mongo module DriverBench module BSON module Full + # "This benchmark tests driver performance decoding documents with top + # level key/value pairs involving the full range of BSON types." + # + # @api private class Decoding < Mongo::DriverBench::BSON::Full::Base include Decodable + + bench_name 'Full BSON Decoding' end end end diff --git a/profile/driver_bench/bson/full/encoding.rb b/profile/driver_bench/bson/full/encoding.rb index f7f9283d87..45b2a845a8 100644 --- a/profile/driver_bench/bson/full/encoding.rb +++ b/profile/driver_bench/bson/full/encoding.rb @@ -7,8 +7,14 @@ module Mongo module DriverBench module BSON module Full + # "This benchmark tests driver performance encoding documents with top + # level key/value pairs involving the full range of BSON types." + # + # @api private class Encoding < Mongo::DriverBench::BSON::Full::Base include Encodable + + bench_name 'Full BSON Encoding' end end end diff --git a/profile/driver_bench/multi_doc.rb b/profile/driver_bench/multi_doc.rb index 35b92eaf1b..4cd55f6519 100644 --- a/profile/driver_bench/multi_doc.rb +++ b/profile/driver_bench/multi_doc.rb @@ -3,3 +3,14 @@ require_relative 'multi_doc/bulk_insert' require_relative 'multi_doc/find_many' require_relative 'multi_doc/grid_fs' + +module Mongo + module DriverBench + module MultiDoc + ALL = [ *BulkInsert::ALL, FindMany, *GridFS::ALL ].freeze + + # MultiBench consists of all Multi-doc micro-benchmarks + BENCH = ALL + end + end +end diff --git a/profile/driver_bench/multi_doc/bulk_insert.rb b/profile/driver_bench/multi_doc/bulk_insert.rb index e2cb7d0e06..608ecb7bf4 100644 --- a/profile/driver_bench/multi_doc/bulk_insert.rb +++ b/profile/driver_bench/multi_doc/bulk_insert.rb @@ -2,3 +2,13 @@ require_relative 'bulk_insert/large_doc' require_relative 'bulk_insert/small_doc' + +module Mongo + module DriverBench + module MultiDoc + module BulkInsert + ALL = [ LargeDoc, SmallDoc ].freeze + end + end + end +end diff --git a/profile/driver_bench/multi_doc/bulk_insert/base.rb b/profile/driver_bench/multi_doc/bulk_insert/base.rb index 06c5625e70..67e190b318 100644 --- a/profile/driver_bench/multi_doc/bulk_insert/base.rb +++ b/profile/driver_bench/multi_doc/bulk_insert/base.rb @@ -6,15 +6,19 @@ module Mongo module DriverBench module MultiDoc module BulkInsert + # Abstract superclass for all bulk insert benchmarks. + # + # @api private class Base < Mongo::DriverBench::MultiDoc::Base - attr_reader :repetitions - attr_reader :bulk_dataset + attr_reader :repetitions, :bulk_dataset def setup super @bulk_dataset = dataset * repetitions end + # How much the benchmark's dataset size ought to be scaled (for + # scoring purposes). def scale @repetitions end diff --git a/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb b/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb index 697948bb9a..94d991e19d 100644 --- a/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb +++ b/profile/driver_bench/multi_doc/bulk_insert/large_doc.rb @@ -6,14 +6,20 @@ module Mongo module DriverBench module MultiDoc module BulkInsert + # "This benchmark tests driver performance inserting multiple, large + # documents to the database." + # + # @api private class LargeDoc < Mongo::DriverBench::MultiDoc::BulkInsert::Base + bench_name 'Large doc bulk insert' + def initialize super @repetitions = 10 end def file_name - "single_and_multi_document/large_doc.json" + 'single_and_multi_document/large_doc.json' end end end diff --git a/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb b/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb index 10aeb19b21..f40dc00cab 100644 --- a/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb +++ b/profile/driver_bench/multi_doc/bulk_insert/small_doc.rb @@ -6,14 +6,20 @@ module Mongo module DriverBench module MultiDoc module BulkInsert + # "This benchmark tests driver performance inserting multiple, small + # documents to the database." + # + # @api private class SmallDoc < Mongo::DriverBench::MultiDoc::BulkInsert::Base + bench_name 'Small doc bulk insert' + def initialize super @repetitions = 10_000 end def file_name - "single_and_multi_document/small_doc.json" + 'single_and_multi_document/small_doc.json' end end end diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb index e55d1027f2..d23fb5ca58 100644 --- a/profile/driver_bench/multi_doc/find_many.rb +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -5,9 +5,17 @@ module Mongo module DriverBench module MultiDoc + # "This benchmark tests driver performance retrieving multiple documents + # from a query." + # + # @api private class FindMany < Mongo::DriverBench::MultiDoc::Base + bench_name 'Find many and empty the cursor' + + private + def file_name - "single_and_multi_document/tweet.json" + 'single_and_multi_document/tweet.json' end def setup diff --git a/profile/driver_bench/multi_doc/grid_fs.rb b/profile/driver_bench/multi_doc/grid_fs.rb index b5edbdedd1..3deb025792 100644 --- a/profile/driver_bench/multi_doc/grid_fs.rb +++ b/profile/driver_bench/multi_doc/grid_fs.rb @@ -2,3 +2,13 @@ require_relative 'grid_fs/download' require_relative 'grid_fs/upload' + +module Mongo + module DriverBench + module MultiDoc + module GridFS + ALL = [ Download, Upload ].freeze + end + end + end +end diff --git a/profile/driver_bench/multi_doc/grid_fs/base.rb b/profile/driver_bench/multi_doc/grid_fs/base.rb index 0d3b439ddc..980ec4c727 100644 --- a/profile/driver_bench/multi_doc/grid_fs/base.rb +++ b/profile/driver_bench/multi_doc/grid_fs/base.rb @@ -6,9 +6,14 @@ module Mongo module DriverBench module MultiDoc module GridFS + # Abstract base class for multi-doc GridFS benchmarks. + # + # @api private class Base < Mongo::DriverBench::MultiDoc::Base private + # how much the dataset size ought to be scaled (for scoring + # purposes). def scale 1 end diff --git a/profile/driver_bench/multi_doc/grid_fs/download.rb b/profile/driver_bench/multi_doc/grid_fs/download.rb index ce02cb7694..b4eadef261 100644 --- a/profile/driver_bench/multi_doc/grid_fs/download.rb +++ b/profile/driver_bench/multi_doc/grid_fs/download.rb @@ -7,17 +7,22 @@ module Mongo module DriverBench module MultiDoc module GridFS + # "This benchmark tests driver performance downloading a GridFS file + # to memory." + # + # @api private class Download < Mongo::DriverBench::MultiDoc::GridFS::Base + bench_name 'GridFS Download' + private - attr_reader :fs_bucket - attr_reader :file_id + attr_reader :fs_bucket, :file_id def setup super @file_id = client.database.fs - .upload_from_stream "gridfstest", dataset + .upload_from_stream 'gridfstest', dataset end def before_task diff --git a/profile/driver_bench/multi_doc/grid_fs/upload.rb b/profile/driver_bench/multi_doc/grid_fs/upload.rb index 2a598579e3..571b463bd8 100644 --- a/profile/driver_bench/multi_doc/grid_fs/upload.rb +++ b/profile/driver_bench/multi_doc/grid_fs/upload.rb @@ -6,7 +6,13 @@ module Mongo module DriverBench module MultiDoc module GridFS + # "This benchmark tests driver performance uploading a GridFS file + # from memory." + # + # @api private class Upload < Mongo::DriverBench::MultiDoc::GridFS::Base + bench_name 'GridFS Upload' + private attr_reader :fs_bucket @@ -17,7 +23,7 @@ def before_task @fs_bucket = client.database.fs @fs_bucket.drop - @fs_bucket.upload_from_stream "one-byte-file", "\n" + @fs_bucket.upload_from_stream 'one-byte-file', "\n" end def do_task diff --git a/profile/driver_bench/parallel.rb b/profile/driver_bench/parallel.rb index 27bdba3d9e..44c0d52ff5 100644 --- a/profile/driver_bench/parallel.rb +++ b/profile/driver_bench/parallel.rb @@ -2,3 +2,14 @@ require_relative 'parallel/gridfs' require_relative 'parallel/ldjson' + +module Mongo + module DriverBench + module Parallel + ALL = [ *GridFS::ALL, *LDJSON::ALL ].freeze + + # ParallelBench consists of all Parallel micro-benchmarks + BENCH = ALL + end + end +end diff --git a/profile/driver_bench/parallel/base.rb b/profile/driver_bench/parallel/base.rb index cd7b1c0276..990b70dea7 100644 --- a/profile/driver_bench/parallel/base.rb +++ b/profile/driver_bench/parallel/base.rb @@ -5,6 +5,9 @@ module Mongo module DriverBench module Parallel + # Abstract base class for parallel micro-benchmarks. + # + # @api private class Base < Mongo::DriverBench::Base private diff --git a/profile/driver_bench/parallel/counter.rb b/profile/driver_bench/parallel/counter.rb index 7574978069..3f6cde8108 100644 --- a/profile/driver_bench/parallel/counter.rb +++ b/profile/driver_bench/parallel/counter.rb @@ -1,17 +1,27 @@ # frozen_string_literal: true -require 'thread' - module Mongo module DriverBench module Parallel + # An implementation of a counter variable that can be waited on, which + # will signal when the variable reaches zero. + # + # @api private class Counter + # Create a new Counter object with the given initial value. + # + # @param [ Integer ] value the starting value of the counter (defaults + # to zero). def initialize(value = 0) @mutex = Thread::Mutex.new @condition = Thread::ConditionVariable.new @counter = value end + # Describes a block where the counter is incremented before executing + # it, and decremented afterward. + # + # @yield Calls the provided block with no arguments. def enter inc yield @@ -19,6 +29,7 @@ def enter dec end + # Waits for the counter to be zero. def wait @mutex.synchronize do return if @counter.zero? @@ -26,10 +37,13 @@ def wait end end + # Increments the counter. def inc @mutex.synchronize { @counter += 1 } end + # Decrements the counter. If the counter reaches zero, + # a signal is sent to any waiting process. def dec @mutex.synchronize do @counter -= 1 if @counter > 0 diff --git a/profile/driver_bench/parallel/dispatcher.rb b/profile/driver_bench/parallel/dispatcher.rb index c6c8c6c51b..7bccd7ea09 100644 --- a/profile/driver_bench/parallel/dispatcher.rb +++ b/profile/driver_bench/parallel/dispatcher.rb @@ -1,16 +1,33 @@ # frozen_string_literal: true require 'etc' -require 'thread' require_relative 'counter' module Mongo module DriverBench module Parallel + # Implements a dispatcher for executing multiple workers in parallel. + # + # @api private class Dispatcher attr_reader :source + # Creates a new dispatcher with the given source. The source may be any + # object that responds to ``#next``. It may be assumed that ``#next`` + # will be called in a thread-safe manner, so the source does not need + # to worry about thread-safety in that regard. Each call to ``#next`` + # on the source object should return the next batch of work to be done. + # When the source is empty, ``#next`` must return ``nil``. + # + # @param [ Object ] source an object responding to ``#next``. + # @param [ Integer ] workers the number of workers to employ in + # performing the task. + # + # @yield The associated block is executed in each worker and must + # describe the worker's task to be accomplished. + # + # @yieldparam [ Object ] batch the next batch to be worked on. def initialize(source, workers: (ENV['WORKERS'] || (Etc.nprocessors * 0.4)).to_i, &block) @source = source @counter = Counter.new @@ -20,6 +37,7 @@ def initialize(source, workers: (ENV['WORKERS'] || (Etc.nprocessors * 0.4)).to_i sleep 0.1 until @threads.all? { |t| t.status == 'sleep' } end + # Runs the workers and waits for them to finish. def run @threads.each(&:wakeup) @counter.wait @@ -27,12 +45,16 @@ def run private + # @return [ Object ] returns the next batch of work to be done (from + # the source object given when the dispatcher was created). def next_batch @source_mutex.synchronize do @source.next end end + # Fetches the next batch and passes it to the block, in a loop. + # Terminates when the next batch is ``nil``. def worker_loop(&block) loop do batch = next_batch or return diff --git a/profile/driver_bench/parallel/gridfs.rb b/profile/driver_bench/parallel/gridfs.rb index 5312304561..9ef69d5451 100644 --- a/profile/driver_bench/parallel/gridfs.rb +++ b/profile/driver_bench/parallel/gridfs.rb @@ -2,3 +2,13 @@ require_relative 'gridfs/download' require_relative 'gridfs/upload' + +module Mongo + module DriverBench + module Parallel + module GridFS + ALL = [ Download, Upload ].freeze + end + end + end +end diff --git a/profile/driver_bench/parallel/gridfs/base.rb b/profile/driver_bench/parallel/gridfs/base.rb index 638f3cdb4d..40a25a0790 100644 --- a/profile/driver_bench/parallel/gridfs/base.rb +++ b/profile/driver_bench/parallel/gridfs/base.rb @@ -6,9 +6,12 @@ module Mongo module DriverBench module Parallel module GridFS + # Abstract base class of parallel GridFS micro-benchmarks. + # + # @api private class Base < Mongo::DriverBench::Parallel::Base def file_name_at(index) - format("parallel/gridfs_multi/file%02d.txt", index) + format('parallel/gridfs_multi/file%02d.txt', index) end private @@ -23,7 +26,7 @@ def setup def prepare_bucket(initialize: true) @bucket = client.database.fs @bucket.drop - @bucket.upload_from_stream "one-byte-file", "\n" if initialize + @bucket.upload_from_stream 'one-byte-file', "\n" if initialize end def upload_file(file_name) diff --git a/profile/driver_bench/parallel/gridfs/download.rb b/profile/driver_bench/parallel/gridfs/download.rb index 3d64ea35cc..9a2671706c 100644 --- a/profile/driver_bench/parallel/gridfs/download.rb +++ b/profile/driver_bench/parallel/gridfs/download.rb @@ -8,9 +8,20 @@ module Mongo module DriverBench module Parallel module GridFS + # This benchmark tests driver performance downloading files from + # GridFS to disk. + # + # @api private class Download < Mongo::DriverBench::Parallel::GridFS::Base + bench_name 'GridFS multi-file download' + private + # The source object to use for this benchmark. Each batch is a tuple + # consisting of the list position, and the element in the list at + # that position. + # + # @api private class Source def initialize(list) @list = list diff --git a/profile/driver_bench/parallel/gridfs/upload.rb b/profile/driver_bench/parallel/gridfs/upload.rb index 24974bd669..3e9cbb6f0f 100644 --- a/profile/driver_bench/parallel/gridfs/upload.rb +++ b/profile/driver_bench/parallel/gridfs/upload.rb @@ -7,7 +7,17 @@ module Mongo module DriverBench module Parallel module GridFS + # "This benchmark tests driver performance uploading files from disk + # to GridFS." + # + # @api private class Upload < Mongo::DriverBench::Parallel::GridFS::Base + bench_name 'GridFS multi-file upload' + + # The source object to use for this benchmark. Each batch consists + # of the name of the file to upload. + # + # @api private class Source def initialize(bench) @n = 0 @@ -16,6 +26,7 @@ def initialize(bench) def next return nil if @n >= 50 + @bench.file_name_at(@n).tap { @n += 1 } end end diff --git a/profile/driver_bench/parallel/ldjson.rb b/profile/driver_bench/parallel/ldjson.rb index 7d6b4b6fa6..1a3c76375c 100644 --- a/profile/driver_bench/parallel/ldjson.rb +++ b/profile/driver_bench/parallel/ldjson.rb @@ -2,3 +2,13 @@ require_relative 'ldjson/export' require_relative 'ldjson/import' + +module Mongo + module DriverBench + module Parallel + module LDJSON + ALL = [ Export, Import ].freeze + end + end + end +end diff --git a/profile/driver_bench/parallel/ldjson/base.rb b/profile/driver_bench/parallel/ldjson/base.rb index 682c860121..ebb554bff4 100644 --- a/profile/driver_bench/parallel/ldjson/base.rb +++ b/profile/driver_bench/parallel/ldjson/base.rb @@ -6,9 +6,12 @@ module Mongo module DriverBench module Parallel module LDJSON + # The abstract base class for parallel LDSON benchmarks. + # + # @api private class Base < Mongo::DriverBench::Parallel::Base def file_name_at(index) - format("parallel/ldjson_multi/ldjson%03d.txt", index) + format('parallel/ldjson_multi/ldjson%03d.txt', index) end private diff --git a/profile/driver_bench/parallel/ldjson/export.rb b/profile/driver_bench/parallel/ldjson/export.rb index 4ff6f00f9b..f6faac1183 100644 --- a/profile/driver_bench/parallel/ldjson/export.rb +++ b/profile/driver_bench/parallel/ldjson/export.rb @@ -10,9 +10,17 @@ module Mongo module DriverBench module Parallel module LDJSON + # "This benchmark tests driver performance exporting documents to a + # set of LDJSON files." + # + # @api private class Export < Mongo::DriverBench::Parallel::LDJSON::Base + bench_name 'LDJSON multi-file export' + private + # The data source for this benchmark; each batch is a set of 5000 + # documents. class DataSource def initialize(collection) @n = 0 diff --git a/profile/driver_bench/parallel/ldjson/import.rb b/profile/driver_bench/parallel/ldjson/import.rb index cc26fc2f20..e7c41f7497 100644 --- a/profile/driver_bench/parallel/ldjson/import.rb +++ b/profile/driver_bench/parallel/ldjson/import.rb @@ -7,9 +7,17 @@ module Mongo module DriverBench module Parallel module LDJSON + # "This benchmark tests driver performance importing documents from a + # set of LDJSON files." + # + # @api private class Import < Mongo::DriverBench::Parallel::LDJSON::Base + bench_name 'LDJSON multi-file import' + private + # The data source for this benchmark. Each batch is the name of a + # file to read documents file. class DataSource def initialize(bench) @n = 0 diff --git a/profile/driver_bench/rake/tasks.rake b/profile/driver_bench/rake/tasks.rake new file mode 100644 index 0000000000..486051a6e2 --- /dev/null +++ b/profile/driver_bench/rake/tasks.rake @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require_relative '../suite' + +desc 'Runs the DriverBench benchmark suite' +task :driver_bench do + Mongo::DriverBench::Suite.run! +end diff --git a/profile/driver_bench/single_doc.rb b/profile/driver_bench/single_doc.rb index c20395dec7..878f818379 100644 --- a/profile/driver_bench/single_doc.rb +++ b/profile/driver_bench/single_doc.rb @@ -3,3 +3,15 @@ require_relative 'single_doc/find_one_by_id' require_relative 'single_doc/insert_one' require_relative 'single_doc/run_command' + +module Mongo + module DriverBench + module SingleDoc + ALL = [ FindOneByID, *InsertOne::ALL, RunCommand ].freeze + + # SingleBench consists of all Single-doc micro-benchmarks + # except "Run Command" + BENCH = (ALL - [ RunCommand ]).freeze + end + end +end diff --git a/profile/driver_bench/single_doc/base.rb b/profile/driver_bench/single_doc/base.rb index 4e379afba2..cf540a9a4e 100644 --- a/profile/driver_bench/single_doc/base.rb +++ b/profile/driver_bench/single_doc/base.rb @@ -5,11 +5,13 @@ module Mongo module DriverBench module SingleDoc + # Abstract base class for all single-doc benchmarks. + # + # @api private class Base < Mongo::DriverBench::Base private - attr_reader :client - attr_reader :collection + attr_reader :client, :collection def setup if file_name @@ -20,6 +22,8 @@ def setup prepare_client end + # The amount by which the dataset size should be scaled (for scoring + # purposes). def scale 10_000 end diff --git a/profile/driver_bench/single_doc/find_one_by_id.rb b/profile/driver_bench/single_doc/find_one_by_id.rb index c896d6dc13..311ea0380b 100644 --- a/profile/driver_bench/single_doc/find_one_by_id.rb +++ b/profile/driver_bench/single_doc/find_one_by_id.rb @@ -5,9 +5,15 @@ module Mongo module DriverBench module SingleDoc + # "This benchmark tests driver performance sending an indexed query to + # the database and reading a single document in response." + # + # @api private class FindOneByID < Mongo::DriverBench::SingleDoc::Base + bench_name 'Find one by ID' + def file_name - "single_and_multi_document/tweet.json" + 'single_and_multi_document/tweet.json' end def setup diff --git a/profile/driver_bench/single_doc/insert_one.rb b/profile/driver_bench/single_doc/insert_one.rb index 3a231267f2..64dc442530 100644 --- a/profile/driver_bench/single_doc/insert_one.rb +++ b/profile/driver_bench/single_doc/insert_one.rb @@ -2,3 +2,13 @@ require_relative 'insert_one/large_doc' require_relative 'insert_one/small_doc' + +module Mongo + module DriverBench + module SingleDoc + module InsertOne + ALL = [ LargeDoc, SmallDoc ].freeze + end + end + end +end diff --git a/profile/driver_bench/single_doc/insert_one/base.rb b/profile/driver_bench/single_doc/insert_one/base.rb index 4f4262a929..afafd9833d 100644 --- a/profile/driver_bench/single_doc/insert_one/base.rb +++ b/profile/driver_bench/single_doc/insert_one/base.rb @@ -6,8 +6,12 @@ module Mongo module DriverBench module SingleDoc module InsertOne + # Abstract base class for "insert one" benchmarks. + # + # @api private class Base < Mongo::DriverBench::SingleDoc::Base attr_reader :repetitions + alias scale repetitions def before_task collection.drop diff --git a/profile/driver_bench/single_doc/insert_one/large_doc.rb b/profile/driver_bench/single_doc/insert_one/large_doc.rb index c97c5680ce..6e3db91ad7 100644 --- a/profile/driver_bench/single_doc/insert_one/large_doc.rb +++ b/profile/driver_bench/single_doc/insert_one/large_doc.rb @@ -6,18 +6,20 @@ module Mongo module DriverBench module SingleDoc module InsertOne + # "This benchmark tests driver performance inserting a single, large + # document to the database." + # + # @api private class LargeDoc < Mongo::DriverBench::SingleDoc::InsertOne::Base + bench_name 'Large doc insertOne' + def initialize super @repetitions = 10 end - def scale - @repetitions - end - def file_name - "single_and_multi_document/large_doc.json" + 'single_and_multi_document/large_doc.json' end end end diff --git a/profile/driver_bench/single_doc/insert_one/small_doc.rb b/profile/driver_bench/single_doc/insert_one/small_doc.rb index b8c2a8c7d7..41289630a5 100644 --- a/profile/driver_bench/single_doc/insert_one/small_doc.rb +++ b/profile/driver_bench/single_doc/insert_one/small_doc.rb @@ -6,18 +6,20 @@ module Mongo module DriverBench module SingleDoc module InsertOne + # "This benchmark tests driver performance inserting a single, small + # document to the database." + # + # @api private class SmallDoc < Mongo::DriverBench::SingleDoc::InsertOne::Base + bench_name 'Small doc insertOne' + def initialize super @repetitions = 10_000 end - def scale - @repetitions - end - def file_name - "single_and_multi_document/small_doc.json" + 'single_and_multi_document/small_doc.json' end end end diff --git a/profile/driver_bench/single_doc/run_command.rb b/profile/driver_bench/single_doc/run_command.rb index b267e9c9e0..c6fa423c38 100644 --- a/profile/driver_bench/single_doc/run_command.rb +++ b/profile/driver_bench/single_doc/run_command.rb @@ -5,7 +5,13 @@ module Mongo module DriverBench module SingleDoc + # "This benchmark tests driver performance sending a command to the + # database and reading a response." + # + # @api private class RunCommand < Mongo::DriverBench::SingleDoc::Base + bench_name 'Run command' + def setup super @dataset_size = { hello: true }.to_bson.length * scale diff --git a/profile/driver_bench/suite.rb b/profile/driver_bench/suite.rb new file mode 100644 index 0000000000..9aca6dd340 --- /dev/null +++ b/profile/driver_bench/suite.rb @@ -0,0 +1,128 @@ +require_relative 'bson' +require_relative 'multi_doc' +require_relative 'parallel' +require_relative 'single_doc' + +module Mongo + module DriverBench + ALL = [ *BSON::ALL, *SingleDoc::ALL, *MultiDoc::ALL, *Parallel::ALL ] + + BENCHES = { + 'BSONBench' => BSON::BENCH, + 'SingleBench' => SingleDoc::BENCH, + 'MultiBench' => MultiDoc::BENCH, + 'ParallelBench' => Parallel::BENCH, + + 'ReadBench' => [ + SingleDoc::FindOneByID, + MultiDoc::FindMany, + MultiDoc::GridFS::Download, + Parallel::LDJSON::Export, + Parallel::GridFS::Download + ].freeze, + + 'WriteBench' => [ + SingleDoc::InsertOne::SmallDoc, + SingleDoc::InsertOne::LargeDoc, + MultiDoc::BulkInsert::SmallDoc, + MultiDoc::BulkInsert::LargeDoc, + MultiDoc::GridFS::Upload, + Parallel::LDJSON::Import, + Parallel::GridFS::Upload + ].freeze + }.freeze + + # A benchmark suite for running all benchmarks and aggregating (and + # reporting) the results. + # + # @api private + class Suite + PERCENTILES = [ 10, 25, 50, 75, 90, 95, 98, 99 ].freeze + + def self.run! + new.run + end + + def run + perf_data = [] + benches = Hash.new { |h,k| h[k] = [] } + + ALL.each do |klass| + result = run_benchmark(klass) + perf_data << compile_perf_data(result) + append_to_benchmarks(klass, result, benches) + end + + perf_data += compile_benchmarks(benches) + + save_perf_data(perf_data) + summarize_perf_data(perf_data) + end + + private + + def run_benchmark(klass) + print klass.bench_name, ': '; $stdout.flush + klass.new.run.tap do |result| + puts format('%4.4g', result[:score]) + end + end + + def compile_perf_data(result) + percentile_data = PERCENTILES.each_with_object({}) do |n, hash| + hash["time-#{n}%"] = result[:percentiles][n] + end + + { + 'info' => { + 'test_name' => result[:name], + 'args' => {}, + }, + 'metrics' => percentile_data.merge('score' => result[:score]), + } + end + + def append_to_benchmarks(klass, result, benches) + BENCHES.each do |benchmark, list| + benches[benchmark] << result[:score] if list.include?(klass) + end + end + + def compile_benchmarks(benches) + benches.keys.each do |key| + benches[key] = benches[key].sum / benches[key].length + end + + benches['DriverBench'] = (benches['ReadBench'] + benches['WriteBench']) / 2 + + benches.map do |bench, score| + { + 'info' => { + 'test_name' => bench, + 'args' => {} + }, + 'metrics' => { + 'score' => score + } + } + end + end + + def summarize_perf_data(data) + puts '===== Performance Results =====' + data.each do |item| + puts format('%s : %4.4g', item['info']['test_name'], item['metrics']['score']) + next unless item['metrics']['time-10%'] + + PERCENTILES.each do |n| + puts format(' %d%% : %4.4g', n, item['metrics']["time-#{n}%"]) + end + end + end + + def save_perf_data(data) + File.write('results.json', data.to_json) + end + end + end +end From d0076306231f552940b45d5357f1f7ce43267f6f Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 19 Jan 2024 14:53:35 -0700 Subject: [PATCH 04/11] make the linter happy --- profile/driver_bench/base.rb | 16 ++++++++-------- profile/driver_bench/bson/base.rb | 2 ++ profile/driver_bench/multi_doc/base.rb | 7 +++++-- profile/driver_bench/multi_doc/find_many.rb | 2 +- profile/driver_bench/parallel/counter.rb | 3 ++- profile/driver_bench/parallel/dispatcher.rb | 14 +++++++++++--- profile/driver_bench/parallel/gridfs/download.rb | 4 ++-- profile/driver_bench/parallel/ldjson/export.rb | 4 ++-- .../driver_bench/single_doc/insert_one/base.rb | 2 +- profile/driver_bench/single_doc/run_command.rb | 2 +- profile/driver_bench/suite.rb | 12 ++++++++---- 11 files changed, 43 insertions(+), 25 deletions(-) diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index 072fa5ee67..444d824a9f 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -70,6 +70,8 @@ def run # # @return [ Array ] the array of timings (in seconds) for # each iteration. + # + # rubocop:disable Metrics/AbcSize def run_benchmark [].tap do |timings| iteration_count = 0 @@ -98,6 +100,7 @@ def run_benchmark teardown end end + # rubocop:enable Metrics/AbcSize # Instantiate a new client. def new_client(uri = ENV['MONGODB_URI']) @@ -116,6 +119,7 @@ def without_gc # DATA_PATH, unless the file name is an absolute path. def path_to_file(file_name) return file_name if file_name.start_with?('/') + File.join(DATA_PATH, file_name) end @@ -146,12 +150,10 @@ def parse_line(document) end # Executed at the start of the micro-benchmark. - def setup - end + def setup; end # Executed before each iteration of the benchmark. - def before_task - end + def before_task; end # Smallest amount of code necessary to do the task, # invoked once per iteration. @@ -160,12 +162,10 @@ def do_task end # Executed after each iteration of the benchmark. - def after_task - end + def after_task; end # Executed at the end of the micro-benchmark. - def teardown - end + def teardown; end end end end diff --git a/profile/driver_bench/bson/base.rb b/profile/driver_bench/bson/base.rb index 59aa69520a..f976a00498 100644 --- a/profile/driver_bench/bson/base.rb +++ b/profile/driver_bench/bson/base.rb @@ -13,8 +13,10 @@ class Base < Mongo::DriverBench::Base # Common setup for these benchmarks. def setup + # rubocop:disable Naming/MemoizedInstanceVariableName @dataset ||= load_file(file_name).first @dataset_size ||= size_of_file(file_name) * 10_000 + # rubocop:enable Naming/MemoizedInstanceVariableName end # Returns the name of the file name that contains diff --git a/profile/driver_bench/multi_doc/base.rb b/profile/driver_bench/multi_doc/base.rb index fa5bddecc6..16323d6140 100644 --- a/profile/driver_bench/multi_doc/base.rb +++ b/profile/driver_bench/multi_doc/base.rb @@ -5,11 +5,13 @@ module Mongo module DriverBench module MultiDoc + # Abstract base class for multi-doc benchmarks. + # + # @api private class Base < Mongo::DriverBench::Base private - attr_reader :client - attr_reader :collection + attr_reader :client, :collection def setup if file_name @@ -20,6 +22,7 @@ def setup prepare_client end + # The amount to scale the dataset size by (for scoring purposes). def scale 10_000 end diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb index d23fb5ca58..c2ad46f4fe 100644 --- a/profile/driver_bench/multi_doc/find_many.rb +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -21,7 +21,7 @@ def file_name def setup super - 10_000.times do |i| + 10_000.times do @collection.insert_one(dataset.first) end end diff --git a/profile/driver_bench/parallel/counter.rb b/profile/driver_bench/parallel/counter.rb index 3f6cde8108..b21f41bee0 100644 --- a/profile/driver_bench/parallel/counter.rb +++ b/profile/driver_bench/parallel/counter.rb @@ -33,6 +33,7 @@ def enter def wait @mutex.synchronize do return if @counter.zero? + @condition.wait(@mutex) end end @@ -46,7 +47,7 @@ def inc # a signal is sent to any waiting process. def dec @mutex.synchronize do - @counter -= 1 if @counter > 0 + @counter -= 1 if @counter.positive? @condition.signal if @counter.zero? end end diff --git a/profile/driver_bench/parallel/dispatcher.rb b/profile/driver_bench/parallel/dispatcher.rb index 7bccd7ea09..aed94325d3 100644 --- a/profile/driver_bench/parallel/dispatcher.rb +++ b/profile/driver_bench/parallel/dispatcher.rb @@ -33,7 +33,15 @@ def initialize(source, workers: (ENV['WORKERS'] || (Etc.nprocessors * 0.4)).to_i @counter = Counter.new @source_mutex = Thread::Mutex.new - @threads = Array.new(workers).map { Thread.new { @counter.enter { Thread.stop; worker_loop(&block) } } } + @threads = Array.new(workers).map do + Thread.new do + @counter.enter do + Thread.stop + worker_loop(&block) + end + end + end + sleep 0.1 until @threads.all? { |t| t.status == 'sleep' } end @@ -55,10 +63,10 @@ def next_batch # Fetches the next batch and passes it to the block, in a loop. # Terminates when the next batch is ``nil``. - def worker_loop(&block) + def worker_loop loop do batch = next_batch or return - block.call(batch) + yield batch end end end diff --git a/profile/driver_bench/parallel/gridfs/download.rb b/profile/driver_bench/parallel/gridfs/download.rb index 9a2671706c..1448a1f257 100644 --- a/profile/driver_bench/parallel/gridfs/download.rb +++ b/profile/driver_bench/parallel/gridfs/download.rb @@ -61,8 +61,8 @@ def do_task @dispatcher.run end - def download_file(n, id) - path = File.join(@destination, file_name_at(n)) + def download_file(index, id) + path = File.join(@destination, file_name_at(index)) FileUtils.mkdir_p(File.dirname(path)) File.open(path, 'w') do |file| diff --git a/profile/driver_bench/parallel/ldjson/export.rb b/profile/driver_bench/parallel/ldjson/export.rb index f6faac1183..6121660299 100644 --- a/profile/driver_bench/parallel/ldjson/export.rb +++ b/profile/driver_bench/parallel/ldjson/export.rb @@ -63,8 +63,8 @@ def teardown FileUtils.rm_rf(@destination) end - def worker_task(n, batch) - path = File.join(@destination, file_name_at(n)) + def worker_task(index, batch) + path = File.join(@destination, file_name_at(index)) FileUtils.mkdir_p(File.dirname(path)) File.write(path, batch.map(&:to_json).join("\n")) end diff --git a/profile/driver_bench/single_doc/insert_one/base.rb b/profile/driver_bench/single_doc/insert_one/base.rb index afafd9833d..cb111a5396 100644 --- a/profile/driver_bench/single_doc/insert_one/base.rb +++ b/profile/driver_bench/single_doc/insert_one/base.rb @@ -19,7 +19,7 @@ def before_task end def do_task - repetitions.times do |i| + repetitions.times do collection.insert_one(dataset) end end diff --git a/profile/driver_bench/single_doc/run_command.rb b/profile/driver_bench/single_doc/run_command.rb index c6fa423c38..55bed0c6a6 100644 --- a/profile/driver_bench/single_doc/run_command.rb +++ b/profile/driver_bench/single_doc/run_command.rb @@ -26,7 +26,7 @@ def cleanup_client end def do_task - 10_000.times do |i| + 10_000.times do client.database.command(hello: true) end end diff --git a/profile/driver_bench/suite.rb b/profile/driver_bench/suite.rb index 9aca6dd340..ddc1875eb4 100644 --- a/profile/driver_bench/suite.rb +++ b/profile/driver_bench/suite.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative 'bson' require_relative 'multi_doc' require_relative 'parallel' @@ -5,7 +7,7 @@ module Mongo module DriverBench - ALL = [ *BSON::ALL, *SingleDoc::ALL, *MultiDoc::ALL, *Parallel::ALL ] + ALL = [ *BSON::ALL, *SingleDoc::ALL, *MultiDoc::ALL, *Parallel::ALL ].freeze BENCHES = { 'BSONBench' => BSON::BENCH, @@ -45,7 +47,7 @@ def self.run! def run perf_data = [] - benches = Hash.new { |h,k| h[k] = [] } + benches = Hash.new { |h, k| h[k] = [] } ALL.each do |klass| result = run_benchmark(klass) @@ -62,7 +64,9 @@ def run private def run_benchmark(klass) - print klass.bench_name, ': '; $stdout.flush + print klass.bench_name, ': ' + $stdout.flush + klass.new.run.tap do |result| puts format('%4.4g', result[:score]) end @@ -89,7 +93,7 @@ def append_to_benchmarks(klass, result, benches) end def compile_benchmarks(benches) - benches.keys.each do |key| + benches.each_key do |key| benches[key] = benches[key].sum / benches[key].length end From 19aeb47beadf6c64fcd0dba6afda31d5688b29d9 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 19 Jan 2024 15:00:44 -0700 Subject: [PATCH 05/11] optimize some of the setup stages --- profile/driver_bench/base.rb | 10 +++++++--- profile/driver_bench/multi_doc/find_many.rb | 5 ++--- profile/driver_bench/single_doc/find_one_by_id.rb | 6 ++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index 444d824a9f..322fc7ca87 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -41,11 +41,15 @@ def self.bench_name(benchmark_name = nil) # Instantiate a new micro-benchmark class. def initialize - @max_iterations = 100 - @min_time = ENV['CHEAT'] ? 10 : 60 + @max_iterations = debug_mode? ? 10 : 100 + @min_time = debug_mode? ? 1 : 60 @max_time = 300 # 5 minutes end + def debug_mode? + ENV['PERF_DEBUG'] + end + # Runs the benchmark and returns the score. # # @return [ Hash ] the score and other @@ -81,7 +85,7 @@ def run_benchmark loop do before_task - timing = without_gc { Benchmark.realtime { ENV['CHEAT'] ? sleep(0.1) : do_task } } + timing = without_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } } after_task iteration_count += 1 diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb index c2ad46f4fe..ba755d8243 100644 --- a/profile/driver_bench/multi_doc/find_many.rb +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -21,9 +21,8 @@ def file_name def setup super - 10_000.times do - @collection.insert_one(dataset.first) - end + docs = 10_000.times.map { dataset.first } + @collection.insert_many(docs) end def do_task diff --git a/profile/driver_bench/single_doc/find_one_by_id.rb b/profile/driver_bench/single_doc/find_one_by_id.rb index 311ea0380b..6a262b059e 100644 --- a/profile/driver_bench/single_doc/find_one_by_id.rb +++ b/profile/driver_bench/single_doc/find_one_by_id.rb @@ -19,10 +19,8 @@ def file_name def setup super - doc = dataset - 10_000.times do |i| - @collection.insert_one(doc.merge(_id: i + 1)) - end + docs = 10_000.times.map { |i| dataset.merge(_id: i + 1) } + @collection.insert_many(docs) end def do_task From 5c34a443cb6cf93a255439e8f79db39d9a1c469f Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 22 Jan 2024 15:18:03 -0700 Subject: [PATCH 06/11] add benchmark task to evergreen config --- .evergreen/config.yml | 26 +++++++++ .evergreen/config/common.yml.erb | 16 +++++ .evergreen/config/standard.yml.erb | 10 ++++ .evergreen/run-benchmarks.sh | 93 ++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100755 .evergreen/run-benchmarks.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 934829aae4..9c0ac7f75d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -377,6 +377,19 @@ functions: set -x .evergreen/test-on-docker -d ${os} MONGODB_VERSION=${mongodb-version} TOPOLOGY=${topology} RVM_RUBY=${ruby} -s .evergreen/run-tests.sh TEST_CMD=true ${PRELOAD_ARG} + "run benchmarks": + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-benchmarks.sh + - command: perf.send + params: + file: ./results.json + "run tests": - command: shell.exec type: test @@ -665,6 +678,9 @@ tasks: - name: "test-mlaunch" commands: - func: "run tests" + - name: "driver-bench" + commands: + - func: "run benchmarks" - name: "test-via-docker" commands: - func: "run tests via docker" @@ -1215,6 +1231,16 @@ axes: USE_PROXY_SERVERLESS: 1 buildvariants: + - matrix_name: DriverBench + matrix_spec: + ruby: "ruby-3.2" + mongodb-version: latest + topology: standalone + os: rhel8 + display_name: DriverBench + tasks: + - name: "driver-bench" + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 54eee122a1..14010142a2 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -374,6 +374,19 @@ functions: set -x .evergreen/test-on-docker -d ${os} MONGODB_VERSION=${mongodb-version} TOPOLOGY=${topology} RVM_RUBY=${ruby} -s .evergreen/run-tests.sh TEST_CMD=true ${PRELOAD_ARG} + "run benchmarks": + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + script: | + ${PREPARE_SHELL} + .evergreen/run-benchmarks.sh + - command: perf.send + params: + file: ./results.json + "run tests": - command: shell.exec type: test @@ -662,6 +675,9 @@ tasks: - name: "test-mlaunch" commands: - func: "run tests" + - name: "driver-bench" + commands: + - func: "run benchmarks" - name: "test-via-docker" commands: - func: "run tests via docker" diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 1964946d4c..e431a07a53 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -41,6 +41,16 @@ %> buildvariants: + - matrix_name: DriverBench + matrix_spec: + ruby: <%= latest_ruby %> + mongodb-version: latest + topology: standalone + os: rhel8 + display_name: DriverBench + tasks: + - name: "driver-bench" + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh new file mode 100755 index 0000000000..38b86206aa --- /dev/null +++ b/.evergreen/run-benchmarks.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -e +set -o pipefail + +MRSS_ROOT=`dirname "$0"`/../spec/shared + +. $MRSS_ROOT/shlib/distro.sh +. $MRSS_ROOT/shlib/set_env.sh +. $MRSS_ROOT/shlib/server.sh +. $MRSS_ROOT/shlib/config.sh +. `dirname "$0"`/functions.sh +. `dirname "$0"`/functions-aws.sh +. `dirname "$0"`/functions-config.sh + +arch=`host_distro` + +set_home +set_env_vars +set_env_python +set_env_ruby + +prepare_server $arch + +install_mlaunch_venv + +# Launching mongod under $MONGO_ORCHESTRATION_HOME +# makes its log available through log collecting machinery + +export dbdir="$MONGO_ORCHESTRATION_HOME"/db +mkdir -p "$dbdir" + +if test -z "$TOPOLOGY"; then + export TOPOLOGY=standalone +fi + +calculate_server_args +launch_server "$dbdir" + +uri_options="$URI_OPTIONS" + +bundle_install + +if test "$TOPOLOGY" = sharded-cluster; then + if test -n "$SINGLE_MONGOS"; then + echo Restricting to a single mongos + hosts=localhost:27017 + else + hosts=localhost:27017,localhost:27018 + fi +elif test "$TOPOLOGY" = replica-set; then + hosts=localhost:27017,localhost:27018 + uri_options="$uri_options&replicaSet=test-rs" +else + hosts=localhost:27017 +fi + +hosts="bob:pwd123@$hosts" + +if test -n "$EXTRA_URI_OPTIONS"; then + uri_options="$uri_options&$EXTRA_URI_OPTIONS" +fi + +export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options" + +set_fcv + +if test "$TOPOLOGY" = replica-set && ! echo "$MONGODB_VERSION" |fgrep -q 2.6; then + ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.setup_tags +fi + +if test "$API_VERSION_REQUIRED" = 1; then + ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.require_api_version + export SERVER_API='version: "1"' +fi + +if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then + # On 3.6 server the sessions collection is not immediately available, + # wait for it to spring into existence + bundle exec rake spec:wait_for_sessions +fi + +export MONGODB_URI="mongodb://$hosts/?appName=test-suite$uri_options" + +echo "Running benchmarks" +bundle exec rake driver_bench + +bm_status=$? +echo "BENCHMARK STATUS: ${bm_status}" + +python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir" + +exit ${test_status} From 9b9379ce3b5037740739bc862cd421d700e41c46 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 22 Jan 2024 15:22:01 -0700 Subject: [PATCH 07/11] rubocop --- profile/driver_bench/multi_doc/find_many.rb | 2 +- profile/driver_bench/single_doc/find_one_by_id.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb index ba755d8243..8454f03e07 100644 --- a/profile/driver_bench/multi_doc/find_many.rb +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -21,7 +21,7 @@ def file_name def setup super - docs = 10_000.times.map { dataset.first } + docs = Array.new(10_000) { dataset.first } @collection.insert_many(docs) end diff --git a/profile/driver_bench/single_doc/find_one_by_id.rb b/profile/driver_bench/single_doc/find_one_by_id.rb index 6a262b059e..1aff97a360 100644 --- a/profile/driver_bench/single_doc/find_one_by_id.rb +++ b/profile/driver_bench/single_doc/find_one_by_id.rb @@ -19,7 +19,7 @@ def file_name def setup super - docs = 10_000.times.map { |i| dataset.merge(_id: i + 1) } + docs = Array.new(10_000) { |i| dataset.merge(_id: i + 1) } @collection.insert_many(docs) end From 7e10579aca523020e6f9feaab328ede7ac163922 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 22 Jan 2024 15:50:04 -0700 Subject: [PATCH 08/11] add task for downloading benchmark data files --- profile/driver_bench/rake/tasks.rake | 36 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/profile/driver_bench/rake/tasks.rake b/profile/driver_bench/rake/tasks.rake index 486051a6e2..b9e148df35 100644 --- a/profile/driver_bench/rake/tasks.rake +++ b/profile/driver_bench/rake/tasks.rake @@ -2,7 +2,37 @@ require_relative '../suite' -desc 'Runs the DriverBench benchmark suite' -task :driver_bench do - Mongo::DriverBench::Suite.run! +task driver_bench: %i[ driver_bench:data driver_bench:run ] + +SPECS_REPO_URI = 'git@github.com:mongodb/specifications.git' +SPECS_PATH = File.expand_path('../../../specifications', __dir__) +DRIVER_BENCH_DATA = File.expand_path('../../data/driver_bench', __dir__) + +namespace :driver_bench do + desc 'Downloads the DriverBench data files, if necessary' + task :data do + if File.directory?('./profile/data/driver_bench') + puts 'DriverBench data files are already downloaded' + next + end + + if File.directory?(SPECS_PATH) + puts 'specifications repo is already checked out' + else + sh 'git', 'clone', SPECS_REPO_URI + end + + mkdir_p DRIVER_BENCH_DATA + + Dir.glob(File.join(SPECS_PATH, 'source/benchmarking/data/*.tgz')) do |archive| + Dir.chdir(DRIVER_BENCH_DATA) do + sh 'tar', 'xzf', archive + end + end + end + + desc 'Runs the DriverBench benchmark suite' + task :run do + Mongo::DriverBench::Suite.run! + end end From 3535b12e137830966c548bb93540674965d9ff2d Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 22 Jan 2024 16:05:20 -0700 Subject: [PATCH 09/11] just use run-tests.sh for benchmarks, too --- .evergreen/config.yml | 4 +- .evergreen/config/common.yml.erb | 2 +- .evergreen/config/standard.yml.erb | 2 +- .evergreen/run-benchmarks.sh | 93 ------------------------------ 4 files changed, 4 insertions(+), 97 deletions(-) delete mode 100755 .evergreen/run-benchmarks.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9c0ac7f75d..dd2240a413 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -385,7 +385,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - .evergreen/run-benchmarks.sh + TEST_CMD="bundle exec rake driver_bench" .evergreen/run-tests.sh - command: perf.send params: file: ./results.json @@ -1236,7 +1236,7 @@ buildvariants: ruby: "ruby-3.2" mongodb-version: latest topology: standalone - os: rhel8 + run_on: rhel80-large display_name: DriverBench tasks: - name: "driver-bench" diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 14010142a2..848506348c 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -382,7 +382,7 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - .evergreen/run-benchmarks.sh + TEST_CMD="bundle exec rake driver_bench" .evergreen/run-tests.sh - command: perf.send params: file: ./results.json diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index e431a07a53..dcca0c673e 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -46,7 +46,7 @@ buildvariants: ruby: <%= latest_ruby %> mongodb-version: latest topology: standalone - os: rhel8 + run_on: rhel80-large display_name: DriverBench tasks: - name: "driver-bench" diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh deleted file mode 100755 index 38b86206aa..0000000000 --- a/.evergreen/run-benchmarks.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -set -e -set -o pipefail - -MRSS_ROOT=`dirname "$0"`/../spec/shared - -. $MRSS_ROOT/shlib/distro.sh -. $MRSS_ROOT/shlib/set_env.sh -. $MRSS_ROOT/shlib/server.sh -. $MRSS_ROOT/shlib/config.sh -. `dirname "$0"`/functions.sh -. `dirname "$0"`/functions-aws.sh -. `dirname "$0"`/functions-config.sh - -arch=`host_distro` - -set_home -set_env_vars -set_env_python -set_env_ruby - -prepare_server $arch - -install_mlaunch_venv - -# Launching mongod under $MONGO_ORCHESTRATION_HOME -# makes its log available through log collecting machinery - -export dbdir="$MONGO_ORCHESTRATION_HOME"/db -mkdir -p "$dbdir" - -if test -z "$TOPOLOGY"; then - export TOPOLOGY=standalone -fi - -calculate_server_args -launch_server "$dbdir" - -uri_options="$URI_OPTIONS" - -bundle_install - -if test "$TOPOLOGY" = sharded-cluster; then - if test -n "$SINGLE_MONGOS"; then - echo Restricting to a single mongos - hosts=localhost:27017 - else - hosts=localhost:27017,localhost:27018 - fi -elif test "$TOPOLOGY" = replica-set; then - hosts=localhost:27017,localhost:27018 - uri_options="$uri_options&replicaSet=test-rs" -else - hosts=localhost:27017 -fi - -hosts="bob:pwd123@$hosts" - -if test -n "$EXTRA_URI_OPTIONS"; then - uri_options="$uri_options&$EXTRA_URI_OPTIONS" -fi - -export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options" - -set_fcv - -if test "$TOPOLOGY" = replica-set && ! echo "$MONGODB_VERSION" |fgrep -q 2.6; then - ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.setup_tags -fi - -if test "$API_VERSION_REQUIRED" = 1; then - ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.require_api_version - export SERVER_API='version: "1"' -fi - -if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then - # On 3.6 server the sessions collection is not immediately available, - # wait for it to spring into existence - bundle exec rake spec:wait_for_sessions -fi - -export MONGODB_URI="mongodb://$hosts/?appName=test-suite$uri_options" - -echo "Running benchmarks" -bundle exec rake driver_bench - -bm_status=$? -echo "BENCHMARK STATUS: ${bm_status}" - -python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir" - -exit ${test_status} From 555a843ba8c3a6defdf5fe899f4e0d2cac684865 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 22 Jan 2024 16:06:50 -0700 Subject: [PATCH 10/11] add `eg` tasks for evergreen --- Rakefile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Rakefile b/Rakefile index 93578ae1b7..627e39389a 100644 --- a/Rakefile +++ b/Rakefile @@ -119,6 +119,32 @@ end task :release => ['release:check_private_key', 'release:do'] +desc 'Build and validate the evergreen config' +task eg: %w[ eg:build eg:validate ] + +# 'eg' == 'evergreen', but evergreen is too many letters for convenience +namespace :eg do + desc 'Builds the .evergreen/config.yml file from the templates' + task :build do + ruby '.evergreen/update-evergreen-configs' + end + + desc 'Validates the .evergreen/config.yml file' + task :validate do + system 'evergreen validate --project mongo-ruby-driver .evergreen/config.yml' + end + + desc 'Updates the evergreen executable to the latest available version' + task :update do + system 'evergreen get-update --install' + end + + desc 'Runs the current branch as an evergreen patch' + task :patch do + system 'evergreen patch --uncommitted --project mongo-ruby-driver --browse --auto-description --yes' + end +end + desc "Generate all documentation" task :docs => 'docs:yard' From 8d561e6a8d7527d7b8aceed3d8a695e4a8eab8ce Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 23 Jan 2024 12:31:16 -0700 Subject: [PATCH 11/11] tweak more things to get it all to work --- .evergreen/config.yml | 4 +-- .evergreen/config/common.yml.erb | 4 +-- profile/driver_bench/base.rb | 16 ++++++--- profile/driver_bench/multi_doc/find_many.rb | 7 ++-- .../driver_bench/single_doc/find_one_by_id.rb | 6 ++-- profile/driver_bench/suite.rb | 34 ++++++++++++------- 6 files changed, 46 insertions(+), 25 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index dd2240a413..3b709daa64 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -385,10 +385,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - TEST_CMD="bundle exec rake driver_bench" .evergreen/run-tests.sh + TEST_CMD="bundle exec rake driver_bench" PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf.json" .evergreen/run-tests.sh - command: perf.send params: - file: ./results.json + file: "${PROJECT_DIRECTORY}/perf.json" "run tests": - command: shell.exec diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 848506348c..ec80952381 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -382,10 +382,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - TEST_CMD="bundle exec rake driver_bench" .evergreen/run-tests.sh + TEST_CMD="bundle exec rake driver_bench" PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf.json" .evergreen/run-tests.sh - command: perf.send params: - file: ./results.json + file: "${PROJECT_DIRECTORY}/perf.json" "run tests": - command: shell.exec diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index 322fc7ca87..e6a7bdb358 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -85,7 +85,7 @@ def run_benchmark loop do before_task - timing = without_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } } + timing = consider_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } } after_task iteration_count += 1 @@ -111,12 +111,18 @@ def new_client(uri = ENV['MONGODB_URI']) Mongo::Client.new(uri) end - # Runs a given block with GC disabled. - def without_gc - GC.disable + # Takes care of garbage collection considerations before + # running the block. + # + # Set BENCHMARK_NO_GC environment variable to suppress GC during + # the core benchmark tasks; note that this may result in obscure issues + # due to memory pressures on larger benchmarks. + def consider_gc + GC.start + GC.disable if ENV['BENCHMARK_NO_GC'] yield ensure - GC.enable + GC.enable if ENV['BENCHMARK_NO_GC'] end # By default, the file name is assumed to be relative to the diff --git a/profile/driver_bench/multi_doc/find_many.rb b/profile/driver_bench/multi_doc/find_many.rb index 8454f03e07..9a53665a3f 100644 --- a/profile/driver_bench/multi_doc/find_many.rb +++ b/profile/driver_bench/multi_doc/find_many.rb @@ -21,8 +21,11 @@ def file_name def setup super - docs = Array.new(10_000) { dataset.first } - @collection.insert_many(docs) + prototype = dataset.first + 10.times do + docs = Array.new(1000, prototype) + @collection.insert_many(docs) + end end def do_task diff --git a/profile/driver_bench/single_doc/find_one_by_id.rb b/profile/driver_bench/single_doc/find_one_by_id.rb index 1aff97a360..57603835ed 100644 --- a/profile/driver_bench/single_doc/find_one_by_id.rb +++ b/profile/driver_bench/single_doc/find_one_by_id.rb @@ -19,8 +19,10 @@ def file_name def setup super - docs = Array.new(10_000) { |i| dataset.merge(_id: i + 1) } - @collection.insert_many(docs) + 10.times do |i| + docs = Array.new(1000) { |j| dataset.merge(_id: (i * 1000) + j + 1) } + @collection.insert_many(docs) + end end def do_task diff --git a/profile/driver_bench/suite.rb b/profile/driver_bench/suite.rb index ddc1875eb4..bedf7862a6 100644 --- a/profile/driver_bench/suite.rb +++ b/profile/driver_bench/suite.rb @@ -73,8 +73,9 @@ def run_benchmark(klass) end def compile_perf_data(result) - percentile_data = PERCENTILES.each_with_object({}) do |n, hash| - hash["time-#{n}%"] = result[:percentiles][n] + percentile_data = PERCENTILES.map do |percentile| + { 'name' => "time-#{percentile}%", + 'value' => result[:percentiles][percentile] } end { @@ -82,7 +83,11 @@ def compile_perf_data(result) 'test_name' => result[:name], 'args' => {}, }, - 'metrics' => percentile_data.merge('score' => result[:score]), + 'metrics' => [ + { 'name' => 'score', + 'value' => result[:score] }, + *percentile_data + ] } end @@ -105,27 +110,32 @@ def compile_benchmarks(benches) 'test_name' => bench, 'args' => {} }, - 'metrics' => { - 'score' => score - } + 'metrics' => [ + { 'name' => 'score', + 'value' => score } + ] } end end + # rubocop:disable Metrics/AbcSize def summarize_perf_data(data) puts '===== Performance Results =====' data.each do |item| - puts format('%s : %4.4g', item['info']['test_name'], item['metrics']['score']) - next unless item['metrics']['time-10%'] + puts format('%s : %4.4g', item['info']['test_name'], item['metrics'][0]['value']) + next unless item['metrics'].length > 1 - PERCENTILES.each do |n| - puts format(' %d%% : %4.4g', n, item['metrics']["time-#{n}%"]) + item['metrics'].each do |metric| + next if metric['name'] == 'score' + + puts format(' %s : %4.4g', metric['name'], metric['value']) end end end + # rubocop:enable Metrics/AbcSize - def save_perf_data(data) - File.write('results.json', data.to_json) + def save_perf_data(data, file_name: ENV['PERFORMANCE_RESULTS_FILE'] || 'results.json') + File.write(file_name, data.to_json) end end end