Add support for computed measures by commands

DOCS.md

@@ -148,6 +148,19 @@ context:
 # ... stripped ...
 ```
 
+A command can be used to compute a custom measure. The regex configuration section is available to
+match and transform the output if needed.
+
+```yaml
+measure:
+  - computed_by_script:
+      command: ./compute_with_context
+      regex:
+        pattern: (\d+)\s(\d+)
+        group: 2
+        transform: tr '.' ''
+```
+
 ### Complete example
 
 ```yaml
@@ -161,6 +174,12 @@ measure:
   - max_rss
   - requests_per_second:
       regex: Requests per second:\s+(?<measure>\d+.\d+)\s+\[#/sec\] \(mean\)
+  - computed_by_script:
+      command: ./compute_with_context
+      regex:
+        pattern: (\d+)\s(\d+)
+        group: 2
+        transform: tr '.' ''
 run: ./run
 loader: ./loader
 repeat: 10

spec/manifest_spec.cr

@@ -94,6 +94,27 @@ describe Benchy::Manifest do
   run: ./run
   YAML
 
+  it_parses "custom meassure with command", <<-YAML
+  name: sample
+  measure:
+    - time
+    - foo:
+        command: ./foo
+  run: ./run
+  YAML
+
+  it_parses "custom meassure with command and regex", <<-YAML
+  name: sample
+  measure:
+    - time
+    - foo:
+        command: ./foo
+        regex:
+          pattern: /hs(<num>\d)f/
+          group: num
+  run: ./run
+  YAML
+
   it_parses "matrix env", <<-YAML
   name: sample
   measure:

src/benchy.cr

@@ -44,7 +44,7 @@ module Benchy
       end
     end
 
-    alias ExtractSampleProc = Proc(String, Sample)
+    alias ExtractSampleProc = Proc(String, Project, Configuration, Sample)
 
     getter name : String
     getter base_dir : Path
@@ -60,7 +60,7 @@ module Benchy
       @context = Hash(String, String).new
       if manifest_context = manifest.context
         manifest_context.each do |key, cmd|
-          @context[key] = get_output(cmd)
+          @context[key] = get_output(cmd, nil)
         end
       end
 
@@ -142,7 +142,7 @@ module Benchy
       run_status = main_status
       run_status = loader_status if loader_status && !loader_status.success?
 
-      measures = extract_measures(main_output + main_error + loader_output + loader_error) if run_status.success?
+      measures = extract_measures(main_output + main_error + loader_output + loader_error, configuration) if run_status.success?
 
       if run_logger
         run_logger.log(name: name, config_index: config_index, run_index: run_index,
@@ -180,11 +180,11 @@ module Benchy
       end
     {% end %}
 
-    private def get_output(cmd : String) : String
-      exec(cmd, nil).chomp
+    def get_output(cmd : String, configuration : Configuration?) : String
+      exec(cmd, configuration).chomp
     end
 
-    private def exec(cmd : String, configuration : Configuration?) : String
+    def exec(cmd : String, configuration : Configuration?) : String
       debug_cmd cmd, configuration.try(&.[:env])
       process = Process.new("/usr/bin/env bash -c '#{cmd}'",
         env: configuration.try(&.[:env]),
@@ -222,9 +222,9 @@ module Benchy
       configurations.first[:env].keys
     end
 
-    def extract_measures(main_output)
+    def extract_measures(main_output, configuration)
       @measure_samplers.transform_values do |proc|
-        proc.call(main_output)
+        proc.call(main_output, self, configuration)
       end
     end
 
@@ -277,14 +277,30 @@ module Benchy
                      pattern = case c = config.regex
                                when String
                                  c
+                               when Nil
+                                 nil
                                else
                                  raise "Not Supported #{key} #{config}"
                                end
 
-                     ->(main_output : String) {
-                       md = main_output.match(Regex.new(pattern, :multiline))
-                       raise "Missing #{key} measure" unless md
-                       md["measure"]?.try(&.to_f64) || md[0].to_f64
+                     command = config.command
+
+                     ->(main_output : String, project : Project, configuration : Configuration) {
+                       output = if (_command = command)
+                                  project.get_output(_command, configuration)
+                                else
+                                  main_output
+                                end
+
+                       if _pattern = pattern
+                         md = output.match(Regex.new(_pattern, :multiline))
+                         raise "Missing #{key} measure" unless md
+                         md["measure"]?.try(&.to_f64) || md[0].to_f64
+                       else
+                         output.to_f64
+                       end
+
+                       # TODO apply transform to output before f64 conversion
                      }
                    end
 

src/instrument.cr

@@ -1,7 +1,7 @@
 module Benchy
   {% if flag?(:darwin) %}
-    BIN_TIME = "/usr/bin/time -l"
-    MAX_RSS_PROC = ->(output : String) {
+    BIN_TIME     = "/usr/bin/time -l"
+    MAX_RSS_PROC = ->(output : String, project : Project, configuration : Project::Configuration) {
       md = output.match /(\d+)\s+maximum resident set size/m
       raise "Missing max_rss measure" unless md
       # in osx it is already in bytes
@@ -10,31 +10,31 @@ module Benchy
     }
 
     TIME_REGEX = /(?<real>(\d+).(\d+))\sreal\s+(?<user>(\d+).(\d+))\suser\s+(?<sys>(\d+).(\d+))\ssys/m
-    TIME_PROC = ->(output : String) {
+    TIME_PROC  = ->(output : String, project : Project, configuration : Project::Configuration) {
       md = output.match TIME_REGEX
       raise "Missing time measure" unless md
       md["real"].to_f64
     }
-    CPU_TIME_PROC = ->(output : String) {
+    CPU_TIME_PROC = ->(output : String, project : Project, configuration : Project::Configuration) {
       # TODO linux support
       md = output.match TIME_REGEX
       raise "Missing time measure" unless md
       md["user"].to_f64 + md["sys"].to_f64
     }
   {% else %}
-    BIN_TIME = "/usr/bin/time -v"
-    MAX_RSS_PROC = ->(output : String) {
+    BIN_TIME     = "/usr/bin/time -v"
+    MAX_RSS_PROC = ->(output : String, project : Project, configuration : Project::Configuration) {
       md = output.match /Maximum resident set size \(kbytes\):\s+(\d+)/m
       raise "Missing max_rss measure" unless md
       # Kb to bytes
       (md[1].to_i64 * 1024).to_f64
     }
-    TIME_PROC = ->(output : String) {
+    TIME_PROC = ->(output : String, project : Project, configuration : Project::Configuration) {
       md = output.match /Elapsed \(wall clock\) time \(h:mm:ss or m:ss\):\s+((?<h>\d+):)?(?<m>\d+):(?<s>\d\d(.\d*)?)/m
       raise "Missing time measure" unless md
       (md["h"]? || 0).to_f64 * 3600.0 + md["m"].to_f64 * 60.0 + md["s"].to_f64
     }
-    CPU_TIME_PROC = ->(output : String) {
+    CPU_TIME_PROC = ->(output : String, project : Project, configuration : Project::Configuration) {
       md_user = output.match /User time \(seconds\):\s+(\d+.\d+)/m
       md_sys = output.match /System time \(seconds\):\s+(\d+.\d+)/m
       raise "Missing time measure" unless md_user && md_sys

src/manifest.cr

@@ -17,7 +17,8 @@ module Benchy
 
     class CustomMeasure
       YAML.mapping(
-        regex: String | CustomRegexMeasure?
+        regex: String | CustomRegexMeasure?,
+        command: String?
       )
     end