From 9f1ed8263074500581b4129d05c92b0b72904573 Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 14:22:50 +0000 Subject: [PATCH 1/7] print output --- benchmarks/benchmarks.jl | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 3af6573cf..e5d640570 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -73,16 +73,16 @@ function run(; to_json=false) ("Smorgasbord", smorgasbord_instance, :simple_dict, :forwarddiff, true), ("Smorgasbord", smorgasbord_instance, :typed_vector, :forwarddiff, true), ("Smorgasbord", smorgasbord_instance, :untyped_vector, :forwarddiff, true), - ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true), - ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true), - ("Smorgasbord", smorgasbord_instance, :typed, :enzyme, true), - ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true), - ("Multivariate 1k", multivariate1k, :typed, :mooncake, true), - ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true), - ("Multivariate 10k", multivariate10k, :typed, :mooncake, true), - ("Dynamic", Models.dynamic(), :typed, :mooncake, true), - ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true), - ("LDA", lda_instance, :typed, :reversediff, true), + # ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true), + # ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true), + # ("Smorgasbord", smorgasbord_instance, :typed, :enzyme, true), + # ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true), + # ("Multivariate 1k", multivariate1k, :typed, :mooncake, true), + # ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true), + # ("Multivariate 10k", multivariate10k, :typed, :mooncake, true), + # ("Dynamic", Models.dynamic(), :typed, :mooncake, true), + # ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true), + # ("LDA", lda_instance, :typed, :reversediff, true), ] # Time running a model-like function that does not use DynamicPPL, as a reference point. @@ -98,12 +98,15 @@ function run(; to_json=false) }[] for (model_name, model, varinfo_choice, adbackend, islinked) in chosen_combinations - @info "Running benchmark for $model_name" + @info "Running benchmark for $model_name, $varinfo_choice, $adbackend, $islinked" relative_eval_time, relative_ad_eval_time = try results = benchmark(model, varinfo_choice, adbackend, islinked) + @info " t(eval) = $(results.primal_time)" + @info " t(grad) = $(results.grad_time)" (results.primal_time / reference_time), (results.grad_time / results.primal_time) catch e + @info "benchmark errored" missing, missing end push!( From a89ac64066a936ea965a87501667a2e8e92075f1 Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 14:59:34 +0000 Subject: [PATCH 2/7] fix --- benchmarks/benchmarks.jl | 2 +- src/test_utils/ad.jl | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index e5d640570..1c85965ff 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -106,7 +106,7 @@ function run(; to_json=false) (results.primal_time / reference_time), (results.grad_time / results.primal_time) catch e - @info "benchmark errored" + @info "benchmark errored: $e" missing, missing end push!( diff --git a/src/test_utils/ad.jl b/src/test_utils/ad.jl index d7a34e6e0..8ee850877 100644 --- a/src/test_utils/ad.jl +++ b/src/test_utils/ad.jl @@ -5,7 +5,13 @@ using Chairmarks: @be import DifferentiationInterface as DI using DocStringExtensions using DynamicPPL: - Model, LogDensityFunction, VarInfo, AbstractVarInfo, getlogjoint_internal, link + DynamicPPL, + Model, + LogDensityFunction, + VarInfo, + AbstractVarInfo, + getlogjoint_internal, + link using LogDensityProblems: logdensity, logdensity_and_gradient using Random: AbstractRNG, default_rng using Statistics: median @@ -298,7 +304,9 @@ function run_ad( # Benchmark grad_time, primal_time = if benchmark + logdensity(ldf, params) # Warm-up primal_benchmark = @be logdensity($ldf, $params) + logdensity_and_gradient(ldf, params) # Warm-up grad_benchmark = @be logdensity_and_gradient($ldf, $params) median_primal = median(primal_benchmark).time median_grad = median(grad_benchmark).time From a90b1a74bd5bc59e344165b70e2f1750cf854b60 Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 15:11:31 +0000 Subject: [PATCH 3/7] reenable --- benchmarks/benchmarks.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 1c85965ff..624d7fc68 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -73,16 +73,16 @@ function run(; to_json=false) ("Smorgasbord", smorgasbord_instance, :simple_dict, :forwarddiff, true), ("Smorgasbord", smorgasbord_instance, :typed_vector, :forwarddiff, true), ("Smorgasbord", smorgasbord_instance, :untyped_vector, :forwarddiff, true), - # ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true), - # ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true), - # ("Smorgasbord", smorgasbord_instance, :typed, :enzyme, true), - # ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true), - # ("Multivariate 1k", multivariate1k, :typed, :mooncake, true), - # ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true), - # ("Multivariate 10k", multivariate10k, :typed, :mooncake, true), - # ("Dynamic", Models.dynamic(), :typed, :mooncake, true), - # ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true), - # ("LDA", lda_instance, :typed, :reversediff, true), + ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true), + ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true), + ("Smorgasbord", smorgasbord_instance, :typed, :enzyme, true), + ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true), + ("Multivariate 1k", multivariate1k, :typed, :mooncake, true), + ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true), + ("Multivariate 10k", multivariate10k, :typed, :mooncake, true), + ("Dynamic", Models.dynamic(), :typed, :mooncake, true), + ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true), + ("LDA", lda_instance, :typed, :reversediff, true), ] # Time running a model-like function that does not use DynamicPPL, as a reference point. From 14b3dc381d49ab53128ffb088b57bc5dafe3db6d Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 17:06:25 +0000 Subject: [PATCH 4/7] add more lines to guide the eye --- benchmarks/benchmarks.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 624d7fc68..7b64b0ba6 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -215,7 +215,10 @@ function combine(head_filename::String, base_filename::String) backend=:text, fit_table_in_display_horizontally=false, fit_table_in_display_vertically=false, - table_format=TextTableFormat(; horizontal_line_at_merged_column_labels=true), + table_format=TextTableFormat(; + horizontal_line_at_merged_column_labels=true, + horizontal_lines_at_data_rows=collect(3:3:length(results_table)), + ), ) println("```") end From 6ec2bbd4a945270144c9c22fdab5b2b9e6936153 Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 17:06:57 +0000 Subject: [PATCH 5/7] reorder table --- benchmarks/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 7b64b0ba6..30cc54933 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -158,7 +158,7 @@ function combine(head_filename::String, base_filename::String) all_testcases = union(Set(keys(head_testcases)), Set(keys(base_testcases))) @info "$(length(all_testcases)) unique test cases found" sorted_testcases = sort( - collect(all_testcases); by=(c -> (c.model_name, c.ad_backend, c.varinfo, c.linked)) + collect(all_testcases); by=(c -> (c.model_name, c.linked, c.varinfo, c.ad_backend)) ) results_table = Tuple{ String,Int,String,String,Bool,String,String,String,String,String,String From 7008d6166e8e25be89a6380231d5b2715c13762a Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 17:10:59 +0000 Subject: [PATCH 6/7] print tgrad / trel as well --- benchmarks/benchmarks.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 30cc54933..20c9243bd 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -163,13 +163,15 @@ function combine(head_filename::String, base_filename::String) results_table = Tuple{ String,Int,String,String,Bool,String,String,String,String,String,String }[] + sublabels = ["base", "this PR", "speedup"] results_colnames = [ [ EmptyCells(5), MultiColumn(3, "t(eval) / t(ref)"), MultiColumn(3, "t(grad) / t(eval)"), + MultiColumn(3, "t(grad) / t(ref)"), ], - [colnames[1:5]..., "base", "this PR", "speedup", "base", "this PR", "speedup"], + [colnames[1:5]..., sublabels..., sublabels..., sublabels...], ] sprint_float(x::Float64) = @sprintf("%.2f", x) sprint_float(m::Missing) = "err" @@ -186,6 +188,10 @@ function combine(head_filename::String, base_filename::String) # Finally that lets us do this division safely speedup_eval = base_eval / head_eval speedup_grad = base_grad / head_grad + # As well as this multiplication, which is t(grad) / t(ref) + head_grad_vs_ref = head_grad * head_eval + base_grad_vs_ref = base_grad * base_eval + speedup_grad_vs_ref = base_grad_vs_ref / head_grad_vs_ref push!( results_table, ( @@ -200,6 +206,9 @@ function combine(head_filename::String, base_filename::String) sprint_float(base_grad), sprint_float(head_grad), sprint_float(speedup_grad), + sprint_float(base_grad_vs_ref), + sprint_float(head_grad_vs_ref), + sprint_float(speedup_grad_vs_ref), ), ) end From 4e57712b1813861d11e157b2fc093a353c5a0414 Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 18 Nov 2025 17:25:42 +0000 Subject: [PATCH 7/7] forgot this type --- benchmarks/benchmarks.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl index 20c9243bd..e8ffa7e0b 100644 --- a/benchmarks/benchmarks.jl +++ b/benchmarks/benchmarks.jl @@ -161,7 +161,20 @@ function combine(head_filename::String, base_filename::String) collect(all_testcases); by=(c -> (c.model_name, c.linked, c.varinfo, c.ad_backend)) ) results_table = Tuple{ - String,Int,String,String,Bool,String,String,String,String,String,String + String, + Int, + String, + String, + Bool, + String, + String, + String, + String, + String, + String, + String, + String, + String, }[] sublabels = ["base", "this PR", "speedup"] results_colnames = [