Permalink
Browse files

Use @jimhester's suggestion for refactoring the gc-aware benchmark

  • Loading branch information...
jennybc committed Apr 10, 2018
1 parent 0cc9e2d commit c8e21c2c82dee4046a3931759f2b3fb21daef1fe
Showing with 184 additions and 181 deletions.
  1. +89 −89 col-benchmark.csv
  2. BIN col-benchmark.png
  3. +17 −14 iterate-over-rows.R
  4. +78 −78 row-benchmark.csv
  5. BIN row-benchmark.png
@@ -4,123 +4,123 @@ ncol,method,time
10,transpose,0
10,transpose,0
10,transpose,0
10,pmap,9.999999992942321e-4
10,pmap,0
10,pmap,0.0010000000002037268
10,pmap,0
10,pmap,9.999999997489795e-4
10,pmap,0
10,pmap,0
10,split_lapply,0
10,split_lapply,0
10,split_lapply,0.0010000000002037268
10,split_lapply,0
10,split_lapply,9.999999992942321e-4
10,lapply_row,0.0010000000002037268
10,lapply_row,0.0010000000002037268
10,split_lapply,9.999999997489795e-4
10,lapply_row,0
10,lapply_row,0.0010000000002037268
10,lapply_row,9.999999997489795e-4
10,lapply_row,0
10,lapply_row,0
10,lapply_row,0.0010000000002037268
10,for_loop,0.0010000000002037268
10,for_loop,0
10,for_loop,0.0010000000002037268
10,for_loop,9.999999997489795e-4
10,for_loop,0
10,for_loop,0
10,for_loop,0.0010000000002037268
100,transpose,0
100,transpose,0
100,transpose,0
100,transpose,0
100,transpose,0
100,pmap,0.0010000000002037268
100,pmap,0
100,pmap,9.999999992942321e-4
100,pmap,9.999999997489795e-4
100,pmap,0.0010000000002037268
100,pmap,0
100,pmap,9.999999997489795e-4
100,split_lapply,0.0019999999999527063
100,split_lapply,0.0019999999999527063
100,split_lapply,0.0029999999997016857
100,split_lapply,0.0029999999997016857
100,split_lapply,0.0030000000006111804
100,split_lapply,0.0039999999999054126
100,split_lapply,0.0029999999997016857
100,lapply_row,0.0029999999997016857
100,lapply_row,0.0029999999997016857
100,lapply_row,0.0030000000006111804
100,lapply_row,0.0029999999997016857
100,lapply_row,0.0029999999997016857
100,for_loop,0.0030000000006111804
100,for_loop,0.0039999999999054126
100,split_lapply,0.0019999999999527063
100,split_lapply,0.003000000000156433
100,lapply_row,0.0019999999999527063
100,lapply_row,0.0019999999999527063
100,lapply_row,0.0020000000004074536
100,lapply_row,0.0019999999999527063
100,lapply_row,0.0019999999999527063
100,for_loop,0.0029999999997016857
100,for_loop,0.0020000000004074536
100,for_loop,0.0019999999999527063
100,for_loop,0.0019999999999527063
100,for_loop,0.0029999999997016857
100,for_loop,0.0029999999997016857
1e3,transpose,0.0010000000002037268
1e3,transpose,0
1e3,transpose,0
1e3,transpose,0.0010000000002037268
1e3,transpose,0
1e3,transpose,0
1e3,pmap,0.005000000000109139
1e3,pmap,0.005000000000109139
1e3,pmap,0.004999999999199645
1e3,pmap,0.006000000000312866
1e3,pmap,0.005000000000109139
1e3,split_lapply,0.02999999999974534
1e3,split_lapply,0.02999999999974534
1e3,split_lapply,0.030000000000654836
1e3,split_lapply,0.027000000000043656
1e3,split_lapply,0.027999999999337888
1e3,lapply_row,0.032000000000152795
1e3,lapply_row,0.03299999999944703
1e3,lapply_row,0.032000000000152795
1e3,lapply_row,0.02999999999974534
1e3,pmap,0.0020000000004074536
1e3,pmap,0.0019999999999527063
1e3,pmap,0.0019999999999527063
1e3,pmap,0.0019999999999527063
1e3,pmap,0.0019999999999527063
1e3,split_lapply,0.022000000000389264
1e3,split_lapply,0.02599999999983993
1e3,split_lapply,0.023999999999887223
1e3,split_lapply,0.028999999999996362
1e3,split_lapply,0.02500000000009095
1e3,lapply_row,0.023000000000138243
1e3,lapply_row,0.021999999999934516
1e3,lapply_row,0.021000000000185537
1e3,lapply_row,0.027000000000043656
1e3,lapply_row,0.023000000000138243
1e3,for_loop,0.02099999999973079
1e3,for_loop,0.021000000000185537
1e3,for_loop,0.02099999999973079
1e3,for_loop,0.021000000000185537
1e3,for_loop,0.027000000000043656
1e3,for_loop,0.028000000000247383
1e3,for_loop,0.032000000000152795
1e3,for_loop,0.03099999999994907
1e3,for_loop,0.02999999999974534
1e4,transpose,0.001999999999497959
1e4,transpose,0.0010000000002037268
1e4,transpose,0.0010000000002037268
1e4,transpose,9.999999992942321e-4
1e4,transpose,9.999999997489795e-4
1e4,transpose,0.0010000000002037268
1e4,pmap,0.04500000000007276
1e4,pmap,0.0500000000001819
1e4,pmap,0.04399999999986903
1e4,pmap,0.03799999999955617
1e4,pmap,0.04800000000068394
1e4,split_lapply,0.2889999999997599
1e4,split_lapply,0.30500000000029104
1e4,split_lapply,0.30599999999958527
1e4,split_lapply,0.3100000000004002
1e4,split_lapply,0.31300000000010186
1e4,lapply_row,0.28800000000046566
1e4,lapply_row,0.3079999999999927
1e4,lapply_row,0.3040000000000873
1e4,lapply_row,0.31599999999980355
1e4,lapply_row,0.31199999999989814
1e4,for_loop,0.29399999999986903
1e4,for_loop,0.29900000000088767
1e4,for_loop,0.29899999999997817
1e4,for_loop,0.2960000000002765
1e4,for_loop,0.2960000000002765
1e5,transpose,0.012999999999919964
1e5,transpose,0.015000000000327418
1e5,transpose,0.018000000000029104
1e5,transpose,0.01599999999962165
1e5,transpose,0.023000000000138243
1e5,pmap,0.49400000000059663
1e5,pmap,1.0659999999998035
1e5,pmap,1.2029999999995198
1e5,pmap,0.555000000000291
1e5,pmap,0.9440000000004147
1e5,split_lapply,3.4459999999999127
1e5,split_lapply,3.4329999999999927
1e5,split_lapply,3.824000000000524
1e5,split_lapply,3.6179999999994834
1e5,split_lapply,3.3140000000003056
1e5,lapply_row,3.2429999999994834
1e5,lapply_row,3.662000000000262
1e5,lapply_row,3.7960000000002765
1e5,lapply_row,3.8400000000001455
1e5,lapply_row,3.7359999999998763
1e5,for_loop,3.2730000000001382
1e5,for_loop,3.5230000000001382
1e5,for_loop,4.154999999999745
1e5,for_loop,3.475000000000364
1e5,for_loop,3.751999999999498
1e4,transpose,9.999999997489795e-4
1e4,transpose,0.0020000000004074536
1e4,pmap,0.02500000000009095
1e4,pmap,0.024999999999636202
1e4,pmap,0.027000000000043656
1e4,pmap,0.026000000000294676
1e4,pmap,0.03099999999994907
1e4,split_lapply,0.24499999999989086
1e4,split_lapply,0.23900000000003274
1e4,split_lapply,0.24899999999979627
1e4,split_lapply,0.2680000000000291
1e4,split_lapply,0.24499999999989086
1e4,lapply_row,0.22000000000025466
1e4,lapply_row,0.2369999999996253
1e4,lapply_row,0.23400000000037835
1e4,lapply_row,0.2339999999999236
1e4,lapply_row,0.22600000000011278
1e4,for_loop,0.24899999999979627
1e4,for_loop,0.23800000000028376
1e4,for_loop,0.2519999999999527
1e4,for_loop,0.26499999999987267
1e4,for_loop,0.25700000000006185
1e5,transpose,0.01499999999987267
1e5,transpose,0.016999999999825377
1e5,transpose,0.016000000000076398
1e5,transpose,0.016999999999825377
1e5,transpose,0.027000000000043656
1e5,pmap,0.5749999999998181
1e5,pmap,0.6639999999997599
1e5,pmap,0.6190000000001419
1e5,pmap,0.7470000000002983
1e5,pmap,0.6419999999998254
1e5,split_lapply,3.2729999999996835
1e5,split_lapply,3.624000000000251
1e5,split_lapply,3.9329999999999927
1e5,split_lapply,3.380000000000109
1e5,split_lapply,3.4890000000000327
1e5,lapply_row,3.199000000000069
1e5,lapply_row,3.630000000000109
1e5,lapply_row,3.9980000000000473
1e5,lapply_row,3.5589999999997417
1e5,lapply_row,3.6010000000001128
1e5,for_loop,3.212999999999738
1e5,for_loop,3.66800000000012
1e5,for_loop,4.114000000000033
1e5,for_loop,3.882000000000062
1e5,for_loop,3.5149999999998727
BIN +3.66 KB (100%) col-benchmark.png
Binary file not shown.
@@ -32,16 +32,21 @@ f_transpose <- function(df) {
transpose(df)
}

## explicit gc, then execute `expr` `n` times w/o explicit gc, return timings
benchmark <- function(n = 1, expr, envir = parent.frame()) {
expr <- substitute(expr)
gc()
map(seq_len(n), ~ system.time(eval(expr, envir), gcFirst = FALSE))
}

run_row_benchmark <- function(nrow, times = 5) {
df <- data.frame(x = rnorm(nrow), y = runif(nrow), z = runif(nrow))
rerun_gc_once <- function(...) {gc(); rerun(...)}
st_no_gc <- function(expr) system.time(expr, gcFirst = FALSE)
res <- list(
transpose = rerun_gc_once(times, st_no_gc(f_transpose(df))),
pmap = rerun_gc_once(times, st_no_gc(f_pmap(df))),
split_lapply = rerun_gc_once(times, st_no_gc(f_split_lapply(df))),
lapply_row = rerun_gc_once(times, st_no_gc(f_lapply_row(df))),
for_loop = rerun_gc_once(times, st_no_gc(f_for_loop(df)))
transpose = benchmark(times, f_transpose(df)),
pmap = benchmark(times, f_pmap(df)),
split_lapply = benchmark(times, f_split_lapply(df)),
lapply_row = benchmark(times, f_lapply_row(df)),
for_loop = benchmark(times, f_for_loop(df))
)
res <- map(res, ~ map_dbl(.x, "elapsed"))
tibble(
@@ -59,14 +64,12 @@ run_col_benchmark <- function(ncol, times = 5) {
z = seq_len(nrow)
)
df <- template[rep_len(seq_len(ncol(template)), length.out = ncol)]
rerun_gc_once <- function(...) {gc(); rerun(...)}
st_no_gc <- function(expr) system.time(expr, gcFirst = FALSE)
res <- list(
transpose = rerun_gc_once(times, st_no_gc(f_transpose(df))),
pmap = rerun_gc_once(times, st_no_gc(f_pmap(df))),
split_lapply = rerun_gc_once(times, st_no_gc(f_split_lapply(df))),
lapply_row = rerun_gc_once(times, st_no_gc(f_lapply_row(df))),
for_loop = rerun_gc_once(times, st_no_gc(f_for_loop(df)))
transpose = benchmark(times, st_no_gc(f_transpose(df))),
pmap = benchmark(times, st_no_gc(f_pmap(df))),
split_lapply = benchmark(times, st_no_gc(f_split_lapply(df))),
lapply_row = benchmark(times, st_no_gc(f_lapply_row(df))),
for_loop = benchmark(times, st_no_gc(f_for_loop(df)))
)
res <- map(res, ~ map_dbl(.x, "elapsed"))
tibble(
Oops, something went wrong.

0 comments on commit c8e21c2

Please sign in to comment.