Added some analysis of total execution times

RegressionTests/make_markdown_files.R

@@ -1,8 +1,8 @@
 source("markdown_functions.R")
 
 ## File in the paths to the test results for the "old" and "new" versions
-op <- "~/tmp/2016_10_31_05__6.0-71/"
-np <- "~/tmp/2016_10_30_22__6.0-72/"
+op <- "~/tmp/2017_01_10_17__6.0-73/"
+np <- "~/tmp/2017_01_10_17__6.0-74/"
 
 mods <- match_objects(op, np)
 
@@ -34,14 +34,33 @@ for(mod in mods) {
              "can be found [here](https://github.com/topepo/caret/blob/master/RegressionTests/Code/",
              mod, ".R).\nA [history of commits](https://github.com/topepo/caret/commits/master/models/files/",
              mod, ".R) for the model code is also available\n\n")) 
-      
-      try(obj_compare(file.path(op, paste0(mod, ".RData")),
-                      file.path(np, paste0(mod, ".RData"))),
-          silent = TRUE)
-      
-      sink()
+
+  try(obj_compare(file.path(op, paste0(mod, ".RData")),
+                  file.path(np, paste0(mod, ".RData"))),
+      silent = TRUE)
+
+  sink()
 }
 
+elapsed_time <- data.frame(test = mods, old = 0*NA, new = 0*NA)
+for(i in seq_along(mods)) {
+  elapsed_time[i, ] <- get_times(file.path(op, paste0(mods[i], ".RData")),
+                                 file.path(np, paste0(mods[i], ".RData")))
+
+}
 
+elapsed_time <- elapsed_time[complete.cases(elapsed_time),]
+elapsed_time$Geo_Mean <- sqrt(elapsed_time$old*elapsed_time$new)
+elapsed_time$Fold_diff <- elapsed_time$old/elapsed_time$new
 
+library(ggplot2)
+png("~/tmp/test_results/_times.png")
+ggplot(elapsed_time, aes(x = Geo_Mean, y = Fold_diff)) + 
+  geom_point() + scale_y_continuous(trans = "log2") +
+  geom_smooth() + 
+  ylab("Fold Difference (O/N)") + xlab("Geo Mean") + 
+  ggtitle("Total Test Execution Time")  +
+  theme_bw()
+dev.off()
 
+exp(t.test(log(elapsed_time$Fold_diff))$conf.int)

RegressionTests/markdown_functions.R

@@ -85,10 +85,13 @@ obj2list <- function(path){
   tests <- vector(mode = "list", length = length(test_obj))
   for(i in seq(along = tests)) tests[[i]] <- get(test_obj[[i]])
   names(tests) <- gsub("^test_", "", test_obj)
-  time <- timestamp
+  time <- format(timestamp, "%Y_%m_%d_%H_%M")
   session <- sInfo
   if(!is.character(time)) time <- NA
-  list(tests = tests, times = time, session = session)
+  list(tests = tests, 
+       times = time, 
+       session = session,
+       elapsed = difftime(timestamp_end, timestamp, units =  "s"))
 }
 
 ## Compare all the common objects from the RData files
@@ -114,10 +117,13 @@ obj_compare <- function(old_path, new_path, verbose = TRUE) {
       " * ", old_all$session$R.version$version.string, "\n", 
       " * ", print_versions(old_all$session), "\n",
       sep = "")
-  
+
   if(!is.na(old_all$times) | !is.na(new_all$times)) {
     cat(" * tested on ", as.character(times$old), 
-        " at ", gsub("_", ":", substring(old_all$time, 12, 16)), "\n",
+        " at ", gsub("_", ":", substring(old_all$time, 12, 16)), 
+        ". \n * total test time: ", 
+        round(old_all$elapsed, 1),
+        "s\n",
         sep = "")
   }
   cat("\n\n")
@@ -131,7 +137,10 @@ obj_compare <- function(old_path, new_path, verbose = TRUE) {
 
   if(!is.na(new_all$times) | !is.na(new_all$times)) {
     cat(" * tested on ", as.character(times$new), 
-        " at ", gsub("_", ":", substring(new_all$time, 12, 16)), "\n",
+        " at ", gsub("_", ":", substring(new_all$time, 12, 16)), 
+        ". \n * total test time: ", 
+        round(new_all$elapsed, 1),
+        "s\n",
         sep = "")
   }
   cat("\n\n")
@@ -477,6 +486,18 @@ train_markdown <- function(x) {
       " evaluated\n", sep = "") 
 }
 
+get_times <- function(old_stuff, new_stuff){
+
+  load(old_stuff)
+  res <- data.frame(test = gsub("\\.RData", "", basename(old_stuff)), 
+                    old = as.numeric(difftime(timestamp_end, timestamp, units =  "s")))
+  rm_list <- ls()
+  rm_list <- rm_list[!(rm_list %in% c("res", "rm_list", "new_stuff"))]
+  rm(list = rm_list)
+  load(new_stuff)
+  res$new <- as.numeric(difftime(timestamp_end, timestamp, units =  "s"))
+  res
+}