Add and test the addition of thresholds to compare endpoint

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: conbenchcoms
 Title: An API wrapper for conbench communications
-Version: 0.0.3
+Version: 0.0.4
 Authors@R: c(
     person("Jonathan", "Keane", , "jon@voltrondata.com", role = c("aut", "ctb"),
            comment = c(ORCID = "0000-0001-7087-9776")),

NEWS.md

@@ -1,3 +1,6 @@
+# conbenchcoms 0.0.4
+* add threshold endpoints to conbenchcoms
+
 # conbenchcoms 0.0.3
 * add limit and days parameters to `benchmark_results`
 

R/compare.R

@@ -3,21 +3,38 @@
 #' @param type the type of comparison to make (one of: runs, benchmarks, batches, commits)
 #' @param baseline the baseline sha of the entity to compare
 #' @param contender the contender sha of the entity to compare
+#' @param zscore_threshold the zscore threshold to mark regressions and improvements. 
+#'   Default is defined at the Conbench api level.
+#' @param pairwise_percent_threshold the pairwise_percent_threshold to mark regressions and improvements.
+#'   Default is defined at the Conbench api level.
 #' @inheritDotParams httr2::resp_body_json
 #'
 #' @return the JSON response
 #' @export
 compare <- function(type = c("runs", "benchmarks", "batches", "commits"),
                     baseline,
                     contender,
+                    zscore_threshold = NULL,
+                    pairwise_percent_threshold = NULL,
                     ...) {
   type <- match.arg(type)
+  stopifnot("zscore_threshold must be numeric" = is.numeric(zscore_threshold) || is.null(zscore_threshold))
+  stopifnot("pairwise_percent_threshold must be numeric" = is.numeric(pairwise_percent_threshold) || is.null(pairwise_percent_threshold))
 
   req <- req_url_path_append(
     conbench_request(),
     "compare",
     type,
-    paste0(baseline, "...", contender))
+    paste0(baseline, "...", contender)
+  )
+
+  if (!is.null(zscore_threshold)) {
+    req <- req_url_query(req, threshold_z = zscore_threshold)
+  }
+
+  if (!is.null(pairwise_percent_threshold)) {
+    req <- req_url_query(req, threshold = pairwise_percent_threshold)
+  }
 
   resp <- conbench_perform(req)
 

tests/testthat/resp-class/localhost/api/compare/runs/10aded...0af-81f373.json

@@ -0,0 +1,18 @@
+[
+    {
+        "analysis": {
+            "lookback_z_score": {
+                "improvement_indicated": false,
+                "regression_indicated": false,
+                "z_score": -0.5,
+                "z_threshold": 11.1
+            },
+            "pairwise": {
+                "improvement_indicated": false,
+                "percent_change": -26,
+                "percent_threshold": 20.2,
+                "regression_indicated": true
+            }
+        }
+    }
+]

tests/testthat/resp-class/localhost/api/compare/runs/ee1...f00d-dd5b56.json

@@ -0,0 +1,18 @@
+[
+    {
+        "analysis": {
+            "lookback_z_score": {
+                "improvement_indicated": false,
+                "regression_indicated": false,
+                "z_score": -0.5,
+                "z_threshold": 10
+            },
+            "pairwise": {
+                "improvement_indicated": false,
+                "percent_change": -26,
+                "percent_threshold": 20,
+                "regression_indicated": true
+            }
+        }
+    }
+]

tests/testthat/test-compare.R

@@ -11,3 +11,60 @@ with_mock_dir(test_path("logged-in"), {
     )
   })
 })
+
+
+with_mock_dir(test_path("resp-class"), {
+  test_that("zscore and pairwise thresholds return correct values", {
+    z_thres <- 10
+    percent_threshold <- 20
+
+    resp <- compare(
+      type = "runs",
+      "ee1",
+      "f00d",
+      zscore_threshold = z_thres,
+      pairwise_percent_threshold = z_thres,
+      simplifyVector = TRUE
+    )
+    returned_z_thres <- unique(resp[["analysis"]][["lookback_z_score"]][["z_threshold"]])
+    expect_equal(returned_z_thres[!is.na(returned_z_thres)], z_thres)
+    returned_percent_thres <- unique(resp[["analysis"]][["pairwise"]][["percent_threshold"]])
+    expect_equal(returned_percent_thres[!is.na(returned_percent_thres)], percent_threshold)
+  })
+
+  test_that("zscore and pairwise thresholds can accept decimals", {
+    z_thres <- 11.1
+    percent_threshold <- 20.2
+
+    resp <- compare(
+      type = "runs",
+      "10aded",
+      "0af",
+      zscore_threshold = z_thres,
+      pairwise_percent_threshold = z_thres,
+      simplifyVector = TRUE
+    )
+    returned_z_thres <- unique(resp[["analysis"]][["lookback_z_score"]][["z_threshold"]])
+    expect_equal(returned_z_thres[!is.na(returned_z_thres)], z_thres)
+    returned_percent_thres <- unique(resp[["analysis"]][["pairwise"]][["percent_threshold"]])
+    expect_equal(returned_percent_thres[!is.na(returned_percent_thres)], percent_threshold)
+  })
+
+  test_that("zscore and pairwise thresholds fail with non-numeric values", {
+    expect_error(compare(
+      type = "runs",
+      "10aded",
+      "0af",
+      zscore_threshold = "foo",
+      simplifyVector = TRUE
+    ), "zscore_threshold must be numeric")
+
+    expect_error(compare(
+      type = "runs",
+      "10aded",
+      "0af",
+      pairwise_percent_threshold = "bar",
+      simplifyVector = TRUE
+    ), "pairwise_percent_threshold must be numeric")
+  })
+})