Skip to content

Return a data.frame from loo_compare() for subsampled loo objects #393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
florence-bockting wants to merge 2 commits into
base: fix-subsampling
Choose a base branch
from
Draft

Return a data.frame from loo_compare() for subsampled loo objects #393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2381fec
fix: return a data.frame from loo_compare() for subsampled loo objects
Sep 3, 2026
8a8f767
chore: update PR number in NEWS.md
Sep 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# loo (development version)

* Change the output of `loo_compare()` for subsampled loo objects from a matrix to a data.frame by @florence-bockting in #393
* Fix `loo_compare()` when used with subsampling: compute model comparison by comparison-model minus reference-model by @florence-bockting in #391
* Update user messages in `print()` by @ishaan-arora-1, @florence-bockting in
#328.
Expand Down
20 changes: 13 additions & 7 deletions R/loo_compare.psis_loo_ss_list.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@ loo_compare.psis_loo_ss_list <- function(x, ...) {
for(i in 2:length(ord)){
elpd_diff_mat[i,] <- loo_compare_ss(ref_loo = x[ord[1]], compare_loo = x[ord[i]])
}
comp <- cbind(elpd_diff_mat, comp)
rownames(comp) <- rnms
comp <- cbind(
data.frame(model = rnms, stringsAsFactors = FALSE),
as.data.frame(elpd_diff_mat),
as.data.frame(comp)
)
rownames(comp) <- NULL

class(comp) <- c("compare.loo_ss", "compare.loo", class(comp))
return(comp)
Expand Down Expand Up @@ -174,11 +178,13 @@ loo_compare_checks.psis_loo_ss_list <- function(loos) {
#' @rdname loo_compare
#' @export
print.compare.loo_ss <- function(x, ..., digits = 1) {
xcopy <- x
if (NCOL(xcopy) >= 2) {
xcopy <- xcopy[, c("elpd_diff", "se_diff", "subsampling_se_diff")]
}
print(.fr(xcopy, digits), quote = FALSE)
cols <- c("model", "elpd_diff", "se_diff", "subsampling_se_diff")
cols <- intersect(cols, colnames(x))

x_sub <- x[, cols, drop = FALSE]
x_sub[setdiff(cols, "model")] <- .fr(x_sub[setdiff(cols, "model")], digits)
print(as.data.frame(x_sub), quote = FALSE, row.names = FALSE)

invisible(x)
}

Expand Down
24 changes: 12 additions & 12 deletions tests/testthat/_snaps/loo_subsampling_cases.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,34 +104,34 @@
Code
print(comp)
Output
elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.5 22.5 0.4
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.5 22.5 0.4

---

Code
print(comp)
Output
elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.1 4.4 0.1
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.1 4.4 0.1

---

Code
print(comp2)
Output
elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.3 4.4 0.1
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.3 4.4 0.1

---

Code
print(comp3)
Output
elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.5 4.4 0.3
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.5 4.4 0.3

32 changes: 18 additions & 14 deletions tests/testthat/test_loo_subsampling.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1047,23 +1047,26 @@ test_that("loo_compare_subsample", {
)
)

expect_equal(lcss[, 1], lcsso[, 1], tolerance = 1)
expect_equal(lcss2[, 1], lcsso[, 1], tolerance = 1)
expect_equal(lcssohh[, 1], lcsso[, 1], tolerance = 1)
expect_equal(lcssf1[, 1], lcsso[, 1], tolerance = 1)
expect_equal(lcssf2[, 1], lcsso[, 1], tolerance = 1)
expect_equal(lcss$elpd_diff, lcsso$elpd_diff, tolerance = 1)
expect_equal(lcss2$elpd_diff, lcsso$elpd_diff, tolerance = 1)
expect_equal(lcssohh$elpd_diff, lcsso$elpd_diff, tolerance = 1)
expect_equal(lcssf1$elpd_diff, lcsso$elpd_diff, tolerance = 1)
expect_equal(lcssf2$elpd_diff, lcsso$elpd_diff, tolerance = 1)

expect_gt(lcss[, 2][2], lcsso[, 2][2])
expect_gt(lcss[, 2][3], lcsso[, 2][3])
expect_gt(lcss2[, 2][2], lcsso[, 2][2])
expect_equal(lcss2[, 2][3], lcsso[, 2][3])
expect_gt(lcssohh[, 2][2], lcsso[, 2][2])
expect_equal(lcssohh[, 2][3], lcsso[, 2][3])
expect_gt(lcss$se_diff[2], lcsso$se_diff[2])
expect_gt(lcss$se_diff[3], lcsso$se_diff[3])
expect_gt(lcss2$se_diff[2], lcsso$se_diff[2])
expect_equal(lcss2$se_diff[3], lcsso$se_diff[3])
expect_gt(lcssohh$se_diff[2], lcsso$se_diff[2])
expect_equal(lcssohh$se_diff[3], lcsso$se_diff[3])

expect_silent(
lcss2m <- loo:::loo_compare.psis_loo_ss_list(x = list(lss2o1, lss3o1))
)
expect_equal(unname(lcss2m[,]), unname(lcsso[1:2, ]))
expect_equal(
lcss2m[, setdiff(colnames(lcss2m), "model")],
lcsso[1:2, setdiff(colnames(lcsso), "model")]
)

expect_snapshot(lcssapi <- loo_compare(lss1, lss2, lss3))
expect_equal(lcssapi, lcss)
Expand All @@ -1073,8 +1076,9 @@ test_that("loo_compare_subsample", {
expect_equal(lcss2mapi, lcss2m)
# check that comparison is comp - ref model (i.e., elpd_diff is neg.)
for (m in list(lcss, lcss2, lcssohh)) {
expect_lt(m[2, "elpd_diff"], 0)
expect_lt(m[3, "elpd_diff"], 0)
expect_lt(m$elpd_diff[2], 0)
expect_lt(m$elpd_diff[3], 0)
expect_true("data.frame" %in% class(m))
}
})

Expand Down
6 changes: 3 additions & 3 deletions vignettes/loo2-large-data.Rmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ print(comp)
```
Warning: Different subsamples in 'model2' and 'model1'. Naive diff SE is used.

elpd_diff se_diff subsampling_se_diff
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.5 22.5 0.4
```
Expand Down Expand Up @@ -542,7 +542,7 @@ print(comp)
```

```
elpd_diff se_diff subsampling_se_diff
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.1 4.4 0.1
```
Expand Down Expand Up @@ -580,7 +580,7 @@ the loo calculations for both `model1` and `model2` are included in the
computations for the comparison.

```
elpd_diff se_diff subsampling_se_diff
model elpd_diff se_diff subsampling_se_diff
model2 0.0 0.0 0.0
model1 -16.3 4.4 0.3
```
Expand Down
Loading