diff --git a/R/tt_dotabulation.R b/R/tt_dotabulation.R
index f642214b5..4625f45a1 100644
--- a/R/tt_dotabulation.R
+++ b/R/tt_dotabulation.R
@@ -1117,7 +1117,11 @@ setMethod("set_def_child_ord", "VarLevWBaselineSplit",
         vals = sort(unlist(value_names(pinfo$values)))
     } else {
         vec = df[[spl_payload(lyt)]]
-        vals = unique(vec)
+        vals <- if(is.factor(vec))
+                levels(vec)
+            else
+                unique(vec)
+
         if(is.factor(vals))
             vals = levels(relevel(droplevels(vals), bline)) # this sorts the levels
     }
diff --git a/tests/testthat/test-lyt-tabulation.R b/tests/testthat/test-lyt-tabulation.R
index 7191868e6..6f8b2c85b 100644
--- a/tests/testthat/test-lyt-tabulation.R
+++ b/tests/testthat/test-lyt-tabulation.R
@@ -963,3 +963,22 @@ test_that("cut functions work", {
 
 
 })
+
+## https://github.com/Roche/rtables/issues/323
+
+test_that("empty factor levels represented correctly when ref group is set", {
+
+    df <- data.frame(
+        val = 1:10,
+        grp = factor(rep("a", 10), levels = c("a", "b"))
+    )
+
+
+    tbl <- basic_table() %>%
+        split_cols_by("grp", ref_group = "a") %>%
+        analyze("val") %>%
+        build_table(df)
+
+    expect_identical(ncol(tbl), 2L)
+
+})