diff --git a/test/overlay-dirs.bats b/test/overlay-dirs.bats
index 5623c6e7e..951f46443 100644
--- a/test/overlay-dirs.bats
+++ b/test/overlay-dirs.bats
@@ -137,22 +137,30 @@ EOF
     echo "modifying file" > dir_to_overlay1/file1.txt
     echo "modifying file" > dir_to_overlay2/file2.txt
     echo "modifying file" > dir_to_overlay3/file3.txt
+    
     NO_DEBUG=1
     stacker build
     echo $output | grep "cache miss because content of 3 overlay dirs changed:"
     echo $output | grep "Changed overlay_dir:"
-    echo $output | grep "dir_to_overlay2"
-    echo $output | grep "file2.txt"
+    # without debug should print only 2 dirs
+    result=$(echo $output | grep -o /dir_to_overlay[1-3]/file[1-3].txt | wc -l)
+    [ $result -eq 2 ]
+
     echo $output | grep "and 1 other overlay_dirs. use --debug for complete output"
     # now with debug
     echo "modifying file again" > dir_to_overlay1/file1.txt
     echo "modifying file again" > dir_to_overlay2/file2.txt
     echo "modifying file again" > dir_to_overlay3/file3.txt
+
     NO_DEBUG=0
     stacker build
     echo $output | grep "cache miss because content of 3 overlay dirs changed:"
     echo $output | grep "Changed overlay_dir:"
+    echo $output | grep "dir_to_overlay1"
+    echo $output | grep "dir_to_overlay2"
     echo $output | grep "dir_to_overlay3"
+    echo $output | grep "file1.txt"
+    echo $output | grep "file2.txt"
     echo $output | grep "file3.txt"
     result=$(echo $output | grep "and 1 other overlay_dirs. use --debug for complete output" || echo "empty")
     echo $result | grep "empty"