diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 34ae302ed..8c9e5acce 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -47,6 +47,18 @@ jobs:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
       - name: Test all exercises using java-test-runner
         run: bin/test-with-test-runner
+      - name: Print summary
+        run: |
+          if [ -f exercises/build/summary.txt ]; then
+            echo "===== TEST SUMMARY ====="
+            cat exercises/build/summary.txt
+            echo "========================"
+          else
+            echo "===== ALL TESTS PASSED ====="
+            echo "No summary file was generated."
+            echo "============================="
+          fi
+        if: always() 
       - name: Archive test results
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
         with:
@@ -64,6 +76,18 @@ jobs:
           fetch-depth: 0
       - name: Test changed exercises using java-test-runner
         run: bin/test-changed-exercise
+      - name: Print summary
+        run: |
+          if [ -f exercises/build/summary.txt ]; then
+            echo "===== TEST SUMMARY ====="
+            cat exercises/build/summary.txt
+            echo "========================"
+          else
+            echo "===== ALL TESTS PASSED ====="
+            echo "No summary file was generated."
+            echo "============================="
+          fi
+        if: always() 
       - name: Archive test results
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
         with:
diff --git a/bin/test-changed-exercise b/bin/test-changed-exercise
index cbc684b79..b18a488e8 100755
--- a/bin/test-changed-exercise
+++ b/bin/test-changed-exercise
@@ -35,7 +35,12 @@ echo "Changed exercises detected:"
 echo "$changed_exercises"
 echo "----------------------------------------"
 
+summary_dir="exercises/build"
+summary_file="${summary_dir}/summary.txt"
+mkdir -p "$summary_dir"
+
 # Run tests
+exit_code=0
 for dir in $changed_exercises; do
   slug=$(basename "$dir")
 
@@ -47,8 +52,27 @@ for dir in $changed_exercises; do
   mkdir -p "$(dirname "$results_path")"
 
   if [[ $dir == exercises/practice/* ]]; then
-    ./exercises/gradlew -p exercises ":practice:$slug:test" 2>&1 | tee "$results_path"
+    ./exercises/gradlew -p exercises ":practice:$slug:test" 2>&1 | tee "$results_path" || true
   elif [[ $dir == exercises/concept/* ]]; then
-    ./exercises/gradlew -p exercises ":concept:$slug:test" 2>&1 | tee "$results_path"
+    ./exercises/gradlew -p exercises ":concept:$slug:test" 2>&1 | tee "$results_path" || true
+  fi
+
+  # Detect failure
+  if grep -q "FAILED" "$results_path"; then
+    exit_code=1
+
+    # Determine practice/slug or concept/slug
+    relative_path=$(echo "$dir" | sed 's|^exercises/||')
+
+    # Create summary.txt with header only on first failure
+    if [ ! -f "$summary_file" ]; then
+        echo "The following exercises have test failures or test errors:" > "$summary_file"
+    fi
+
+    # Append the correct path (practice/slug or concept/slug)
+    echo "$relative_path" >> "$summary_file"
   fi
+
 done
+
+exit $exit_code
\ No newline at end of file
diff --git a/bin/test-with-test-runner b/bin/test-with-test-runner
index eed2d74b0..5e5dc8889 100755
--- a/bin/test-with-test-runner
+++ b/bin/test-with-test-runner
@@ -12,6 +12,10 @@ docker pull exercism/java-test-runner
 
 exit_code=0
 
+summary_dir="exercises/build"
+summary_file="${summary_dir}/summary.txt"
+mkdir -p "$summary_dir"
+
 function run_test_runner() {
     local slug=$1
     local solution_dir=$2
@@ -56,6 +60,18 @@ function verify_exercise() {
 
     if [[ $(jq -r '.status' "${results_file}") != "pass" ]]; then
         echo "${slug}: ${implementation_file_key} solution did not pass the tests"
+
+        # Determine practice/slug or concept/slug
+        local relative_path=$(echo "$dir" | sed -E 's|.*/exercises/||')
+
+        # Create summary.txt with header only on first failure
+        if [ ! -f "$summary_file" ]; then
+            echo "The following exercises have test failures or test errors:" > "$summary_file"
+        fi
+
+        # Append the correct path (practice/slug or concept/slug)
+        echo "$relative_path" >> "$summary_file"
+
         exit_code=1
     fi