Merge pull request #86 from hvaara/re-progress-checker-fix

Reintroduce progress checker from #48

Author: terryyz

bigcodebench/evaluate.py

@@ -322,14 +322,13 @@ def evaluate(
                     assert len(completion_id) == len(problems), f"Missing problems in samples. Expected {len(problems)} problems, got {len(completion_id)}"
 
                     def stucking_checker():
-                        while remainings:
-                            last_size = len(remainings)
-                            time.sleep(240)
-                            if last_size != len(remainings) or len(remainings) == 0:
-                                continue
-                            # Potential stucking
-                            warn("No samples had finished testing in the last 240s")
-                            warn(f"{len(remainings)} samples to be tested: {remainings}")
+                        not_done = futures
+                        while len(not_done) > 0:
+                            done, not_done = wait(not_done, timeout=240, return_when=FIRST_COMPLETED)
+
+                            if len(done) == 0:
+                                warn("No samples have finished testing in the last 240s")
+                                warn(f"{len(remainings)} samples to be tested: {remainings}")
 
                     threading.Thread(target=stucking_checker).start()