From c1a85de5ce3c51065c16dac774293221856d55b5 Mon Sep 17 00:00:00 2001 From: Erin Limbo Date: Wed, 27 May 2026 22:10:24 +0000 Subject: [PATCH 1/2] fix: prevent silent errors on DB query timeouts and extend deadline --- evalbench/evaluator/evaluator.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/evalbench/evaluator/evaluator.py b/evalbench/evaluator/evaluator.py index 23e35de1..178ebc9c 100644 --- a/evalbench/evaluator/evaluator.py +++ b/evalbench/evaluator/evaluator.py @@ -173,12 +173,23 @@ def evaluate( record_successful_sql_gen(progress_reporting) try: - db_conn = db_queue.get(timeout=60) + db_conn = db_queue.get(timeout=180) work = sqlexecwork.SQLExecWork( db_conn, self.config, eval_output, db_queue ) self.sqlrunner.execute_work(work) exec_future_to_eval[self.sqlrunner.futures[-1]] = eval_output + except queue.Empty: + error_msg = f"Timeout Error: Waited too long (queue.Empty) for database '{eval_output.get('database', 'unknown')}'" + logging.error(error_msg) + eval_output["generated_error"] = error_msg + + record_successful_sql_exec(progress_reporting) + work = scorework.ScorerWork( + self.config, eval_output, scoring_results, global_models + ) + self.scoringrunner.execute_work(work) + score_future_to_eval[self.scoringrunner.futures[-1]] = eval_output except Exception as e: logging.error( From fbfd042a6e61d4cf8422e0099564a5ac6bd0cc58 Mon Sep 17 00:00:00 2001 From: Erin Limbo Date: Wed, 27 May 2026 22:44:00 +0000 Subject: [PATCH 2/2] pystyle --- evalbench/evaluator/evaluator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/evalbench/evaluator/evaluator.py b/evalbench/evaluator/evaluator.py index 178ebc9c..32bdbb8f 100644 --- a/evalbench/evaluator/evaluator.py +++ b/evalbench/evaluator/evaluator.py @@ -173,7 +173,7 @@ def evaluate( record_successful_sql_gen(progress_reporting) try: - db_conn = db_queue.get(timeout=180) + db_conn = db_queue.get(timeout=180) work = sqlexecwork.SQLExecWork( db_conn, self.config, eval_output, db_queue ) @@ -183,13 +183,14 @@ def evaluate( error_msg = f"Timeout Error: Waited too long (queue.Empty) for database '{eval_output.get('database', 'unknown')}'" logging.error(error_msg) eval_output["generated_error"] = error_msg - + record_successful_sql_exec(progress_reporting) work = scorework.ScorerWork( self.config, eval_output, scoring_results, global_models ) self.scoringrunner.execute_work(work) - score_future_to_eval[self.scoringrunner.futures[-1]] = eval_output + score_future_to_eval[self.scoringrunner.futures[-1] + ] = eval_output except Exception as e: logging.error(