From 99c7038ac70884551fed58b2c92a3ac78205601e Mon Sep 17 00:00:00 2001 From: Dylan Jew Date: Thu, 23 Jul 2026 11:44:45 -0400 Subject: [PATCH 1/2] Gracefully handle invalid ghost fuzzer configurations When a job's fuzzer gets detached, it leaves a FuzzerJob mapping with an empty string for the fuzzer. These empty fuzzers were getting scheduled due to their weight remaining positive. When the workers picked up these ghost tasks, setup.py would fetch an invalid Fuzzer entity that lacked a blobstore_key and pass it to blobs.py in get_gcs_path, which triggered a TypeError. This change raises an InvalidFuzzerError up front instead, gracefully clearing the bad tasks without crashing. --- src/clusterfuzz/_internal/bot/tasks/setup.py | 3 +++ .../_internal/bot/tasks/utasks/fuzz_task.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/setup.py b/src/clusterfuzz/_internal/bot/tasks/setup.py index e1e6e1d5275..81492530ca1 100644 --- a/src/clusterfuzz/_internal/bot/tasks/setup.py +++ b/src/clusterfuzz/_internal/bot/tasks/setup.py @@ -593,6 +593,9 @@ def preprocess_update_fuzzer_and_data_bundles( if not fuzzer: logs.error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError + if not fuzzer.blobstore_key: + logs.error(f'Fuzzer {fuzzer_name} does not have a blobstore_key.') + raise errors.InvalidFuzzerError update_input = uworker_msg_pb2.SetupInput( # pylint: disable=no-member fuzzer_name=fuzzer_name, diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py index 3d979d852ce..3b24aaa48ca 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py @@ -29,6 +29,7 @@ from google.cloud import ndb from clusterfuzz._internal.base import dates +from clusterfuzz._internal.base import errors from clusterfuzz._internal.base import utils from clusterfuzz._internal.bot import testcase_manager from clusterfuzz._internal.bot.fuzzers import builtin @@ -1216,15 +1217,15 @@ def write_crashes_to_big_query(group, newly_created_testcase, existing_testcase, # Happens in case the big query function is disabled (local development). return - errors = result.get('insertErrors', []) - failed_count = len(errors) + insert_errors = result.get('insertErrors', []) + failed_count = len(insert_errors) monitoring_metrics.BIG_QUERY_WRITE_COUNT.increment_by( row_count - failed_count, {'success': True}) monitoring_metrics.BIG_QUERY_WRITE_COUNT.increment_by( failed_count, {'success': False}) - for error in errors: + for error in insert_errors: logs.error( ('Ignoring error writing the crash ' f'({group.crashes[error["index"]].crash_type}) to BigQuery.'), @@ -2290,7 +2291,11 @@ def utask_preprocess(fuzzer_name, job_type, uworker_env): # Delay adding the fuzz target to logs context until it is chosen in # preprocess. with logs.fuzzer_log_context(fuzzer_name, job_type, fuzz_target=None): - return _utask_preprocess(fuzzer_name, job_type, uworker_env) + try: + return _utask_preprocess(fuzzer_name, job_type, uworker_env) + except errors.InvalidFuzzerError: + logs.error('Fuzzer %s is invalid or no longer exists.' % fuzzer_name) + return None def save_fuzz_targets(output): From f4b7c152ed6f56cd9d816ee3531dac70ee7437f9 Mon Sep 17 00:00:00 2001 From: Dylan Jew Date: Thu, 23 Jul 2026 12:31:37 -0400 Subject: [PATCH 2/2] Fix test --- src/clusterfuzz/_internal/tests/core/bot/tasks/setup_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/tests/core/bot/tasks/setup_test.py b/src/clusterfuzz/_internal/tests/core/bot/tasks/setup_test.py index 6e39dbe6b1c..f7dbe70daae 100644 --- a/src/clusterfuzz/_internal/tests/core/bot/tasks/setup_test.py +++ b/src/clusterfuzz/_internal/tests/core/bot/tasks/setup_test.py @@ -222,7 +222,10 @@ def setUp(self): self.fuzzer_name = 'fuzzer' data_bundle_name = 'data_bundle_name' data_types.Fuzzer( - name=self.fuzzer_name, data_bundle_name=data_bundle_name).put() + name=self.fuzzer_name, + data_bundle_name=data_bundle_name, + blobstore_key='blobstore_key', + ).put() self.data_bundle = data_types.DataBundle(name=data_bundle_name) self.data_bundle.put() data_types.DataBundle(name=data_bundle_name).put()