RFC: Remove broad-except and bare-except lint warnings (#5492)

Summary: These warnings are extremely low-signal (esp in a library that does general exception handling), I don't think I've ever changed my code as result of seeing the warning pop up. I claim that we should remove them.
dagster-io · Nov 2, 2021 · 6971d50 · 6971d50
1 parent 7e97d35
commit 6971d50
Show file tree

Hide file tree

Showing 59 changed files with 90 additions and 91 deletions.
diff --git a/.pylintrc b/.pylintrc
@@ -16,7 +16,7 @@
 # W1201, W1202 disable log format warning. False positives (I think)
 # W0707 disable raise-missing-from which we cant use because py2 back compat
 
-disable=C,R,duplicate-code,W0511,W1201,W1202,W0707,no-init
+disable=C,R,duplicate-code,W0511,W1201,W1202,W0707,no-init,broad-except,bare-except
 
 # See: https://github.com/getsentry/responses/issues/74
 [TYPECHECK]

diff --git a/examples/airline_demo/airline_demo_tests/conftest.py b/examples/airline_demo/airline_demo_tests/conftest.py
@@ -27,7 +27,7 @@ def is_postgres_running():
 
         # header, one line for container, trailing \n
         return len(lines) == 3
-    except:  # pylint: disable=bare-except
+    except:
         return False
 
 
@@ -67,7 +67,7 @@ def postgres(pg_hostname):  # pylint: disable=redefined-outer-name
             try:
                 subprocess.check_output(["docker-compose", "stop", "test-postgres-db-airline"])
                 subprocess.check_output(["docker-compose", "rm", "-f", "test-postgres-db-airline"])
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 pass
             subprocess.check_output(["docker-compose", "up", "-d", "test-postgres-db-airline"])
 

diff --git a/examples/dbt_example/dbt_example_tests/conftest.py b/examples/dbt_example/dbt_example_tests/conftest.py
@@ -27,7 +27,7 @@ def is_postgres_running():
 
         # header, one line for container, trailing \n
         return len(lines) == 3
-    except:  # pylint: disable=bare-except
+    except:
         return False
 
 
@@ -65,7 +65,7 @@ def postgres(pg_hostname):  # pylint: disable=redefined-outer-name
                     subprocess.check_output(
                         ["docker-compose", "rm", "-f", "dbt_example_postgresql"]
                     )
-                except Exception:  # pylint: disable=broad-except
+                except Exception:
                     pass
                 subprocess.check_output(["docker-compose", "up", "-d", "dbt_example_postgresql"])
 

diff --git a/integration_tests/python_modules/dagster-k8s-test-infra/dagster_k8s_test_infra/cluster.py b/integration_tests/python_modules/dagster-k8s-test-infra/dagster_k8s_test_infra/cluster.py
@@ -152,7 +152,7 @@ def local_port_forward_postgres(namespace):
                 )
                 conn.close()
                 break
-            except:  # pylint: disable=bare-except, broad-except
+            except:
                 time.sleep(1)
                 continue
 
@@ -363,7 +363,7 @@ def check_export_runs(instance):
 
         try:
             export_run(instance, run, output_file)
-        except Exception as e:  # pylint: disable=broad-except
+        except Exception as e:
             print(f"Hit an error exporting dagster-debug {output_file}: {e}")
             continue
 

diff --git a/python_modules/dagit/dagit/graphql.py b/python_modules/dagit/dagit/graphql.py
@@ -227,7 +227,7 @@ async def _handle_async_results(results: AsyncGenerator, operation_id: str, webs
                 payload["errors"] = [format_graphql_error(err) for err in result.errors]
 
             await _send_message(websocket, GraphQLWS.DATA, payload, operation_id)
-    except Exception as error:  # pylint: disable=broad-except
+    except Exception as error:
         if not isinstance(error, GraphQLError):
             error = GraphQLError(str(error))
 

diff --git a/python_modules/dagit/dagit/telemetry.py b/python_modules/dagit/dagit/telemetry.py
@@ -64,7 +64,7 @@ def upload_logs(stop_event, raise_errors=False):
                 in_progress = False
 
             stop_event.wait(600)  # Sleep for 10 minutes
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         if raise_errors:
             raise
 
@@ -107,6 +107,6 @@ def _upload_logs(dagster_log_dir, log_size, dagster_log_queue_dir, raise_errors)
             if success:
                 os.remove(curr_full_path)
 
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         if raise_errors:
             raise
diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/external.py b/python_modules/dagster-graphql/dagster_graphql/implementation/external.py
@@ -59,7 +59,7 @@ def get_subset_external_pipeline(context, selector):
 
     try:
         subset_result = repository_location.get_subset_external_pipeline_result(selector)
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         error_info = serializable_error_info_from_exc_info(sys.exc_info())
         raise UserFacingGraphQLError(
             GrapheneInvalidSubsetError(

diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/utils.py b/python_modules/dagster-graphql/dagster_graphql/implementation/utils.py
@@ -34,7 +34,7 @@ def _fn(*args, **kwargs):
             return fn(*args, **kwargs)
         except UserFacingGraphQLError as de_exception:
             return de_exception.error
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             return GraphenePythonError(serializable_error_info_from_exc_info(sys.exc_info()))
 
     return _fn

diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py b/python_modules/dagster-graphql/dagster_graphql/schema/instigation.py
@@ -182,7 +182,7 @@ def resolve_evaluationResult(self, graphene_info):
                 schedule_name=external_schedule.name,
                 scheduled_execution_time=schedule_time,
             )
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             schedule_data = serializable_error_info_from_exc_info(sys.exc_info())
 
         return GrapheneTickEvaluation(schedule_data)

diff --git a/python_modules/dagster/dagster/cli/pipeline.py b/python_modules/dagster/dagster/cli/pipeline.py
@@ -913,7 +913,7 @@ def _execute_backfill_command_at_location(
             repo_handle,
             partition_set_name,
         )
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         error_info = serializable_error_info_from_exc_info(sys.exc_info())
         raise DagsterBackfillFailedError(
             "Failure fetching partition names: {error_message}".format(
@@ -958,7 +958,7 @@ def _execute_backfill_command_at_location(
                     partition_names=partition_names,
                 )
             )
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error_info = serializable_error_info_from_exc_info(sys.exc_info())
             instance.add_backfill(
                 backfill_job.with_status(BulkActionStatus.FAILED).with_error(error_info)

diff --git a/python_modules/dagster/dagster/cli/sensor.py b/python_modules/dagster/dagster/cli/sensor.py
@@ -284,7 +284,7 @@ def execute_preview_command(
                         last_run_key,
                         cursor,
                     )
-                except Exception:  # pylint: disable=broad-except
+                except Exception:
                     error_info = serializable_error_info_from_exc_info(sys.exc_info())
                     print_fn(
                         "Failed to resolve sensor for {sensor_name} : {error_info}".format(

diff --git a/python_modules/dagster/dagster/core/definitions/inference.py b/python_modules/dagster/dagster/core/definitions/inference.py
@@ -32,7 +32,7 @@ def _infer_input_description_from_docstring(fn: Callable) -> Dict[str, Optional[
     try:
         docstring = parse(doc_str)
         return {p.arg_name: p.description for p in docstring.params}
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         return {}
 
 
@@ -48,7 +48,7 @@ def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]:
             return None
 
         return docstring.returns.description
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         return None
 
 

diff --git a/python_modules/dagster/dagster/core/definitions/reconstructable.py b/python_modules/dagster/dagster/core/definitions/reconstructable.py
@@ -340,7 +340,7 @@ def make_bar_job():
             and inspect.getmodule(target).__name__ != "__main__"
         ):
             return ReconstructablePipeline.for_module(target.__module__, target.__name__)
-    except:  # pylint: disable=bare-except
+    except:
         pass
 
     python_file = get_python_file_from_target(target)

diff --git a/python_modules/dagster/dagster/core/definitions/utils.py b/python_modules/dagster/dagster/core/definitions/utils.py
@@ -91,7 +91,7 @@ def validate_tags(tags: Optional[Dict[str, Any]]) -> Dict[str, Any]:
                 )
 
                 valid = seven.json.loads(str_val) == value
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 pass
 
             if not valid:

diff --git a/python_modules/dagster/dagster/core/execution/api.py b/python_modules/dagster/dagster/core/execution/api.py
@@ -787,7 +787,7 @@ def pipeline_execution_iterator(
         pipeline_canceled_info = serializable_error_info_from_exc_info(sys.exc_info())
         if pipeline_context.raise_on_error:
             raise
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         pipeline_exception_info = serializable_error_info_from_exc_info(sys.exc_info())
         if pipeline_context.raise_on_error:
             raise  # finally block will run before this is re-raised

diff --git a/python_modules/dagster/dagster/core/execution/plan/execute_plan.py b/python_modules/dagster/dagster/core/execution/plan/execute_plan.py
@@ -284,7 +284,7 @@ def _dagster_event_sequence_for_step(step_context: StepExecutionContext) -> Iter
             raise dagster_error
 
     # case (6) in top comment
-    except Exception as unexpected_exception:  # pylint: disable=broad-except
+    except Exception as unexpected_exception:
         step_context.capture_step_exception(unexpected_exception)
         yield step_failure_event_from_exc_info(
             step_context,

diff --git a/python_modules/dagster/dagster/core/executor/child_process_executor.py b/python_modules/dagster/dagster/core/executor/child_process_executor.py
@@ -66,7 +66,6 @@ def _execute_command_in_child_process(event_queue, command):
                 event_queue.put(step_event)
             event_queue.put(ChildProcessDoneEvent(pid=pid))
 
-        # pylint: disable=broad-except
         except (
             Exception,
             KeyboardInterrupt,

diff --git a/python_modules/dagster/dagster/core/host_representation/grpc_server_registry.py b/python_modules/dagster/dagster/core/host_representation/grpc_server_registry.py
@@ -181,7 +181,7 @@ def _get_grpc_endpoint(self, repository_location_origin):
                     startup_timeout=self._startup_timeout,
                 )
                 self._all_processes.append(server_process)
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 server_process = serializable_error_info_from_exc_info(sys.exc_info())
                 new_server_id = None
 

diff --git a/python_modules/dagster/dagster/core/storage/event_log/in_memory.py b/python_modules/dagster/dagster/core/storage/event_log/in_memory.py
@@ -96,7 +96,7 @@ def store_event(self, event):
         for handler in handlers:
             try:
                 handler(event)
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 logging.exception("Exception in callback for event watch on run %s.", run_id)
 
     def delete_events(self, run_id):

diff --git a/...on_modules/dagster/dagster/core/storage/event_log/sqlite/consolidated_sqlite_event_log.py b/...on_modules/dagster/dagster/core/storage/event_log/sqlite/consolidated_sqlite_event_log.py
@@ -162,7 +162,7 @@ def on_modified(self):
                 status = None
                 try:
                     status = callback(event)
-                except Exception:  # pylint: disable=broad-except
+                except Exception:
                     logging.exception("Exception in callback for event watch on run %s.", run_id)
 
                 if (

diff --git a/python_modules/dagster/dagster/core/storage/event_log/sqlite/sqlite_event_log.py b/python_modules/dagster/dagster/core/storage/event_log/sqlite/sqlite_event_log.py
@@ -417,7 +417,7 @@ def _process_log(self):
             status = None
             try:
                 status = self._cb(event)
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 logging.exception("Exception in callback for event watch on run %s.", self._run_id)
 
             if (

diff --git a/python_modules/dagster/dagster/core/storage/sql.py b/python_modules/dagster/dagster/core/storage/sql.py
@@ -68,7 +68,7 @@ def handle_schema_errors(conn, alembic_config, msg=None):
                 db_revision, head_revision = check_alembic_revision(alembic_config, conn)
         # If exceptions were raised during the revision check, we want to swallow them and
         # allow the original exception to fall through
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             pass
 
         if db_revision != head_revision:

diff --git a/python_modules/dagster/dagster/core/telemetry.py b/python_modules/dagster/dagster/core/telemetry.py
@@ -324,7 +324,7 @@ def _set_telemetry_instance_id():
         with open(telemetry_id_path, "w") as telemetry_id_file:
             yaml.dump({INSTANCE_ID_STR: instance_id}, telemetry_id_file, default_flow_style=False)
         return instance_id
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         return "<<unable_to_write_instance_id>>"
 
 

diff --git a/python_modules/dagster/dagster/core/workspace/context.py b/python_modules/dagster/dagster/core/workspace/context.py
@@ -528,7 +528,7 @@ def _load_location(self, origin):
         error = None
         try:
             location = self._create_location_from_origin(origin)
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error = serializable_error_info_from_exc_info(sys.exc_info())
             warnings.warn(
                 "Error loading repository location {location_name}:{error_string}".format(

diff --git a/python_modules/dagster/dagster/daemon/backfill.py b/python_modules/dagster/dagster/daemon/backfill.py
@@ -105,7 +105,7 @@ def execute_backfill_iteration(instance, workspace, logger, debug_crash_flags=No
                     )
                     instance.update_backfill(backfill_job.with_status(BulkActionStatus.COMPLETED))
                     yield
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error_info = serializable_error_info_from_exc_info(sys.exc_info())
             instance.update_backfill(
                 backfill_job.with_status(BulkActionStatus.FAILED).with_error(error_info)

diff --git a/python_modules/dagster/dagster/daemon/controller.py b/python_modules/dagster/dagster/daemon/controller.py
@@ -186,7 +186,7 @@ def _daemon_heartbeat_healthy(self, daemon_type):
             if is_healthy:
                 self._last_healthy_heartbeat_times[daemon_type] = now
             return is_healthy
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             self._logger.warning(
                 "Error attempting to check {daemon_type} heartbeat:".format(
                     daemon_type=daemon_type,

diff --git a/python_modules/dagster/dagster/daemon/daemon.py b/python_modules/dagster/dagster/daemon/daemon.py
@@ -102,7 +102,7 @@ def run_loop(
                             heartbeat_interval_seconds,
                             error_interval_seconds,
                         )
-                    except Exception:  # pylint: disable=broad-except
+                    except Exception:
                         self._logger.error(
                             "Failed to add heartbeat: \n{}".format(
                                 serializable_error_info_from_exc_info(sys.exc_info())
@@ -135,7 +135,7 @@ def _run_iteration(
                         self._errors.appendleft((result, pendulum.now("UTC")))
                 except StopIteration:
                     break
-                except Exception:  # pylint: disable=broad-except
+                except Exception:
                     error_info = serializable_error_info_from_exc_info(sys.exc_info())
                     self._logger.error("Caught error:\n{}".format(error_info))
                     self._errors.appendleft((error_info, pendulum.now("UTC")))
@@ -148,7 +148,7 @@ def _run_iteration(
                             heartbeat_interval_seconds,
                             error_interval_seconds,
                         )
-                    except Exception:  # pylint: disable=broad-except
+                    except Exception:
                         self._logger.error(
                             "Failed to add heartbeat: \n{}".format(
                                 serializable_error_info_from_exc_info(sys.exc_info())

diff --git a/python_modules/dagster/dagster/daemon/monitoring/monitoring_daemon.py b/python_modules/dagster/dagster/daemon/monitoring/monitoring_daemon.py
@@ -88,7 +88,7 @@ def execute_monitoring_iteration(instance, workspace, logger, _debug_crash_flags
                 pass
             else:
                 check.invariant(False, f"Unexpected run status: {run.status}")
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error_info = serializable_error_info_from_exc_info(sys.exc_info())
             logger.error(f"Hit error while monitoring run {run.run_id}: " f"{str(error_info)}")
             yield error_info

diff --git a/python_modules/dagster/dagster/daemon/run_coordinator/queued_run_coordinator_daemon.py b/python_modules/dagster/dagster/daemon/run_coordinator/queued_run_coordinator_daemon.py
@@ -147,7 +147,7 @@ def run_iteration(self, instance, workspace):
 
             try:
                 self._dequeue_run(instance, run, workspace)
-            except Exception:  # pylint: disable=broad-except
+            except Exception:
                 error_info = serializable_error_info_from_exc_info(sys.exc_info())
 
                 message = (

diff --git a/python_modules/dagster/dagster/daemon/sensor.py b/python_modules/dagster/dagster/daemon/sensor.py
@@ -237,7 +237,7 @@ def execute_sensor_iteration(
                     job_state,
                     sensor_debug_crash_flags,
                 )
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error_info = serializable_error_info_from_exc_info(sys.exc_info())
             logger.error(
                 "Sensor daemon caught an error for sensor {sensor_name} : {error_info}".format(
@@ -365,7 +365,7 @@ def _evaluate_sensor(
                     run_id=run.run_id, sensor_name=external_sensor.name
                 )
             )
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             error_info = serializable_error_info_from_exc_info(sys.exc_info())
             context.logger.error(
                 f"Run {run.run_id} created successfully but failed to launch: " f"{str(error_info)}"

diff --git a/python_modules/dagster/dagster/grpc/client.py b/python_modules/dagster/dagster/grpc/client.py
@@ -363,7 +363,7 @@ def start_run(self, execute_run_args):
                 )
                 return res.serialized_start_run_result
 
-            except Exception:  # pylint: disable=bare-except
+            except Exception:
                 pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id)
                 instance.report_engine_event(
                     message="Unexpected error in IPC client",

diff --git a/python_modules/dagster/dagster/grpc/impl.py b/python_modules/dagster/dagster/grpc/impl.py
@@ -73,7 +73,7 @@ def core_execute_run(recon_pipeline, pipeline_run, instance, resume_from_failure
     # try to load the pipeline definition early
     try:
         recon_pipeline.get_definition()
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         yield instance.report_engine_event(
             "Could not load pipeline definition.",
             pipeline_run,
@@ -92,7 +92,7 @@ def core_execute_run(recon_pipeline, pipeline_run, instance, resume_from_failure
             message="Run execution terminated by interrupt",
             pipeline_run=pipeline_run,
         )
-    except Exception:  # pylint: disable=broad-except
+    except Exception:
         yield instance.report_engine_event(
             "An exception was thrown during execution that is likely a framework error, "
             "rather than an error in user code.",
@@ -128,7 +128,7 @@ def _run_in_subprocess(
 
         pid = os.getpid()
 
-    except:  # pylint: disable=bare-except
+    except:
         serializable_error_info = serializable_error_info_from_exc_info(sys.exc_info())
         event = IPCErrorMessage(
             serializable_error_info=serializable_error_info,
@@ -357,7 +357,7 @@ def get_external_execution_plan_snapshot(recon_pipeline, args):
             ),
             args.pipeline_snapshot_id,
         )
-    except:  # pylint: disable=bare-except
+    except:
         return ExecutionPlanSnapshotErrorData(
             error=serializable_error_info_from_exc_info(sys.exc_info())
         )