Skip to content

Commit

Permalink
Show serialization exceptions in DAG parsing log (#17277)
Browse files Browse the repository at this point in the history
Make sure that any exceptions that happen when writing serialized DAGs
to the db get written to the DAG parsing log, instead of only being added
to `import_errors` for consumption via the UI.

(cherry picked from commit 9cd5a97)
  • Loading branch information
jedcunningham authored and jhtimmins committed Aug 9, 2021
1 parent 798237a commit ed76d11
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 1 deletion.
1 change: 1 addition & 0 deletions airflow/models/dagbag.py
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,7 @@ def _serialize_dag_capturing_errors(dag, session):
except OperationalError:
raise
except Exception:
self.log.exception("Failed to write serialized DAG: %s", dag.full_filepath)
return [(dag.fileloc, traceback.format_exc(limit=-self.dagbag_import_error_traceback_depth))]

# Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case
Expand Down
4 changes: 3 additions & 1 deletion tests/models/test_dagbag.py
Original file line number Diff line number Diff line change
Expand Up @@ -703,7 +703,9 @@ def test_serialized_dag_errors_are_import_errors(self, mock_serialize):
)
assert dagbag.import_errors == {}

dagbag.sync_to_db(session=session)
with self.assertLogs(level="ERROR") as cm:
dagbag.sync_to_db(session=session)
self.assertIn("SerializationError", "\n".join(cm.output))

assert path in dagbag.import_errors
err = dagbag.import_errors[path]
Expand Down

0 comments on commit ed76d11

Please sign in to comment.