Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: Avoid heavy imports in operator constructor
After a previous commit [1] improving import time on operator import, this commit fixes the import time issues when instantiating any of the dbt operators. The main issue still present was the enum imports in the operator constructors, which make DAGs slow to import any time they instantiated one of them. Profiling can be easily run locally, with the following command: ```shell python -X importtime -c "from airflow_dbt_python.operators.dbt import DbtRunOperator; DbtRunOperator(task_id='test')" 2>import-times.log ``` [1] 67c8d78
- Loading branch information
1 parent
3ca0fa2
commit ed205ca
Showing
4 changed files
with
39 additions
and
37 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
"""Utils for project operators and hooks.""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
"""Enumerations with allowed set of values.""" | ||
from enum import Enum | ||
|
||
|
||
class FromStrEnum(Enum): | ||
"""Access enum variants with strings ensuring uppercase.""" | ||
|
||
@classmethod | ||
def from_str(cls, s: str): | ||
"""Instantiate an Enum from a string.""" | ||
return cls[s.replace("-", "_").upper()] | ||
|
||
|
||
class LogFormat(FromStrEnum): | ||
"""Allowed dbt log formats.""" | ||
|
||
DEFAULT = "default" | ||
JSON = "json" | ||
TEXT = "text" | ||
|
||
|
||
class Output(FromStrEnum): | ||
"""Allowed output arguments.""" | ||
|
||
JSON = "json" | ||
NAME = "name" | ||
PATH = "path" | ||
SELECTOR = "selector" | ||
|
||
def __eq__(self, other): | ||
"""Override equality for string comparison.""" | ||
if isinstance(other, str): | ||
return other.upper() == self.name | ||
return Enum.__eq__(self, other) |