diff --git a/.flake8 b/.flake8 index 3aa19a94..8e259722 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,8 @@ [flake8] max-line-length = 88 docstring-convention = google -# Will finish these later -extend-ignore = D101,D107 +ignore = D107, W503 per-file-ignores = - tests/*.py: D1,D2,D4 - examples/*.py: D1,D2,D4 - __version__.py: D100 __init__.py: F401 exclude = airflow @@ -16,6 +12,7 @@ exclude = setup.py build dist + tests .venv .tox .mypy_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75341573..75a3dac6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,8 @@ repos: hooks: - id: flake8 additional_dependencies: [flake8-docstrings] + args: ["--config", ".flake8"] + exclude: tests/ - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.902 diff --git a/airflow_dbt_python/__version__.py b/airflow_dbt_python/__version__.py index 8201c2f7..c847a74e 100644 --- a/airflow_dbt_python/__version__.py +++ b/airflow_dbt_python/__version__.py @@ -1,3 +1,4 @@ +"""The module's version information.""" __author__ = "Tomás Farías Santana" __copyright__ = "Copyright 2021 Tomás Farías Santana" __title__ = "airflow-dbt-python" diff --git a/airflow_dbt_python/hooks/dbt_s3.py b/airflow_dbt_python/hooks/dbt_s3.py index 75d694f9..37e46f15 100644 --- a/airflow_dbt_python/hooks/dbt_s3.py +++ b/airflow_dbt_python/hooks/dbt_s3.py @@ -7,6 +7,12 @@ class DbtS3Hook(S3Hook): + """Subclass of S3Hook with methods to pull dbt-related files. + + A dbt hook should provide a method to pull a dbt profiles file (profiles.yml) and + all the files corresponding to a project. + """ + def get_dbt_profiles( self, s3_profiles_url: str, profiles_dir: Optional[str] = None ) -> Path: diff --git a/airflow_dbt_python/operators/dbt.py b/airflow_dbt_python/operators/dbt.py index 4486a5c2..36170fbc 100644 --- a/airflow_dbt_python/operators/dbt.py +++ b/airflow_dbt_python/operators/dbt.py @@ -95,7 +95,14 @@ def __init__( self._dbt_s3_hook = None def execute(self, context: dict): - """Execute dbt command with prepared arguments.""" + """Execute dbt command with prepared arguments. + + Execution requires setting up a directory with the dbt project files and + overriding the logging. + + Args: + context: The Airflow's task context + """ with self.dbt_directory() as dbt_dir: # type: str with self.override_dbt_logging(dbt_dir): args = self.prepare_args() @@ -125,6 +132,9 @@ def execute(self, context: dict): def xcom_push_artifacts(self, context: dict, dbt_directory: str): """Read dbt artifacts and push them to XCom. + Artifacts are read from the target/ directory in dbt_directory. This method will + fail if the required artifact is not found. + Args: context: The Airflow task's context. dbt_directory: A directory containing a dbt project. Artifacts will be @@ -160,7 +170,11 @@ def prepare_args(self) -> list[Optional[str]]: return args def args_list(self) -> list[str]: - """Build a list of arguments to pass to dbt.""" + """Build a list of arguments to pass to dbt. + + Building involves creating a list of flags for dbt to parse given the operators + attributes and the values specified by __dbt_args__. + """ args = [] for arg in self.__dbt_args__: value = getattr(self, arg, None) @@ -289,7 +303,12 @@ def serializable_result( class DbtRunOperator(DbtBaseOperator): - """Executes dbt run.""" + """Executes a dbt run command. + + The run command executes SQL model files against the given target. The + documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/run. + """ command = "run" @@ -337,7 +356,12 @@ def __init__( class DbtSeedOperator(DbtBaseOperator): - """Executes dbt seed.""" + """Executes a dbt seed command. + + The seed command loads csv files into the the given target. The + documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/seed. + """ command = "seed" @@ -373,7 +397,12 @@ def __init__( class DbtTestOperator(DbtBaseOperator): - """Executes dbt test.""" + """Executes a dbt test command. + + The test command runs data and/or schema tests. The + documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/test. + """ command = "test" @@ -424,7 +453,12 @@ def __init__( class DbtCompileOperator(DbtBaseOperator): - """Executes dbt compile.""" + """Executes a dbt compile command. + + The compile command generates SQL files. The + documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/compile. + """ command = "compile" @@ -469,7 +503,11 @@ def __init__( class DbtDepsOperator(DbtBaseOperator): - """Executes dbt deps.""" + """Executes a dbt deps command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/deps. + """ command = "deps" @@ -478,7 +516,11 @@ def __init__(self, **kwargs) -> None: class DbtCleanOperator(DbtBaseOperator): - """Executes dbt clean.""" + """Executes a dbt clean command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/debug. + """ command = "clean" @@ -487,7 +529,11 @@ def __init__(self, **kwargs) -> None: class DbtDebugOperator(DbtBaseOperator): - """Execute dbt debug.""" + """Executes a dbt debug command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/debug. + """ command = "debug" @@ -505,7 +551,11 @@ def __init__( class DbtSnapshotOperator(DbtBaseOperator): - """Execute dbt snapshot.""" + """Executes a dbt snapshot command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/snapshot. + """ command = "snapshot" @@ -535,7 +585,11 @@ def __init__( class DbtLsOperator(DbtBaseOperator): - """Execute dbt list (or ls).""" + """Executes a dbt list (or ls) command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/list. + """ command = "ls" @@ -572,7 +626,11 @@ def __init__( class DbtRunOperationOperator(DbtBaseOperator): - """Execute dbt run-operation.""" + """Executes a dbt run-operation command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/run-operation. + """ command = "run-operation" @@ -591,7 +649,11 @@ def __init__( class DbtParseOperator(DbtBaseOperator): - """Execute dbt parse.""" + """Executes a dbt parse command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/parse. + """ command = "parse" @@ -603,7 +665,11 @@ def __init__( class DbtSourceOperator(DbtBaseOperator): - """Execute dbt source.""" + """Executes a dbt source command. + + The documentation for the dbt command can be found here: + https://docs.getdbt.com/reference/commands/source. + """ command = "source" @@ -640,7 +706,7 @@ def __init__( class DbtBuildOperator(DbtBaseOperator): - """Execute dbt build. + """Executes a dbt build command. The build command combines the run, test, seed, and snapshot commands into one. The full Documentation for the dbt build command can be found here: diff --git a/examples/__init__.py b/examples/__init__.py index e69de29b..8f6e9364 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Usage examples of airflow-dbt-python.""" diff --git a/examples/basic_dag.py b/examples/basic_dag.py index ce57e578..165e204b 100644 --- a/examples/basic_dag.py +++ b/examples/basic_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which dbt runs a project -""" +"""Sample basic DAG which dbt runs a project.""" import datetime as dt from airflow import DAG diff --git a/examples/complete_dbt_workflow_dag.py b/examples/complete_dbt_workflow_dag.py index 74ff87c3..1f76d95d 100644 --- a/examples/complete_dbt_workflow_dag.py +++ b/examples/complete_dbt_workflow_dag.py @@ -1,5 +1,6 @@ -""" -Sample DAG showcasing a complete dbt workflow +"""Sample DAG showcasing a complete dbt workflow. + +The complete workflow includes a sequence of source, seed, and several run commands. """ import datetime as dt diff --git a/examples/dbt_project_in_s3_dag.py b/examples/dbt_project_in_s3_dag.py index 039a41b8..7ed7eb73 100644 --- a/examples/dbt_project_in_s3_dag.py +++ b/examples/dbt_project_in_s3_dag.py @@ -1,6 +1,4 @@ -""" -Sample basic DAG which showcases a dbt project being pulled from S3 -""" +"""Sample basic DAG which showcases a dbt project being pulled from S3.""" import datetime as dt from airflow import DAG diff --git a/examples/use_dbt_artifacts_dag.py b/examples/use_dbt_artifacts_dag.py index eeb3e877..8f2cc439 100644 --- a/examples/use_dbt_artifacts_dag.py +++ b/examples/use_dbt_artifacts_dag.py @@ -1,6 +1,4 @@ -""" -Sample DAG to showcase pulling dbt artifacts from XCOM -""" +"""Sample DAG to showcase pulling dbt artifacts from XCOM.""" import datetime as dt from airflow import DAG @@ -10,9 +8,7 @@ def process_dbt_artifacts(**context): - """ - Report which model or models took the longest to compile and execute - """ + """Report which model or models took the longest to compile and execute.""" run_results = context["ti"].xcom_pull( key="run_results.json", task_ids="dbt_run_daily" ) diff --git a/tests/conftest.py b/tests/conftest.py index fbfbb531..3d9e7039 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,8 @@ +"""Conftest file including setting common fixtures. + +Common fixtures include a connection to a postgres database, a set of sample model and + seed files, dbt configuration files, and temporary directories for everything. +""" import boto3 import pytest from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_base.py b/tests/test_dbt_base.py index 456b84d9..52410d43 100644 --- a/tests/test_dbt_base.py +++ b/tests/test_dbt_base.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBaseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_build.py b/tests/test_dbt_build.py index 90994a3c..f84e792c 100644 --- a/tests/test_dbt_build.py +++ b/tests/test_dbt_build.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtBuildOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_clean.py b/tests/test_dbt_clean.py index fdf53166..606fd066 100644 --- a/tests/test_dbt_clean.py +++ b/tests/test_dbt_clean.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCleanOperator.""" from unittest.mock import patch from airflow_dbt_python.operators.dbt import DbtCleanOperator, DbtCompileOperator diff --git a/tests/test_dbt_compile.py b/tests/test_dbt_compile.py index 24b427de..9eda40bd 100644 --- a/tests/test_dbt_compile.py +++ b/tests/test_dbt_compile.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtCompileOperator.""" from unittest.mock import patch from dbt.contracts.results import RunStatus diff --git a/tests/test_dbt_debug.py b/tests/test_dbt_debug.py index f23e60a2..9ef2e015 100644 --- a/tests/test_dbt_debug.py +++ b/tests/test_dbt_debug.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDebugOperator.""" from unittest.mock import patch from dbt.version import __version__ as DBT_VERSION diff --git a/tests/test_dbt_deps.py b/tests/test_dbt_deps.py index 88a0463f..f9ffc488 100644 --- a/tests/test_dbt_deps.py +++ b/tests/test_dbt_deps.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtDepsOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_list.py b/tests/test_dbt_list.py index c34e1e73..06bef0ab 100644 --- a/tests/test_dbt_list.py +++ b/tests/test_dbt_list.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtListOperator.""" from itertools import chain from unittest.mock import patch diff --git a/tests/test_dbt_parse.py b/tests/test_dbt_parse.py index a679d675..54b1db5d 100644 --- a/tests/test_dbt_parse.py +++ b/tests/test_dbt_parse.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtParseOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_run.py b/tests/test_dbt_run.py index 805559dd..1e131391 100644 --- a/tests/test_dbt_run.py +++ b/tests/test_dbt_run.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_run_operation.py b/tests/test_dbt_run_operation.py index 3d8b7f4f..908cc2ad 100644 --- a/tests/test_dbt_run_operation.py +++ b/tests/test_dbt_run_operation.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtRunOperationOperator.""" from unittest.mock import patch import pytest diff --git a/tests/test_dbt_s3_hook.py b/tests/test_dbt_s3_hook.py index 936be434..7322eb2c 100644 --- a/tests/test_dbt_s3_hook.py +++ b/tests/test_dbt_s3_hook.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtS3Hook.""" import pytest try: diff --git a/tests/test_dbt_seed.py b/tests/test_dbt_seed.py index c97ecf43..0f7a6ee1 100644 --- a/tests/test_dbt_seed.py +++ b/tests/test_dbt_seed.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSeedOperator.""" import json from unittest.mock import patch diff --git a/tests/test_dbt_snapshot.py b/tests/test_dbt_snapshot.py index 6bdd31f8..0eeb9f1c 100644 --- a/tests/test_dbt_snapshot.py +++ b/tests/test_dbt_snapshot.py @@ -1,9 +1,10 @@ +"""Unit test module for DbtSnapshotOperator.""" from unittest.mock import patch import pytest -from airflow import AirflowException from dbt.contracts.results import RunStatus +from airflow import AirflowException from airflow_dbt_python.operators.dbt import DbtSnapshotOperator diff --git a/tests/test_dbt_source.py b/tests/test_dbt_source.py index 3639b3d4..d1a3de83 100644 --- a/tests/test_dbt_source.py +++ b/tests/test_dbt_source.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtSourceOperator.""" from pathlib import Path from unittest.mock import patch diff --git a/tests/test_dbt_test.py b/tests/test_dbt_test.py index dbad6bb6..1c72a730 100644 --- a/tests/test_dbt_test.py +++ b/tests/test_dbt_test.py @@ -1,3 +1,4 @@ +"""Unit test module for DbtTestOperator.""" from unittest.mock import patch import pytest