From 4141f20fe86af6d08150768530013800a7506843 Mon Sep 17 00:00:00 2001
From: GitHub Action <tomas@tomasfarias.dev>
Date: Mon, 18 Oct 2021 19:16:46 +0200
Subject: [PATCH 1/2] docs: Add documentation for dbt operators

---
 airflow_dbt_python/operators/dbt.py | 96 ++++++++++++++++++++++++-----
 1 file changed, 81 insertions(+), 15 deletions(-)

diff --git a/airflow_dbt_python/operators/dbt.py b/airflow_dbt_python/operators/dbt.py
index 4486a5c2..36170fbc 100644
--- a/airflow_dbt_python/operators/dbt.py
+++ b/airflow_dbt_python/operators/dbt.py
@@ -95,7 +95,14 @@ def __init__(
         self._dbt_s3_hook = None
 
     def execute(self, context: dict):
-        """Execute dbt command with prepared arguments."""
+        """Execute dbt command with prepared arguments.
+
+        Execution requires setting up a directory with the dbt project files and
+        overriding the logging.
+
+        Args:
+            context: The Airflow's task context
+        """
         with self.dbt_directory() as dbt_dir:  # type: str
             with self.override_dbt_logging(dbt_dir):
                 args = self.prepare_args()
@@ -125,6 +132,9 @@ def execute(self, context: dict):
     def xcom_push_artifacts(self, context: dict, dbt_directory: str):
         """Read dbt artifacts and push them to XCom.
 
+        Artifacts are read from the target/ directory in dbt_directory. This method will
+        fail if the required artifact is not found.
+
         Args:
             context: The Airflow task's context.
             dbt_directory: A directory containing a dbt project. Artifacts will be
@@ -160,7 +170,11 @@ def prepare_args(self) -> list[Optional[str]]:
         return args
 
     def args_list(self) -> list[str]:
-        """Build a list of arguments to pass to dbt."""
+        """Build a list of arguments to pass to dbt.
+
+        Building involves creating a list of flags for dbt to parse given the operators
+        attributes and the values specified by __dbt_args__.
+        """
         args = []
         for arg in self.__dbt_args__:
             value = getattr(self, arg, None)
@@ -289,7 +303,12 @@ def serializable_result(
 
 
 class DbtRunOperator(DbtBaseOperator):
-    """Executes dbt run."""
+    """Executes a dbt run command.
+
+    The run command executes SQL model files against the given target. The
+    documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/run.
+    """
 
     command = "run"
 
@@ -337,7 +356,12 @@ def __init__(
 
 
 class DbtSeedOperator(DbtBaseOperator):
-    """Executes dbt seed."""
+    """Executes a dbt seed command.
+
+    The seed command loads csv files into the  the given target. The
+    documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/seed.
+    """
 
     command = "seed"
 
@@ -373,7 +397,12 @@ def __init__(
 
 
 class DbtTestOperator(DbtBaseOperator):
-    """Executes dbt test."""
+    """Executes a dbt test command.
+
+    The test command runs data and/or schema tests. The
+    documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/test.
+    """
 
     command = "test"
 
@@ -424,7 +453,12 @@ def __init__(
 
 
 class DbtCompileOperator(DbtBaseOperator):
-    """Executes dbt compile."""
+    """Executes a dbt compile command.
+
+    The compile command generates SQL files. The
+    documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/compile.
+    """
 
     command = "compile"
 
@@ -469,7 +503,11 @@ def __init__(
 
 
 class DbtDepsOperator(DbtBaseOperator):
-    """Executes dbt deps."""
+    """Executes a dbt deps command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/deps.
+    """
 
     command = "deps"
 
@@ -478,7 +516,11 @@ def __init__(self, **kwargs) -> None:
 
 
 class DbtCleanOperator(DbtBaseOperator):
-    """Executes dbt clean."""
+    """Executes a dbt clean command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/debug.
+    """
 
     command = "clean"
 
@@ -487,7 +529,11 @@ def __init__(self, **kwargs) -> None:
 
 
 class DbtDebugOperator(DbtBaseOperator):
-    """Execute dbt debug."""
+    """Executes a dbt debug command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/debug.
+    """
 
     command = "debug"
 
@@ -505,7 +551,11 @@ def __init__(
 
 
 class DbtSnapshotOperator(DbtBaseOperator):
-    """Execute dbt snapshot."""
+    """Executes a dbt snapshot command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/snapshot.
+    """
 
     command = "snapshot"
 
@@ -535,7 +585,11 @@ def __init__(
 
 
 class DbtLsOperator(DbtBaseOperator):
-    """Execute dbt list (or ls)."""
+    """Executes a dbt list (or ls) command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/list.
+    """
 
     command = "ls"
 
@@ -572,7 +626,11 @@ def __init__(
 
 
 class DbtRunOperationOperator(DbtBaseOperator):
-    """Execute dbt run-operation."""
+    """Executes a dbt run-operation command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/run-operation.
+    """
 
     command = "run-operation"
 
@@ -591,7 +649,11 @@ def __init__(
 
 
 class DbtParseOperator(DbtBaseOperator):
-    """Execute dbt parse."""
+    """Executes a dbt parse command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/parse.
+    """
 
     command = "parse"
 
@@ -603,7 +665,11 @@ def __init__(
 
 
 class DbtSourceOperator(DbtBaseOperator):
-    """Execute dbt source."""
+    """Executes a dbt source command.
+
+    The documentation for the dbt command can be found here:
+    https://docs.getdbt.com/reference/commands/source.
+    """
 
     command = "source"
 
@@ -640,7 +706,7 @@ def __init__(
 
 
 class DbtBuildOperator(DbtBaseOperator):
-    """Execute dbt build.
+    """Executes a dbt build command.
 
     The build command combines the run, test, seed, and snapshot commands into one. The
     full Documentation for the dbt build command can be found here:

From d71636edb3ced5e777105261bff8b240da230964 Mon Sep 17 00:00:00 2001
From: GitHub Action <tomas@tomasfarias.dev>
Date: Mon, 18 Oct 2021 19:48:59 +0200
Subject: [PATCH 2/2] docs: Update docstrings in all files except tests

All flake8-docstring errors have been cleared except for those coming
from test files since test files are not part of the documentation
pipeline. Eventually will work on them though just so that we can
eliminate the final ignore.
---
 .flake8                               | 7 ++-----
 .pre-commit-config.yaml               | 2 ++
 airflow_dbt_python/__version__.py     | 1 +
 airflow_dbt_python/hooks/dbt_s3.py    | 6 ++++++
 examples/__init__.py                  | 1 +
 examples/basic_dag.py                 | 4 +---
 examples/complete_dbt_workflow_dag.py | 5 +++--
 examples/dbt_project_in_s3_dag.py     | 4 +---
 examples/use_dbt_artifacts_dag.py     | 8 ++------
 tests/conftest.py                     | 5 +++++
 tests/test_dbt_base.py                | 1 +
 tests/test_dbt_build.py               | 1 +
 tests/test_dbt_clean.py               | 1 +
 tests/test_dbt_compile.py             | 1 +
 tests/test_dbt_debug.py               | 1 +
 tests/test_dbt_deps.py                | 1 +
 tests/test_dbt_list.py                | 1 +
 tests/test_dbt_parse.py               | 1 +
 tests/test_dbt_run.py                 | 1 +
 tests/test_dbt_run_operation.py       | 1 +
 tests/test_dbt_s3_hook.py             | 1 +
 tests/test_dbt_seed.py                | 1 +
 tests/test_dbt_snapshot.py            | 3 ++-
 tests/test_dbt_source.py              | 1 +
 tests/test_dbt_test.py                | 1 +
 25 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/.flake8 b/.flake8
index 3aa19a94..8e259722 100644
--- a/.flake8
+++ b/.flake8
@@ -1,12 +1,8 @@
 [flake8]
 max-line-length = 88
 docstring-convention = google
-# Will finish these later
-extend-ignore = D101,D107
+ignore = D107, W503
 per-file-ignores =
-    tests/*.py: D1,D2,D4
-    examples/*.py: D1,D2,D4
-    __version__.py: D100
     __init__.py: F401
 exclude =
     airflow
@@ -16,6 +12,7 @@ exclude =
     setup.py
     build
     dist
+    tests
     .venv
     .tox
     .mypy_cache
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 75341573..75a3dac6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,6 +10,8 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: [flake8-docstrings]
+        args: ["--config", ".flake8"]
+        exclude: tests/
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.902
diff --git a/airflow_dbt_python/__version__.py b/airflow_dbt_python/__version__.py
index 8201c2f7..c847a74e 100644
--- a/airflow_dbt_python/__version__.py
+++ b/airflow_dbt_python/__version__.py
@@ -1,3 +1,4 @@
+"""The module's version information."""
 __author__ = "Tomás Farías Santana"
 __copyright__ = "Copyright 2021 Tomás Farías Santana"
 __title__ = "airflow-dbt-python"
diff --git a/airflow_dbt_python/hooks/dbt_s3.py b/airflow_dbt_python/hooks/dbt_s3.py
index 75d694f9..37e46f15 100644
--- a/airflow_dbt_python/hooks/dbt_s3.py
+++ b/airflow_dbt_python/hooks/dbt_s3.py
@@ -7,6 +7,12 @@
 
 
 class DbtS3Hook(S3Hook):
+    """Subclass of S3Hook with methods to pull dbt-related files.
+
+    A dbt hook should provide a method to pull a dbt profiles file (profiles.yml) and
+    all the files corresponding to a project.
+    """
+
     def get_dbt_profiles(
         self, s3_profiles_url: str, profiles_dir: Optional[str] = None
     ) -> Path:
diff --git a/examples/__init__.py b/examples/__init__.py
index e69de29b..8f6e9364 100644
--- a/examples/__init__.py
+++ b/examples/__init__.py
@@ -0,0 +1 @@
+"""Usage examples of airflow-dbt-python."""
diff --git a/examples/basic_dag.py b/examples/basic_dag.py
index ce57e578..165e204b 100644
--- a/examples/basic_dag.py
+++ b/examples/basic_dag.py
@@ -1,6 +1,4 @@
-"""
-Sample basic DAG which dbt runs a project
-"""
+"""Sample basic DAG which dbt runs a project."""
 import datetime as dt
 
 from airflow import DAG
diff --git a/examples/complete_dbt_workflow_dag.py b/examples/complete_dbt_workflow_dag.py
index 74ff87c3..1f76d95d 100644
--- a/examples/complete_dbt_workflow_dag.py
+++ b/examples/complete_dbt_workflow_dag.py
@@ -1,5 +1,6 @@
-"""
-Sample DAG showcasing a complete dbt workflow
+"""Sample DAG showcasing a complete dbt workflow.
+
+The complete workflow includes a sequence of source, seed, and several run commands.
 """
 import datetime as dt
 
diff --git a/examples/dbt_project_in_s3_dag.py b/examples/dbt_project_in_s3_dag.py
index 039a41b8..7ed7eb73 100644
--- a/examples/dbt_project_in_s3_dag.py
+++ b/examples/dbt_project_in_s3_dag.py
@@ -1,6 +1,4 @@
-"""
-Sample basic DAG which showcases a dbt project being pulled from S3
-"""
+"""Sample basic DAG which showcases a dbt project being pulled from S3."""
 import datetime as dt
 
 from airflow import DAG
diff --git a/examples/use_dbt_artifacts_dag.py b/examples/use_dbt_artifacts_dag.py
index eeb3e877..8f2cc439 100644
--- a/examples/use_dbt_artifacts_dag.py
+++ b/examples/use_dbt_artifacts_dag.py
@@ -1,6 +1,4 @@
-"""
-Sample DAG to showcase pulling dbt artifacts from XCOM
-"""
+"""Sample DAG to showcase pulling dbt artifacts from XCOM."""
 import datetime as dt
 
 from airflow import DAG
@@ -10,9 +8,7 @@
 
 
 def process_dbt_artifacts(**context):
-    """
-    Report which model or models took the longest to compile and execute
-    """
+    """Report which model or models took the longest to compile and execute."""
     run_results = context["ti"].xcom_pull(
         key="run_results.json", task_ids="dbt_run_daily"
     )
diff --git a/tests/conftest.py b/tests/conftest.py
index fbfbb531..3d9e7039 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,3 +1,8 @@
+"""Conftest file including setting common fixtures.
+
+Common fixtures include a connection to a postgres database, a set of sample model and
+ seed files, dbt configuration files, and temporary directories for everything.
+"""
 import boto3
 import pytest
 from dbt.version import __version__ as DBT_VERSION
diff --git a/tests/test_dbt_base.py b/tests/test_dbt_base.py
index 456b84d9..52410d43 100644
--- a/tests/test_dbt_base.py
+++ b/tests/test_dbt_base.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtBaseOperator."""
 from pathlib import Path
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_build.py b/tests/test_dbt_build.py
index 90994a3c..f84e792c 100644
--- a/tests/test_dbt_build.py
+++ b/tests/test_dbt_build.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtBuildOperator."""
 import json
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_clean.py b/tests/test_dbt_clean.py
index fdf53166..606fd066 100644
--- a/tests/test_dbt_clean.py
+++ b/tests/test_dbt_clean.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtCleanOperator."""
 from unittest.mock import patch
 
 from airflow_dbt_python.operators.dbt import DbtCleanOperator, DbtCompileOperator
diff --git a/tests/test_dbt_compile.py b/tests/test_dbt_compile.py
index 24b427de..9eda40bd 100644
--- a/tests/test_dbt_compile.py
+++ b/tests/test_dbt_compile.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtCompileOperator."""
 from unittest.mock import patch
 
 from dbt.contracts.results import RunStatus
diff --git a/tests/test_dbt_debug.py b/tests/test_dbt_debug.py
index f23e60a2..9ef2e015 100644
--- a/tests/test_dbt_debug.py
+++ b/tests/test_dbt_debug.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtDebugOperator."""
 from unittest.mock import patch
 
 from dbt.version import __version__ as DBT_VERSION
diff --git a/tests/test_dbt_deps.py b/tests/test_dbt_deps.py
index 88a0463f..f9ffc488 100644
--- a/tests/test_dbt_deps.py
+++ b/tests/test_dbt_deps.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtDepsOperator."""
 from unittest.mock import patch
 
 import pytest
diff --git a/tests/test_dbt_list.py b/tests/test_dbt_list.py
index c34e1e73..06bef0ab 100644
--- a/tests/test_dbt_list.py
+++ b/tests/test_dbt_list.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtListOperator."""
 from itertools import chain
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_parse.py b/tests/test_dbt_parse.py
index a679d675..54b1db5d 100644
--- a/tests/test_dbt_parse.py
+++ b/tests/test_dbt_parse.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtParseOperator."""
 from pathlib import Path
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_run.py b/tests/test_dbt_run.py
index 805559dd..1e131391 100644
--- a/tests/test_dbt_run.py
+++ b/tests/test_dbt_run.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtRunOperator."""
 import json
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_run_operation.py b/tests/test_dbt_run_operation.py
index 3d8b7f4f..908cc2ad 100644
--- a/tests/test_dbt_run_operation.py
+++ b/tests/test_dbt_run_operation.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtRunOperationOperator."""
 from unittest.mock import patch
 
 import pytest
diff --git a/tests/test_dbt_s3_hook.py b/tests/test_dbt_s3_hook.py
index 936be434..7322eb2c 100644
--- a/tests/test_dbt_s3_hook.py
+++ b/tests/test_dbt_s3_hook.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtS3Hook."""
 import pytest
 
 try:
diff --git a/tests/test_dbt_seed.py b/tests/test_dbt_seed.py
index c97ecf43..0f7a6ee1 100644
--- a/tests/test_dbt_seed.py
+++ b/tests/test_dbt_seed.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtSeedOperator."""
 import json
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_snapshot.py b/tests/test_dbt_snapshot.py
index 6bdd31f8..0eeb9f1c 100644
--- a/tests/test_dbt_snapshot.py
+++ b/tests/test_dbt_snapshot.py
@@ -1,9 +1,10 @@
+"""Unit test module for DbtSnapshotOperator."""
 from unittest.mock import patch
 
 import pytest
-from airflow import AirflowException
 from dbt.contracts.results import RunStatus
 
+from airflow import AirflowException
 from airflow_dbt_python.operators.dbt import DbtSnapshotOperator
 
 
diff --git a/tests/test_dbt_source.py b/tests/test_dbt_source.py
index 3639b3d4..d1a3de83 100644
--- a/tests/test_dbt_source.py
+++ b/tests/test_dbt_source.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtSourceOperator."""
 from pathlib import Path
 from unittest.mock import patch
 
diff --git a/tests/test_dbt_test.py b/tests/test_dbt_test.py
index dbad6bb6..1c72a730 100644
--- a/tests/test_dbt_test.py
+++ b/tests/test_dbt_test.py
@@ -1,3 +1,4 @@
+"""Unit test module for DbtTestOperator."""
 from unittest.mock import patch
 
 import pytest