Skip to content

Commit

Permalink
test: Add dagbag tests
Browse files Browse the repository at this point in the history
  • Loading branch information
1ambda committed Sep 4, 2023
1 parent 204e161 commit 08b3c43
Show file tree
Hide file tree
Showing 10 changed files with 193 additions and 84 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,7 @@ jobs:
SKIP: sqlfluff-fix,sqlfluff-lint
with:
extra_args: --color=always --show-diff-on-failure

- name: pytest
run: |
make test;
118 changes: 64 additions & 54 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ repos:
hooks:
- id: ruff
name: 'python ruff'
files: ^(dags|dbts|notebook)/
files: ^(dags|dbts|dags-test)/
args: [ --fix, --exit-non-zero-on-fix ]
types_or: [ python, pyi, jupyter ]

Expand All @@ -64,7 +64,7 @@ repos:
- id: mypy
name: 'python type check'
verbose: true
files: ^(dags|dbts)/
files: ^(dags|dbts|dags-test)/
types: [ python ]
args: [ "--config-file", "./mypy.ini", ]
additional_dependencies:
Expand All @@ -84,58 +84,68 @@ repos:
- pytest
- hypothesis

# - repo: https://github.com/psf/black
# rev: 23.7.0
# hooks:
# - id: black
# name: 'python black'
# verbose: true
# files: ^(dags|dbts)/
# types: [ python ]
# args: [ --line-length=120 ]
#
# - id: black-jupyter
# name: 'jupyter black'
# verbose: true
# files: ^(notebook)/
#
# - repo: https://github.com/PyCQA/autoflake
# rev: v2.2.0
# hooks:
# - id: autoflake
# name: 'python autoflake'
# args: [
# --in-place,
# --remove-all-unused-imports,
# --remove-unused-variables,
# ]
#
# - repo: https://github.com/pycqa/flake8
# rev: 6.1.0
# hooks:
# - id: flake8
# name: 'python flake8'
# verbose: true
# files: ^(dags|dbts)/
# types: [ python ]
# additional_dependencies:
# - flake8-blind-except
# - flake8-docstrings
# - flake8-bugbear
# - flake8-comprehensions
# - flake8-docstrings
# - flake8-implicit-str-concat
# - flake8-simplify
# - Flake8-pyproject
# - pydocstyle>=5.0.0
#
# - repo: https://github.com/pycqa/isort
# rev: 5.12.0
# hooks:
# - id: isort
# name: 'python isort'
# verbose: true
# files: ^(dags|dbts)/
- repo: local
hooks:
- id: pytest
name: 'python pytest'
entry: make test
language: system
pass_filenames: false
types: [ python ]
stages: [ commit, push ]

# - repo: https://github.com/psf/black
# rev: 23.7.0
# hooks:
# - id: black
# name: 'python black'
# verbose: true
# files: ^(dags|dbts)/
# types: [ python ]
# args: [ --line-length=120 ]
#
# - id: black-jupyter
# name: 'jupyter black'
# verbose: true
# files: ^(notebook)/
#
# - repo: https://github.com/PyCQA/autoflake
# rev: v2.2.0
# hooks:
# - id: autoflake
# name: 'python autoflake'
# args: [
# --in-place,
# --remove-all-unused-imports,
# --remove-unused-variables,
# ]
#
# - repo: https://github.com/pycqa/flake8
# rev: 6.1.0
# hooks:
# - id: flake8
# name: 'python flake8'
# verbose: true
# files: ^(dags|dbts)/
# types: [ python ]
# additional_dependencies:
# - flake8-blind-except
# - flake8-docstrings
# - flake8-bugbear
# - flake8-comprehensions
# - flake8-docstrings
# - flake8-implicit-str-concat
# - flake8-simplify
# - Flake8-pyproject
# - pydocstyle>=5.0.0
#
# - repo: https://github.com/pycqa/isort
# rev: 5.12.0
# hooks:
# - id: isort
# name: 'python isort'
# verbose: true
# files: ^(dags|dbts)/

####################################################################################################
# GIT
Expand Down
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,15 @@ compose.trino:
compose.dbt:
COMPOSE_PROFILES=trino,airflow docker-compose up

.PHONY:test
test:
@ echo ""
@ echo ""
@ echo "[$(TAG)] ($(shell date '+%H:%M:%S')) - Executing pytest"
@ AIRFLOW_HOME=$(shell pwd) poetry run pytest dags-test/
@ echo ""
@ echo ""

.PHONY:check
check:
@ echo ""
Expand Down
24 changes: 24 additions & 0 deletions dags-test/test_dag_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import os
from pathlib import Path

from airflow.models import DagBag

CONST_DAG_DIRECTORY = "dags"

CONST_AIRFLOW_HOME = os.environ.get("AIRFLOW_HOME", Path(__file__).parent.parent)


def test_no_import_errors() -> None:
dag_folder = f"{CONST_AIRFLOW_HOME}/{CONST_DAG_DIRECTORY}"
dag_bag = DagBag(dag_folder=dag_folder, include_examples=False)
assert len(dag_bag.import_errors) == 0, "No Import Failures"



def test_retries_present() -> None:
dag_folder = f"{CONST_AIRFLOW_HOME}/{CONST_DAG_DIRECTORY}"
dag_bag = DagBag(dag_folder=dag_folder, include_examples=False)
for dag in dag_bag.dags:
retries = dag_bag.dags[dag].default_args.get("retries", [])
error_msg = f"Retries are set for DAG {dag}"
assert retries >= 1, error_msg
4 changes: 2 additions & 2 deletions dags/config/dag.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
from typing import Any, Optional, Dict
from typing import Any

CONST_DEFAULT_ARGS = {
"owner": "airflow",
"retries": 1,
}


def build_default_args(custom_args: Optional[Dict[Any, Any]] = None) -> Dict[Any, Any]:
def build_default_args(custom_args: dict[Any, Any] | None = None) -> dict[Any, Any]:
actual_args = CONST_DEFAULT_ARGS.copy()

if custom_args:
Expand Down
52 changes: 26 additions & 26 deletions dags/dag_dbt.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,50 +8,50 @@

import pendulum
from airflow.decorators import dag

from airflow_dbt.operators.dbt_operator import (
DbtRunOperator,
DbtDepsOperator,
DbtDepsOperator,
DbtRunOperator,
)
from config.dag import build_default_args

import config.dag

dag_args = config.dag.build_default_args()
dag_args = build_default_args()


@dag(
dag_id=Path(__file__).stem,
tags=["dbt"],
description=__doc__[0 : __doc__.find(".")],
description=__doc__[0: __doc__.find(".")],
doc_md=__doc__,
default_args=dag_args,
start_date=pendulum.datetime(2023, 9, 1, tz="Asia/Seoul"),
schedule='5 0 * * *',
schedule="5 0 * * *",
catchup=False,
dagrun_timeout=dt.timedelta(minutes=60),
)
def generate_dag() -> None:
dbt_home = "/opt/airflow/dbts"
dbt_target = "dev"

dbt_home = "/opt/airflow/dbts"
dbt_target = "dev"
task_dbt_deps = DbtDepsOperator(
task_id="task_dbt_deps",
dir=dbt_home,
profiles_dir=dbt_home,
target=dbt_target,
full_refresh=False,
)

task_dbt_deps = DbtDepsOperator(
task_id="task_dbt_deps",
dir=dbt_home,
profiles_dir=dbt_home,
target=dbt_target,
full_refresh=False,
)
task_dbt_run = DbtRunOperator(
task_id="task_dbt_run",
dir=dbt_home,
profiles_dir=dbt_home,
target=dbt_target,
full_refresh=False,
)

task_dbt_run = DbtRunOperator(
task_id="task_dbt_run",
dir=dbt_home,
profiles_dir=dbt_home,
target=dbt_target,
full_refresh=False,
)

task_dbt_deps >> task_dbt_run
task_dbt_deps >> task_dbt_run


generate_dag()

if __name__ == "__main__":
dag.test()
7 changes: 6 additions & 1 deletion mypy.ini
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
[mypy]

mypy_path=./dags:./dbts
python_version=3.11
mypy_path=./dags:./dbts:./dags-test

strict=True
ignore_missing_imports=True
disallow_untyped_decorators=False
show_error_codes = True
show_error_context = True
show_column_numbers = True
warn_no_return = false
2 changes: 1 addition & 1 deletion poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

50 changes: 50 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,57 @@ sqlfluff = "^2.3.1"
sqlfluff-templater-dbt = "^2.3.1"
ruff = "^0.0.287"
black = "^23.7.0"
pytest = "^7.4.1"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"


[tool.ruff]
line-length = 100 # defaults to 88 like black
target-version = "py311" # the python version to target, useful when considering code upgrades, defaults to "py310"
fix = true
show-source = true

format = "grouped"
select = [
# pycodestyle
"E",
"W",
# pyflakes
"F",
# pylint
"PLC",
"PLE",
"PLW",
# isort
"I",
# flake8-builtins
"A",
# flake8-bugbear
"B",
# flake8-commas
"COM",
# flake8-comprehensions
"C4",
# flake8-datetimez
"DTZ",
# flake8-implicit-str-concat
"ISC",
# flake8-pie
"PIE",
# flake8-pytest-style
"PT",
# flake8-quotes
"Q",
# flake8-tidy-imports
"TID",
# explicitly select rules under the nursery-flag introduced in ruff 0.0.269
# remove once enabled via the "E" selector
"E111", "E112", "E113", "E114", "E115", "E116", "E117",
"E201", "E202", "E203", "E211", "E221", "E222", "E223",
"E224", "E225", "E226", "E227", "E228", "E231", "E251",
"E252", "E261", "E262", "E265", "E266", "E271", "E272",
"E273", "E274", "E275",
]
7 changes: 7 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[pytest]
filterwarnings =
ignore::DeprecationWarning

log_cli=true
log_level=INFO
log_cli_level=INFO

0 comments on commit 08b3c43

Please sign in to comment.