From 0db33afa07fa3c388397c12d794d1bafbb069589 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Thu, 16 May 2024 15:05:56 -0400 Subject: [PATCH] feat(dbt): support `dbt-core==1.8.*` --- .../dagster_buildkite/steps/packages.py | 2 +- pyright/alt-1/requirements-pinned.txt | 68 +++++---- pyright/master/requirements-pinned.txt | 142 +++++++++--------- .../dagster_dbt/core/resources_v2.py | 30 +++- .../dagster-dbt/dagster_dbt/utils.py | 5 +- .../dagster-dbt/dagster_dbt/version.py | 2 +- .../core/test_resources_v2.py | 8 +- .../legacy/test_resources.py | 8 +- python_modules/libraries/dagster-dbt/tox.ini | 2 + 9 files changed, 143 insertions(+), 124 deletions(-) diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index 8b6d942640cf6..aa77d49d1dac5 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -476,7 +476,7 @@ def tox_factors_for_folder(tests_folder_name: str) -> List[str]: "python_modules/libraries/dagster-dbt", pytest_tox_factors=[ f"{deps_factor}-{command_factor}" - for deps_factor in ["dbt16", "dbt17", "pydantic1"] + for deps_factor in ["dbt16", "dbt17", "dbt18", "pydantic1"] for command_factor in ["cloud", "core", "legacy"] ], unsupported_python_versions=[ diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt index e0ec964ad212b..d6dd9746157d2 100644 --- a/pyright/alt-1/requirements-pinned.txt +++ b/pyright/alt-1/requirements-pinned.txt @@ -1,5 +1,5 @@ -agate==1.7.1 -aiobotocore==2.12.3 +agate==1.9.1 +aiobotocore==2.12.4 aiofile==3.8.8 aiohttp==3.9.5 aioitertools==0.11.0 @@ -14,7 +14,7 @@ argon2-cffi-bindings==21.2.0 arrow==1.3.0 asn1crypto==1.5.1 -e examples/assets_pandas_type_metadata -astroid==3.1.0 +astroid==3.2.1 asttokens==2.4.1 async-lru==2.0.4 async-timeout==4.0.3 @@ -23,8 +23,8 @@ babel==2.15.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 -boto3==1.34.69 -botocore==1.34.69 +boto3==1.34.106 +botocore==1.34.106 buildkite-test-collector==0.1.7 cachetools==5.3.3 caio==0.9.13 @@ -40,6 +40,7 @@ coverage==7.5.1 croniter==2.0.5 cryptography==42.0.7 cycler==0.12.1 +daff==1.3.46 -e python_modules/dagster -e python_modules/libraries/dagster-aws -e python_modules/libraries/dagster-dbt @@ -61,17 +62,19 @@ cycler==0.12.1 -e python_modules/libraries/dagster-spark -e python_modules/dagster-webserver db-dtypes==1.2.0 -dbt-core==1.7.14 -dbt-duckdb==1.7.4 +dbt-adapters==1.1.1 +dbt-common==1.0.4 +dbt-core==1.8.0 +dbt-duckdb==1.8.0 dbt-extractor==0.5.1 -dbt-semantic-interfaces==0.4.4 -dbt-snowflake==1.7.3 +dbt-semantic-interfaces==0.5.1 +dbt-snowflake==1.8.1 debugpy==1.8.1 decopatch==1.4.10 decorator==5.1.1 deepdiff==7.0.1 defusedxml==0.7.1 -deltalake==0.17.2 +deltalake==0.17.4 dill==0.3.8 distlib==0.3.8 docker==7.0.0 @@ -88,11 +91,11 @@ frozenlist==1.4.1 fsspec==2024.3.1 gcsfs==2024.3.1 google-api-core==2.19.0 -google-api-python-client==2.127.0 +google-api-python-client==2.129.0 google-auth==2.29.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 -google-cloud-bigquery==3.22.0 +google-cloud-bigquery==3.23.0 google-cloud-core==2.4.1 google-cloud-storage==2.16.0 google-crc32c==1.5.0 @@ -112,7 +115,7 @@ httplib2==0.22.0 httptools==0.6.1 httpx==0.27.0 humanfriendly==10.0 -hypothesis==6.100.4 +hypothesis==6.102.4 idna==3.7 importlib-metadata==6.11.0 iniconfig==2.0.0 @@ -144,11 +147,11 @@ kiwisolver==1.4.5 leather==0.4.0 logbook==1.5.3 makefun==1.15.2 -mako==1.3.3 +mako==1.3.5 markdown-it-py==3.0.0 markupsafe==2.1.5 mashumaro==3.13 -matplotlib==3.8.4 +matplotlib==3.9.0 matplotlib-inline==0.1.7 mccabe==0.7.0 mdurl==0.1.2 @@ -179,19 +182,19 @@ orjson==3.10.3 overrides==7.7.0 packaging==24.0 pandas==2.2.2 -pandas-stubs==2.2.1.240316 -pandera==0.19.0 +pandas-stubs==2.2.2.240514 +pandera==0.19.3 pandocfilters==1.5.1 parsedatetime==2.6 parso==0.8.4 -pathspec==0.11.2 +pathspec==0.12.1 pendulum==3.0.0 pexpect==4.9.0 pillow==10.3.0 pip==24.0 -platformdirs==4.2.1 +platformdirs==4.2.2 pluggy==1.5.0 -polars==0.20.23 +polars==0.20.26 -e examples/project_fully_featured prometheus-client==0.20.0 prompt-toolkit==3.0.43 @@ -203,7 +206,7 @@ ptyprocess==0.7.0 pure-eval==0.2.2 py==1.11.0 py4j==0.10.9.7 -pyarrow==16.0.0 +pyarrow==16.1.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1-modules==0.4.0 @@ -212,7 +215,7 @@ pydantic==2.7.1 pydantic-core==2.18.2 pygments==2.18.0 pyjwt==2.8.0 -pylint==3.1.0 +pylint==3.2.0 pyopenssl==24.1.0 pyparsing==3.1.2 pyright==1.1.356 @@ -251,15 +254,15 @@ send2trash==1.8.3 setuptools==69.5.1 shellingham==1.5.4 six==1.16.0 -slack-sdk==3.27.1 +slack-sdk==3.27.2 sniffio==1.3.1 snowflake-connector-python==3.10.0 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 sqlalchemy==1.4.52 -sqlglot==23.13.7 -sqlglotrs==0.2.0 +sqlglot==23.15.8 +sqlglotrs==0.2.5 sqlparse==0.5.0 stack-data==0.6.3 starlette==0.37.2 @@ -273,7 +276,7 @@ time-machine==2.14.1 tinycss2==1.3.0 toml==0.10.2 tomli==2.0.1 -tomlkit==0.12.4 +tomlkit==0.12.5 toposort==1.10 tornado==6.4 tox==3.25.0 @@ -295,25 +298,24 @@ types-pyopenssl==24.1.0.20240425 types-python-dateutil==2.9.0.20240316 types-pytz==2024.1.0.20240417 types-pyyaml==6.0.12.20240311 -types-requests==2.31.0.6 -types-setuptools==69.5.0.20240423 +types-requests==2.31.0.20240406 +types-setuptools==69.5.0.20240513 types-simplejson==3.19.0.20240310 -types-six==1.16.21.20240425 +types-six==1.16.21.20240513 types-sqlalchemy==1.4.53.34 types-tabulate==0.9.0.20240106 types-toml==0.10.8.20240310 types-tzlocal==5.1.0.1 -types-urllib3==1.26.25.14 typing-extensions==4.11.0 typing-inspect==0.9.0 tzdata==2024.1 universal-pathlib==0.2.2 uri-template==1.3.0 uritemplate==4.1.1 -urllib3==1.26.18 +urllib3==2.2.1 uvicorn==0.29.0 uvloop==0.19.0 -virtualenv==20.26.1 +virtualenv==20.26.2 watchdog==4.0.0 watchfiles==0.21.0 wcwidth==0.2.13 @@ -324,4 +326,4 @@ websockets==12.0 wheel==0.43.0 wrapt==1.16.0 yarl==1.9.4 -zipp==3.18.1 +zipp==3.18.2 diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index 790be83ca6351..a5a4d60128c51 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -1,5 +1,5 @@ -acryl-datahub==0.13.1.3 -agate==1.7.1 +acryl-datahub==0.13.2.3 +agate==1.9.1 aiofile==3.8.8 aiofiles==23.2.1 aiohttp==3.9.5 @@ -13,18 +13,17 @@ aniso8601==9.0.1 ansicolors==1.1.8 anyio==4.3.0 apache-airflow==2.7.3 -apache-airflow-providers-apache-spark==4.7.2 -apache-airflow-providers-cncf-kubernetes==8.1.1 -apache-airflow-providers-common-sql==1.12.0 +apache-airflow-providers-apache-spark==4.8.0 +apache-airflow-providers-cncf-kubernetes==8.2.0 +apache-airflow-providers-common-sql==1.13.0 apache-airflow-providers-docker==3.9.1 -apache-airflow-providers-ftp==3.8.0 +apache-airflow-providers-ftp==3.9.0 apache-airflow-providers-http==4.1.0 -apache-airflow-providers-imap==3.5.0 -apache-airflow-providers-sqlite==3.7.1 +apache-airflow-providers-imap==3.6.0 +apache-airflow-providers-sqlite==3.8.0 apeye==1.4.1 apeye-core==1.1.5 apispec==6.6.1 -appdirs==1.4.4 appnope==0.1.4 argcomplete==3.3.0 argon2-cffi==23.1.0 @@ -44,23 +43,23 @@ autoflake==2.3.1 -e python_modules/automation avro==1.11.3 avro-gen3==0.7.12 -aws-sam-translator==1.87.0 +aws-sam-translator==1.88.0 aws-xray-sdk==2.13.0 azure-core==1.30.1 azure-identity==1.16.0 -azure-storage-blob==12.19.1 -azure-storage-file-datalake==12.14.0 -babel==2.14.0 +azure-storage-blob==12.20.0 +azure-storage-file-datalake==12.15.0 +babel==2.15.0 backoff==2.2.1 -bcrypt==4.1.2 +bcrypt==4.1.3 beautifulsoup4==4.12.3 billiard==4.2.0 bitmath==1.3.3.1 bleach==6.1.0 -blinker==1.8.1 +blinker==1.8.2 bokeh==3.4.1 -boto3==1.34.97 -botocore==1.34.97 +boto3==1.34.106 +botocore==1.34.106 buildkite-test-collector==0.1.7 cachecontrol==0.14.0 cached-property==1.5.2 @@ -72,7 +71,7 @@ cattrs==23.2.3 celery==5.4.0 certifi==2024.2.2 cffi==1.16.0 -cfn-lint==0.87.1 +cfn-lint==0.87.2 chardet==5.2.0 charset-normalizer==3.3.2 click==8.1.7 @@ -88,15 +87,16 @@ coloredlogs==14.0 colorlog==4.8.0 comm==0.2.2 configupdater==3.2 -confluent-kafka==2.3.0 +confluent-kafka==2.4.0 connexion==2.14.2 contourpy==1.2.1 -coverage==7.5.0 +coverage==7.5.1 cron-descriptor==1.4.3 croniter==2.0.5 -cryptography==42.0.5 -cssutils==2.10.2 +cryptography==42.0.7 +cssutils==2.11.0 cycler==0.12.1 +daff==1.3.46 -e python_modules/dagit -e python_modules/dagster -e python_modules/libraries/dagster-airbyte @@ -160,35 +160,37 @@ cycler==0.12.1 -e python_modules/libraries/dagster-wandb -e python_modules/dagster-webserver -e python_modules/libraries/dagstermill -dask==2024.4.2 -dask-expr==1.0.14 +dask==2024.5.0 +dask-expr==1.1.0 dask-jobqueue==0.8.5 dask-kubernetes==2022.9.0 dask-yarn==0.9 databricks-api==0.9.0 databricks-cli==0.18.0 databricks-sdk==0.8.0 -dataclasses-json==0.6.5 +dataclasses-json==0.6.6 datadog==0.49.1 dataproperty==1.0.1 db-dtypes==1.2.0 -dbt-core==1.7.14 -dbt-duckdb==1.7.4 +dbt-adapters==1.1.1 +dbt-common==1.0.4 +dbt-core==1.8.0 +dbt-duckdb==1.8.0 dbt-extractor==0.5.1 -dbt-semantic-interfaces==0.4.4 +dbt-semantic-interfaces==0.5.1 debugpy==1.8.1 decopatch==1.4.10 decorator==5.1.1 deepdiff==7.0.1 defusedxml==0.7.1 -deltalake==0.17.2 +deltalake==0.17.4 deprecated==1.2.14 -e examples/development_to_production dict2css==0.3.0.post1 diff-match-patch==20200713 dill==0.3.8 distlib==0.3.8 -distributed==2024.4.2 +distributed==2024.5.0 distro==1.9.0 dlt==0.4.3 dnspython==2.6.1 @@ -215,8 +217,8 @@ filelock==3.14.0 flask==2.2.5 flask-appbuilder==4.3.6 flask-babel==2.0.0 -flask-caching==2.2.0 -flask-cors==4.0.0 +flask-caching==2.3.0 +flask-cors==4.0.1 flask-jwt-extended==4.6.0 flask-limiter==3.6.0 flask-login==0.6.3 @@ -232,11 +234,11 @@ gitdb==4.0.11 gitpython==3.1.43 giturlparse==0.12.0 google-api-core==2.19.0 -google-api-python-client==2.127.0 +google-api-python-client==2.129.0 google-auth==2.29.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.0 -google-cloud-bigquery==3.21.0 +google-cloud-bigquery==3.23.0 google-cloud-core==2.4.1 google-cloud-storage==2.16.0 google-crc32c==1.5.0 @@ -263,7 +265,7 @@ httptools==0.6.1 httpx==0.27.0 humanfriendly==10.0 humanize==4.9.0 -hypothesis==6.100.2 +hypothesis==6.102.4 idna==3.7 ijson==3.2.3 imagesize==1.4.1 @@ -280,7 +282,7 @@ isodate==0.6.1 isoduration==20.11.0 itsdangerous==2.2.0 jedi==0.19.1 -jinja2==3.1.3 +jinja2==3.1.4 jmespath==1.0.1 joblib==1.4.2 jschema-to-python==1.2.3 @@ -301,7 +303,7 @@ jupyter-events==0.10.0 jupyter-lsp==2.2.5 jupyter-server==2.14.0 jupyter-server-terminals==0.5.3 -jupyterlab==4.1.8 +jupyterlab==4.2.0 jupyterlab-pygments==0.3.0 jupyterlab-server==2.27.1 jupyterlab-widgets==3.0.10 @@ -313,20 +315,20 @@ kubernetes==29.0.0 kubernetes-asyncio==29.0.0 langchain==0.1.11 langchain-community==0.0.34 -langchain-core==0.1.50 +langchain-core==0.1.52 langchain-openai==0.1.3 -langchain-text-splitters==0.0.1 -langsmith==0.1.53 +langchain-text-splitters==0.0.2 +langsmith==0.1.59 lazy-object-proxy==1.10.0 leather==0.4.0 -limits==3.11.0 +limits==3.12.0 linkify-it-py==2.0.3 lkml==1.3.4 locket==1.0.0 lockfile==0.12.2 logbook==1.5.3 makefun==1.15.2 -mako==1.3.3 +mako==1.3.5 markdown==3.6 markdown-it-py==3.0.0 markupsafe==2.1.5 @@ -334,10 +336,10 @@ marshmallow==3.21.2 marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 mashumaro==3.13 -matplotlib==3.8.4 +matplotlib==3.9.0 matplotlib-inline==0.1.3 mbstrdecoder==1.1.3 -mdit-py-plugins==0.4.0 +mdit-py-plugins==0.4.1 mdurl==0.1.2 minimal-snowplow-tracker==0.0.2 mistune==3.0.2 @@ -365,7 +367,7 @@ networkx==3.3 nh3==0.2.17 nodeenv==1.8.0 noteable-origami==0.0.35 -notebook==7.1.3 +notebook==7.2.0 notebook-shim==0.2.4 numpy==1.26.4 oauth2client==4.1.3 @@ -374,7 +376,7 @@ objgraph==3.6.1 onnx==1.16.0 onnxconverter-common==1.13.0 onnxruntime==1.17.3 -openai==1.25.1 +openai==1.30.1 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 opentelemetry-api==1.24.0 @@ -390,20 +392,20 @@ orjson==3.10.3 outcome==1.3.0.post0 overrides==7.7.0 packaging==23.2 -pandas==2.0.3 +pandas==2.2.2 pandas-gbq==0.22.0 -pandas-stubs==2.2.1.240316 -pandera==0.18.3 +pandas-stubs==2.2.2.240514 +pandera==0.19.3 pandocfilters==1.5.1 papermill==2.6.0 papermill-origami==0.0.30 paramiko==3.4.0 parsedatetime==2.6 parso==0.8.4 -partd==1.4.1 +partd==1.4.2 path==16.14.0 pathable==0.4.3 -pathspec==0.11.2 +pathspec==0.12.1 pathvalidate==3.2.0 pbr==6.0.0 pendulum==2.1.2 @@ -411,11 +413,11 @@ pexpect==4.9.0 pillow==10.3.0 pip==24.0 pkginfo==1.10.0 -platformdirs==4.2.1 +platformdirs==4.2.2 plotly==5.22.0 pluggy==1.5.0 ply==3.11 -polars==0.20.23 +polars==0.20.26 portalocker==2.8.2 prison==0.2.1 progressbar2==4.4.2 @@ -431,7 +433,7 @@ pure-eval==0.2.2 py==1.11.0 py-partiql-parser==0.5.0 py4j==0.10.9.7 -pyarrow==16.0.0 +pyarrow==16.1.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1-modules==0.4.0 @@ -439,7 +441,7 @@ pycparser==2.22 pydantic==1.10.15 pydata-google-auth==1.8.2 pyflakes==3.2.0 -pygments==2.17.2 +pygments==2.18.0 pyjwt==2.8.0 pynacl==1.5.0 pyopenssl==24.1.0 @@ -473,7 +475,7 @@ querystring-parser==1.2.4 rapidfuzz==3.9.0 readme-renderer==43.0 referencing==0.31.1 -regex==2024.4.28 +regex==2024.5.15 requests==2.31.0 requests-file==2.0.0 requests-mock==1.12.1 @@ -485,7 +487,7 @@ rfc3339-validator==0.1.4 rfc3986-validator==0.1.1 rich==13.7.1 rich-argparse==1.4.0 -rpds-py==0.18.0 +rpds-py==0.18.1 rsa==4.9 ruamel-yaml==0.17.17 s3transfer==0.10.1 @@ -494,11 +496,11 @@ scikit-learn==1.4.2 scipy==1.13.0 scrapbook==0.5.0 seaborn==0.13.2 -selenium==4.20.0 +selenium==4.21.0 semver==3.0.2 send2trash==1.8.3 sending==0.3.0 -sentry-sdk==2.0.1 +sentry-sdk==2.2.0 setproctitle==1.3.3 setuptools==69.5.1 shellingham==1.5.4 @@ -506,7 +508,7 @@ simplejson==3.19.2 six==1.16.0 skein==0.8.2 skl2onnx==1.16.0 -slack-sdk==3.27.1 +slack-sdk==3.27.2 sling==1.2.9 sling-mac-arm64==1.2.9 smmap==5.0.1 @@ -531,8 +533,8 @@ sphinxcontrib-serializinghtml==1.1.10 sqlalchemy==1.4.52 sqlalchemy-jsonfield==1.0.2 sqlalchemy-utils==0.41.2 -sqlglot==23.12.2 -sqlglotrs==0.2.0 +sqlglot==23.15.8 +sqlglotrs==0.2.5 sqlparse==0.5.0 sshpubkeys==3.3.1 sshtunnel==0.4.0 @@ -544,16 +546,16 @@ syrupy==4.6.1 tabledata==1.3.3 tabulate==0.9.0 tblib==3.0.0 -tenacity==8.2.3 +tenacity==8.3.0 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 threadpoolctl==3.5.0 -tiktoken==0.6.0 +tiktoken==0.7.0 tinycss2==1.3.0 toml==0.10.2 tomli==2.0.1 -tomlkit==0.12.4 +tomlkit==0.12.5 toolz==0.12.1 toposort==1.10 torch==2.3.0 @@ -562,7 +564,7 @@ tornado==6.4 tox==3.25.0 tqdm==4.66.4 traitlets==5.14.3 -trio==0.25.0 +trio==0.25.1 trio-websocket==0.11.1 -e examples/tutorial_notebook_assets twilio==9.0.5 @@ -585,9 +587,9 @@ types-python-dateutil==2.9.0.20240316 types-pytz==2024.1.0.20240417 types-pyyaml==6.0.12.20240311 types-requests==2.31.0.6 -types-setuptools==69.5.0.20240423 +types-setuptools==69.5.0.20240513 types-simplejson==3.19.0.20240310 -types-six==1.16.21.20240425 +types-six==1.16.21.20240513 types-sqlalchemy==1.4.53.34 types-tabulate==0.9.0.20240106 types-toml==0.10.8.20240310 @@ -607,7 +609,7 @@ uvicorn==0.29.0 uvloop==0.19.0 vine==5.1.0 virtualenv==20.25.0 -wandb==0.16.6 +wandb==0.17.0 watchdog==4.0.0 watchfiles==0.21.0 wcwidth==0.2.13 @@ -632,4 +634,4 @@ xmltodict==0.12.0 xyzservices==2024.4.0 yarl==1.9.4 zict==3.0.0 -zipp==3.18.1 +zipp==3.18.2 diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py index dc858a8b125a2..b571f8b790d8d 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resources_v2.py @@ -59,7 +59,6 @@ from dbt.config import RuntimeConfig from dbt.config.runtime import load_profile, load_project from dbt.contracts.results import NodeStatus, TestStatus -from dbt.events.functions import cleanup_event_logger from dbt.flags import get_flags, set_from_args from dbt.node_types import NodeType from dbt.version import __version__ as dbt_version @@ -101,6 +100,16 @@ ) from .utils import get_future_completion_state_or_err +IS_DBT_CORE_VERSION_LESS_THAN_1_8_0 = version.parse(dbt_version) < version.parse("1.8.0") + +if IS_DBT_CORE_VERSION_LESS_THAN_1_8_0: + from dbt.events.functions import cleanup_event_logger # type: ignore + + REFABLE_NODE_TYPES = NodeType.refable() # type: ignore +else: + from dbt.node_types import REFABLE_NODE_TYPES as REFABLE_NODE_TYPES + from dbt_common.events.event_manager_client import cleanup_event_logger + logger = get_dagster_logger() @@ -256,7 +265,7 @@ def to_default_asset_events( is_node_successful = node_status == NodeStatus.Success is_node_finished = bool(event_node_info.get("node_finished_at")) if ( - node_resource_type in NodeType.refable() + node_resource_type in REFABLE_NODE_TYPES and is_node_successful and not is_node_ephemeral ): @@ -1304,7 +1313,12 @@ def _get_unique_target_path(self, *, context: Optional[OpExecutionContext]) -> P return current_target_path.joinpath(path) - def _initialize_adapter(self, args: Sequence[str]) -> BaseAdapter: + def _initialize_adapter(self) -> BaseAdapter: + if not IS_DBT_CORE_VERSION_LESS_THAN_1_8_0: + from dbt_common.context import set_invocation_context + + set_invocation_context(os.environ.copy()) + # constructs a dummy set of flags, using the `run` command (ensures profile/project reqs get loaded) profiles_dir = self.profiles_dir if self.profiles_dir else self.project_dir set_from_args(Namespace(profiles_dir=profiles_dir), None) @@ -1327,7 +1341,13 @@ def _initialize_adapter(self, args: Sequence[str]) -> BaseAdapter: pass cleanup_event_logger() - register_adapter(config) + if IS_DBT_CORE_VERSION_LESS_THAN_1_8_0: + register_adapter(config) # type: ignore + else: + from dbt.mp_context import get_mp_context + + register_adapter(config, get_mp_context()) + adapter = cast(BaseAdapter, get_adapter(config)) # reset the adapter since the dummy flags may be different from the flags for the actual subcommand reset_adapters() @@ -1582,7 +1602,7 @@ def my_dbt_op(dbt: DbtCliResource): adapter: Optional[BaseAdapter] = None with pushd(self.project_dir): try: - adapter = self._initialize_adapter(args) + adapter = self._initialize_adapter() except: # defer exceptions until they can be raised in the runtime context of the invocation diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py b/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py index ec41123a55844..f6ab50002a792 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/utils.py @@ -253,10 +253,7 @@ def __getattr__(self, item): return _DictShim(ret) if isinstance(ret, dict) else ret manifest = Manifest( - nodes={ - unique_id: _DictShim(info) - for unique_id, info in manifest_json["nodes"].items() # type: ignore - }, + nodes={unique_id: _DictShim(info) for unique_id, info in manifest_json["nodes"].items()}, sources={ unique_id: _DictShim(info) for unique_id, info in manifest_json["sources"].items() # type: ignore diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/version.py b/python_modules/libraries/dagster-dbt/dagster_dbt/version.py index 8189229002ea1..acdd0d0e4cc94 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/version.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/version.py @@ -1,3 +1,3 @@ -DBT_CORE_VERSION_UPPER_BOUND = "1.8" +DBT_CORE_VERSION_UPPER_BOUND = "1.9" __version__ = "1!0+dev" diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py index 924145efc87d9..c5f8d1198e642 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resources_v2.py @@ -106,16 +106,12 @@ def test_dbt_cli_failure() -> None: dbt = DbtCliResource(project_dir=os.fspath(test_exceptions_path), target="error_dev") - with pytest.raises( - DagsterDbtCliRuntimeError, match="Env var required but not provided: 'DBT_DUCKDB_THREADS'" - ): + with pytest.raises(Exception, match="Env var required but not provided: 'DBT_DUCKDB_THREADS'"): dbt.cli(["parse"]).wait() project = DbtProject(project_dir=os.fspath(test_exceptions_path), target="error_dev") dbt = DbtCliResource(project_dir=project) - with pytest.raises( - DagsterDbtCliRuntimeError, match="Env var required but not provided: 'DBT_DUCKDB_THREADS'" - ): + with pytest.raises(Exception, match="Env var required but not provided: 'DBT_DUCKDB_THREADS'"): dbt.cli(["parse"]).wait() diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/test_resources.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/test_resources.py index 4dece33a410c6..939b314059db2 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/test_resources.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/test_resources.py @@ -167,7 +167,7 @@ def my_dbt_solid(context): def test_extra_args_run(dbt_seed, test_project_dir, dbt_config_dir): - my_vars = {"foo": 1, "bar": "baz"} + my_vars = {"foo": "1", "bar": "baz"} @op(required_resource_keys={"dbt"}) def my_dbt_solid(context): @@ -183,7 +183,7 @@ def my_dbt_solid(context): def test_models_and_extra_run(dbt_seed, test_project_dir, dbt_config_dir): - my_vars = {"foo": 1, "bar": "baz"} + my_vars = {"foo": "1", "bar": "baz"} @op(required_resource_keys={"dbt"}) def my_dbt_solid(context): @@ -199,7 +199,7 @@ def my_dbt_solid(context): def test_exclude_run(dbt_seed, test_project_dir, dbt_config_dir): - my_vars = {"foo": 1, "bar": "baz"} + my_vars = {"foo": "1", "bar": "baz"} @op(required_resource_keys={"dbt"}) def my_dbt_solid(context): @@ -216,7 +216,7 @@ def my_dbt_solid(context): def test_merged_extra_flags_run(dbt_seed, test_project_dir, dbt_config_dir): configured_vars = {"hello": "world"} - my_vars = {"foo": 1, "bar": "baz"} + my_vars = {"foo": "1", "bar": "baz"} @op(required_resource_keys={"dbt"}) def my_dbt_solid(context): diff --git a/python_modules/libraries/dagster-dbt/tox.ini b/python_modules/libraries/dagster-dbt/tox.ini index 9b7ecfeff9a24..ad01433aae058 100644 --- a/python_modules/libraries/dagster-dbt/tox.ini +++ b/python_modules/libraries/dagster-dbt/tox.ini @@ -14,6 +14,8 @@ deps = dbt16: dbt-duckdb==1.6.* dbt17: dbt-core==1.7.* dbt17: dbt-duckdb==1.7.* + dbt18: dbt-core==1.8.* + dbt18: dbt-duckdb==1.8.* pydantic1: pydantic!=1.10.7,<2.0.0 -e .[test] allowlist_externals =