chore(ci): cleanup build to remove dependencies duckdb, dev (datahub-…

…project#7267)

docker/datahub-ingestion/base-requirements.txt

@@ -13,7 +13,6 @@ apache-airflow-providers-common-sql==1.3.3
 apache-airflow-providers-ftp==3.3.0
 apache-airflow-providers-http==4.1.1
 apache-airflow-providers-imap==3.1.1
-apache-airflow-providers-snowflake==4.0.2
 apache-airflow-providers-sqlite==3.3.1
 apispec==3.3.2
 appdirs==1.4.4
@@ -33,18 +32,15 @@ azure-storage-blob==12.14.1
 azure-storage-file-datalake==12.9.1
 Babel==2.11.0
 backcall==0.2.0
+backoff==2.2.1
 beautifulsoup4==4.11.2
-black==22.12.0
 bleach==6.0.0
 blinker==1.5
 blis==0.7.9
-boto3==1.26.63
-boto3-stubs==1.26.63
-botocore==1.29.63
-botocore-stubs==1.29.63
+boto3==1.26.65
+botocore==1.29.65
 bowler==0.9.0
 bracex==2.3.post1
-build==0.10.0
 cached-property==1.5.2
 cachelib==0.9.0
 cachetools==5.3.0
@@ -67,7 +63,6 @@ confection==0.0.4
 ConfigUpdater==3.1.1
 confluent-kafka==1.8.2
 connexion==2.14.2
-coverage==7.1.0
 cron-descriptor==1.2.35
 croniter==1.3.8
 cryptography==37.0.4
@@ -78,16 +73,13 @@ databricks-cli==0.17.3
 databricks-dbapi==0.6.0
 debugpy==1.6.6
 decorator==5.1.1
-deepdiff==6.2.3
 defusedxml==0.7.1
 deltalake==0.7.0
 Deprecated==1.2.13
 dill==0.3.6
-distlib==0.3.6
 dnspython==2.3.0
 docker==6.0.1
 docutils==0.19
-duckdb==0.6.1
 ecdsa==0.18.0
 elasticsearch==7.13.4
 email-validator==1.3.1
@@ -102,11 +94,8 @@ fastjsonschema==2.16.2
 feast==0.29.0
 filelock==3.9.0
 fissix==21.11.13
-flake8==6.0.0
-flake8-tidy-imports==4.8.0
 Flask==2.2.2
 flatdict==4.0.1
-freezegun==1.2.2
 frozenlist==1.3.3
 fsspec==2023.1.0
 future==0.18.3
@@ -127,6 +116,8 @@ google-cloud-logging==3.1.1
 google-crc32c==1.5.0
 google-resumable-media==2.4.1
 googleapis-common-protos==1.58.0
+gql==3.4.0
+graphql-core==3.2.3
 graphviz==0.20.1
 great-expectations==0.15.41
 greenlet==2.0.2
@@ -147,31 +138,27 @@ idna==3.4
 ijson==3.2.0.post0
 importlib-metadata==6.0.0
 inflection==0.5.1
-iniconfig==2.0.0
 ipaddress==1.0.23
 ipykernel==6.17.1
 ipython==8.9.0
 ipython-genutils==0.2.0
 ipywidgets==8.0.4
 iso3166==2.1.1
 isodate==0.6.1
-isort==5.12.0
 itsdangerous==2.1.2
-jaraco.classes==3.2.3
 jedi==0.18.2
 Jinja2==3.1.2
 jmespath==1.0.1
+JPype1==1.4.1
 jsonlines==3.1.0
 jsonpatch==1.32
-jsonpickle==3.0.1
 jsonpointer==2.3
 jsonschema==4.17.3
 jupyter-server==1.23.5
 jupyter_client==7.4.9
 jupyter_core==4.12.0
 jupyterlab-pygments==0.2.2
 jupyterlab-widgets==3.0.5
-keyring==23.13.1
 langcodes==3.3.0
 lark==1.1.4
 lazy-object-proxy==1.9.0
@@ -192,26 +179,21 @@ marshmallow-enum==1.5.1
 marshmallow-oneofschema==3.0.1
 marshmallow-sqlalchemy==0.26.1
 matplotlib-inline==0.1.6
-mccabe==0.7.0
 mdit-py-plugins==0.3.3
 mdurl==0.1.2
-mistune==2.0.4
+mistune==2.0.5
 mixpanel==4.10.0
 mmh3==3.0.0
 more-itertools==9.0.0
 moreorless==0.4.0
-moto==4.1.1
+moto==4.1.2
 msal==1.16.0
 msal-extensions==1.0.0
 msrest==0.7.1
 multidict==6.0.4
 murmurhash==1.0.9
-mypy==0.991
-mypy-boto3-glue==1.26.53
-mypy-boto3-s3==1.26.62
-mypy-boto3-sagemaker==1.26.61
-mypy-boto3-sts==1.26.57
-mypy-extensions==0.4.3
+mypy==1.0.0
+mypy-extensions==1.0.0
 nbclassic==0.5.1
 nbclient==0.6.3
 nbconvert==7.2.9
@@ -221,12 +203,10 @@ networkx==3.0
 notebook==6.5.2
 notebook_shim==0.2.2
 ntlm-auth==1.5.0
-numpy==1.24.1
+numpy==1.24.2
 oauthlib==3.2.2
 okta==1.7.0
 openpyxl==3.1.0
-ordered-set==4.1.0
-orjson==3.8.5
 oscrypto==1.3.0
 packaging==21.3
 pandas==1.5.3
@@ -241,8 +221,7 @@ pendulum==2.1.2
 pexpect==4.8.0
 phonenumbers==8.13.0
 pickleshare==0.7.5
-pkginfo==1.9.6
-platformdirs==2.6.2
+platformdirs==3.0.0
 pluggy==1.0.0
 portalocker==2.7.0
 preshed==3.0.8
@@ -260,7 +239,7 @@ pure-sasl==0.6.2
 pyarrow==8.0.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycodestyle==2.10.0
+pyathena==2.4.1
 pycountry==22.3.5
 pycparser==2.21
 pycryptodome==3.17
@@ -269,18 +248,13 @@ pydantic==1.10.4
 pydash==6.0.0
 pydeequ==1.0.1
 pydruid==0.6.5
-pyflakes==3.0.1
 Pygments==2.14.0
+pymongo==4.3.3
 PyMySQL==1.0.2
 pyOpenSSL==22.0.0
 pyparsing==3.0.9
-pyproject_hooks==1.0.0
 pyrsistent==0.19.3
 pyspark==3.0.3
-pytest==7.2.1
-pytest-asyncio==0.20.3
-pytest-cov==4.0.0
-pytest-docker==1.0.1
 python-daemon==2.3.2
 python-dateutil==2.8.2
 python-dotenv==0.21.1
@@ -289,6 +263,7 @@ python-ldap==3.4.3
 python-nvd3==0.15.0
 python-slugify==8.0.0
 python-stdnum==1.18
+python-tds==1.11.0
 python-utils==3.4.5
 python3-openid==3.2.0
 pytz==2022.7.1
@@ -297,12 +272,10 @@ pytzdata==2020.1
 PyYAML==6.0
 pyzmq==25.0.0
 ratelimiter==1.2.0.post0
-readme-renderer==37.3
 redash-toolbelt==0.1.9
 regex==2022.10.31
 requests==2.28.2
 requests-file==1.5.1
-requests-mock==1.10.0
 requests-ntlm==1.1.0
 requests-oauthlib==1.3.1
 requests-toolbelt==0.10.1
@@ -314,7 +287,7 @@ rsa==4.9
 ruamel.yaml==0.17.17
 s3transfer==0.6.0
 sasl3==0.2.11
-schwifty==2022.9.0
+schwifty==2023.2.0
 scipy==1.10.0
 Send2Trash==1.8.0
 setproctitle==1.3.2
@@ -333,6 +306,7 @@ sql-metadata==2.2.2
 SQLAlchemy==1.4.41
 sqlalchemy-bigquery==1.6.1
 SQLAlchemy-JSONField==1.0.1.post0
+sqlalchemy-pytds==0.3.4
 sqlalchemy-redshift==0.8.12
 SQLAlchemy-Utils==0.39.0
 sqlalchemy2-stubs==0.0.2a32
@@ -361,31 +335,9 @@ tornado==6.2
 tqdm==4.64.1
 traitlets==5.2.1.post0
 trino==0.321.0
-twine==4.0.2
 typeguard==2.13.3
 typer==0.7.0
-types-awscrt==0.16.4
-types-cachetools==5.3.0.0
-types-click==0.1.12
-types-click-spinner==0.1.13.1
-types-dataclasses==0.6.6
-types-Deprecated==1.2.9
-types-freezegun==1.1.10
-types-pkg-resources==0.1.3
-types-protobuf==4.21.0.4
-types-PyMySQL==1.0.19.2
-types-pyOpenSSL==23.0.0.2
-types-python-dateutil==2.8.19.6
-types-pytz==2022.7.1.0
-types-PyYAML==6.0.12.4
-types-requests==2.28.11.8
-types-s3transfer==0.6.0.post5
-types-six==1.16.21.4
-types-tabulate==0.9.0.0
-types-termcolor==1.1.6
-types-toml==0.10.8.2
-types-ujson==5.7.0.0
-types-urllib3==1.26.25.4
+types-toml==0.10.8.3
 typing-inspect==0.8.0
 typing_extensions==4.4.0
 tzdata==2022.7
@@ -397,15 +349,16 @@ uritemplate==4.1.1
 urllib3==1.26.14
 uvicorn==0.20.0
 uvloop==0.17.0
+vertica-python==1.2.0
+vertica-sqlalchemy-dialect==0.0.1
 vininfo==1.7.0
-virtualenv==20.17.1
 volatile==2.1.0
 wasabi==0.10.1
 watchfiles==0.18.1
 wcmatch==8.4.1
 wcwidth==0.2.6
 webencodings==0.5.1
-websocket-client==1.5.0
+websocket-client==1.5.1
 websockets==10.4
 Werkzeug==2.2.2
 widgetsnbextension==4.0.5