From a963b3820ff25ecf1a2d84e4e53d1a9ba1d73412 Mon Sep 17 00:00:00 2001 From: Michael Kotliar Date: Tue, 21 Sep 2021 14:08:33 -0400 Subject: [PATCH] Package updates. Small big fixes (#71) * Use the latest versions of Airflow, cwltool and constraints * Fix unit tests * Refactored tests * Not important changes * Fixin docker-compose to be able to run conformance tests * Fix bug with ARG in dockerfile * Fix bug with passing ARG to dockerfile from docker compose * Not important changes * Updated logging template to correspond to the latest Airflow * Update dockerfile to be able to set Ubuntu & Python versions, GUI, UID Run everything from the airflow user * Update Airflow to 2.1.4 * Fix confilting in Airflow 2.1.4 constraint * Fix docker-compose to run tests for custom Ubuntu and Python * Preparing to release --- .travis.yml | 69 +- README.dev.md | 32 + README.md | 9 +- cwl_airflow/components/init/config.py | 2 +- .../airflow_local_settings.py | 30 +- cwl_airflow/utilities/cwl.py | 2 +- docs/index.rst | 2 +- docs/readme/how_to_use.md | 2 +- packaging/constraints/constraints-3.6.txt | 579 ++++++++++------- packaging/constraints/constraints-3.7.txt | 587 +++++++++++------- packaging/constraints/constraints-3.8.txt | 587 +++++++++++------- packaging/constraints/constraints-3.9.txt | 526 ++++++++++++++++ packaging/docker_compose/local_executor/.env | 11 + .../local_executor/cwl_airflow/Dockerfile | 89 ++- .../local_executor/docker-compose.yml | 24 +- setup.py | 8 +- test_requirements.txt | 8 +- .../run_conformance_tests.sh} | 25 +- tests/run_conformance_tests_docker.sh | 10 + tests/run_cwl_v1.1_tests.sh | 19 - tests/run_cwl_v1.2_tests.sh | 6 - tests/run_unit_tests_docker.sh | 19 + tests/run_workflows_tests.sh | 6 - tests/{ => unit_tests}/__init__.py | 0 tests/{ => unit_tests}/run_unit_tests.sh | 3 +- tests/{ => unit_tests}/test_conformance.py | 2 +- tests/{ => unit_tests}/test_cwl.py | 2 +- tests/{ => unit_tests}/test_helpers.py | 2 +- tests/{ => unit_tests}/test_init_config.py | 2 +- tests/{ => unit_tests}/test_parser.py | 2 +- 30 files changed, 1807 insertions(+), 858 deletions(-) create mode 100644 README.dev.md create mode 100644 packaging/constraints/constraints-3.9.txt rename tests/{test_conformance.sh => conformance_tests/run_conformance_tests.sh} (84%) create mode 100755 tests/run_conformance_tests_docker.sh delete mode 100755 tests/run_cwl_v1.1_tests.sh delete mode 100755 tests/run_cwl_v1.2_tests.sh create mode 100755 tests/run_unit_tests_docker.sh delete mode 100755 tests/run_workflows_tests.sh rename tests/{ => unit_tests}/__init__.py (100%) rename tests/{ => unit_tests}/run_unit_tests.sh (61%) rename tests/{ => unit_tests}/test_conformance.py (95%) rename tests/{ => unit_tests}/test_cwl.py (99%) rename tests/{ => unit_tests}/test_helpers.py (99%) rename tests/{ => unit_tests}/test_init_config.py (98%) rename tests/{ => unit_tests}/test_parser.py (98%) diff --git a/.travis.yml b/.travis.yml index 64726d4..3e305fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python dist: bionic @@ -6,7 +5,7 @@ os: - linux services: - - docker +- docker python: - 3.7 @@ -23,7 +22,7 @@ jobs: - pip install . --constraint ./packaging/constraints/constraints-$TRAVIS_PYTHON_VERSION.txt before_script: # to overwrite from the main config - echo "Skip" - script: ./tests/run_unit_tests.sh + script: ./tests/unit_tests/run_unit_tests.sh after_success: - coveralls deploy: @@ -33,41 +32,47 @@ jobs: secure: Mji1koR4nyt/KgoycpuvgIp9toFVNYaSxUmNY6EVt0pmIpRb/GKbw6TdyfAdtnSAwH3BcSUC/R1hCwyaXfv1GDPFYqv9Yg1MaNHR1clvo8E8KIIPt1JDqPM47lgPQQFFbwB+Cc6uSV0Nn9oDBkhWEPQqV3kI/GJkSUzSs/yjZqR4C+aZxsJzE+VX2ZzeGCD3x4mzhAAWan4MLrdgANaXQVTHhyHIhTp3l109FblYimMvx8HqKotMiM+32mVFxgwf/pMw/N8gDOFXd4VrtlaOqqHpn4VJko+jSNYuAdKn62N2KFKqExyU39ycvU9ngYaU38nmCjJdibRgNyxfdH6LfndS9xzu3KPY64ACLG1i8Ym+57Q7wSJZAb2WF/b8av1RnkKMUGHHYXBzVIGk7Abvuhde0DsV0lr9XsapQn7XySmhdBWYazZTr+AtgIdsx7AmHV1ug6nPp3tIQzW1+YAOf295Puwqbrn+SF3jYw6167jAl5M1a81kxqli1UTsLgpcaTbTD1ofwLn4gP3VuU1f4fKGzhrxl6ybHW+LpO/wkcN2wJDdBbqz5OQIYfshMQEooIODOw1OonmwbY3vcMATuvi7Hz3mIElqpu3TVxH9aoBzcvL1148wPhZF8u87T8nDgsHeUT66I56ILGcZszASolt2Cb6oPZmxg2jgajTREwk= on: tags: true + + # Still valid, but soon will be deprecated, if we start using Airflow API - name: DAG with embedded workflow (just one test) script: cwl-airflow test --suite workflows/tests/conformance_tests.yaml --spin --range 1 --embed - name: DAG with attached workflow using combined API call (just one test) script: cwl-airflow test --suite workflows/tests/conformance_tests.yaml --spin --range 1 --combine - name: DAG with embedded workflow using combined API call (just one test) script: cwl-airflow test --suite workflows/tests/conformance_tests.yaml --spin --range 1 --embed --combine - - name: Test of `init --upgrade` - before_install: - - mkdir -p ~/airflow/dags - - cp ./tests/data/dags/bam_bedgraph_bigwig_single_old_format.py ~/airflow/dags - - cp ./tests/data/workflows/bam-bedgraph-bigwig-single.cwl ~/airflow/dags - install: - - pip install . --constraint ./packaging/constraints/constraints-$TRAVIS_PYTHON_VERSION.txt - before_script: - - cwl-airflow init --upgrade - - rm -f ~/airflow/dags/bam-bedgraph-bigwig-single.cwl - script: airflow dags list # to check if all DAGs are correct - - name: Test packaging for Ubuntu 18.04, Python 3.6 - install: - - ./packaging/portable/ubuntu/pack.sh 18.04 3.6 $TRAVIS_BRANCH - - ls ./packaging/portable/ubuntu/build/ - - tar xzf "./packaging/portable/ubuntu/build/python_3.6_with_cwl_airflow_${TRAVIS_BRANCH}_ubuntu_18.04.tar.gz" - before_script: - - ./python3/bin_portable/airflow --help # to generate airflow.cfg - - sed -i'.backup' -e 's/^executor.*/executor = LocalExecutor/g' ~/airflow/airflow.cfg - - sed -i'.backup' -e 's/^parsing_processes.*/parsing_processes = 1/g' ~/airflow/airflow.cfg - - sed -i'.backup' -e 's/^sql_alchemy_pool_enabled.*/sql_alchemy_pool_enabled = False/g' ~/airflow/airflow.cfg - - sed -i'.backup' -e 's/^dag_dir_list_interval =.*/dag_dir_list_interval = 60/g' ~/airflow/airflow.cfg - - sed -i'.backup' -e 's/^parallelism =.*/parallelism = 1/g' ~/airflow/airflow.cfg - - sed -i'.backup' -e 's/^sql_alchemy_conn.*/sql_alchemy_conn = mysql:\/\/airflow:airflow@127.0.0.1:6603\/airflow/g' ~/airflow/airflow.cfg - - ./python3/bin_portable/cwl-airflow init - - ./python3/bin_portable/airflow connections add process_report --conn-type http --conn-host localhost --conn-port 3070 # to add process_report connection - - ./python3/bin_portable/airflow scheduler > /dev/null 2>&1 & - - ./python3/bin_portable/cwl-airflow api --replay 600 > /dev/null 2>&1 & - script: ./python3/bin_portable/cwl-airflow test --suite workflows/tests/conformance_tests.yaml --spin --range 1 + + # TEST DEPRECATED as it not needed anymore. Everything already got updated. + # - name: Test of `init --upgrade` + # before_install: + # - mkdir -p ~/airflow/dags + # - cp ./tests/data/dags/bam_bedgraph_bigwig_single_old_format.py ~/airflow/dags + # - cp ./tests/data/workflows/bam-bedgraph-bigwig-single.cwl ~/airflow/dags + # install: + # - pip install . --constraint ./packaging/constraints/constraints-$TRAVIS_PYTHON_VERSION.txt + # before_script: + # - cwl-airflow init --upgrade + # - rm -f ~/airflow/dags/bam-bedgraph-bigwig-single.cwl + # script: airflow dags list # to check if all DAGs are correct + + # TEST DEPRECATED as the packaging scripts should be replaced + # - name: Test packaging for Ubuntu 18.04, Python 3.6 + # install: + # - ./packaging/portable/ubuntu/pack.sh 18.04 3.6 $TRAVIS_BRANCH + # - ls ./packaging/portable/ubuntu/build/ + # - tar xzf "./packaging/portable/ubuntu/build/python_3.6_with_cwl_airflow_${TRAVIS_BRANCH}_ubuntu_18.04.tar.gz" + # before_script: + # - ./python3/bin_portable/airflow --help # to generate airflow.cfg + # - sed -i'.backup' -e 's/^executor.*/executor = LocalExecutor/g' ~/airflow/airflow.cfg + # - sed -i'.backup' -e 's/^parsing_processes.*/parsing_processes = 1/g' ~/airflow/airflow.cfg + # - sed -i'.backup' -e 's/^sql_alchemy_pool_enabled.*/sql_alchemy_pool_enabled = False/g' ~/airflow/airflow.cfg + # - sed -i'.backup' -e 's/^dag_dir_list_interval =.*/dag_dir_list_interval = 60/g' ~/airflow/airflow.cfg + # - sed -i'.backup' -e 's/^parallelism =.*/parallelism = 1/g' ~/airflow/airflow.cfg + # - sed -i'.backup' -e 's/^sql_alchemy_conn.*/sql_alchemy_conn = mysql:\/\/airflow:airflow@127.0.0.1:6603\/airflow/g' ~/airflow/airflow.cfg + # - ./python3/bin_portable/cwl-airflow init + # - ./python3/bin_portable/airflow connections add process_report --conn-type http --conn-host localhost --conn-port 3070 # to add process_report connection + # - ./python3/bin_portable/airflow scheduler > /dev/null 2>&1 & + # - ./python3/bin_portable/cwl-airflow api --replay 600 > /dev/null 2>&1 & + # script: ./python3/bin_portable/cwl-airflow test --suite workflows/tests/conformance_tests.yaml --spin --range 1 before_install: - git clone https://github.com/datirium/workflows.git --recursive diff --git a/README.dev.md b/README.dev.md new file mode 100644 index 0000000..ba38ae5 --- /dev/null +++ b/README.dev.md @@ -0,0 +1,32 @@ +## **Notes for developers** + +When running on MacOS, you might need to set up the following env variable before starting `airflow scheduler/webserver` + +``` +export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES +``` + +**Conformance and unit tests were run for** +- macOS 11.4 + - Python 3.8.6 +- Ubuntu 18.04 + - Python 3.6.8 + - Python 3.7.9 + - Python 3.8.10 +- Ubuntu 20.04 + - Python 3.6.8 + - Python 3.7.9 + - Python 3.8.10 + +*For Ubuntu the Python versions were selected based on latest available binary release at the time of testing. + +**To run conformance tests in Docker container** +``` +cd tests +./run_conformance_tests_docker.sh $UBUNTU_VERSION $PYTHON_VERSION $CWL_AIRFLOW_VERSION $REPO_URL $SUITE +``` +**To run unit tests in Docker container** +``` +cd tests +./run_unit_tests_docker.sh $UBUNTU_VERSION $PYTHON_VERSION $CWL_AIRFLOW_VERSION +``` \ No newline at end of file diff --git a/README.md b/README.md index 53f7ece..d75d0c4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ # **CWL-Airflow** -Python package to extend **[Apache-Airflow 2.0.1](https://airflow.apache.org)** +Python package to extend **[Apache-Airflow 2.1.4](https://airflow.apache.org)** functionality with **[CWL v1.1](https://www.commonwl.org/v1.1/)** support ## **Cite as** @@ -29,9 +29,4 @@ pip install cwl-airflow==1.0.18 Published version [documentation](https://cwl-airflow.readthedocs.io/en/1.0.18/) ## **Notes** - -When running on MacOS, you might need to set up the following env variable before starting `airflow scheduler/webserver` - -``` -export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES -``` \ No newline at end of file +Notes for developers can be found in [DEV README](https://github.com/Barski-lab/cwl-airflow/blob/master/README.dev.md) diff --git a/cwl_airflow/components/init/config.py b/cwl_airflow/components/init/config.py index 88788ee..6e986e8 100644 --- a/cwl_airflow/components/init/config.py +++ b/cwl_airflow/components/init/config.py @@ -15,7 +15,7 @@ with CleanAirflowImport(): from airflow.configuration import conf from airflow.exceptions import AirflowConfigException - from airflow.utils.dag_processing import list_py_file_paths + from airflow.utils.file import list_py_file_paths from cwl_airflow.utilities.cwl import overwrite_deprecated_dag diff --git a/cwl_airflow/config_templates/airflow_local_settings.py b/cwl_airflow/config_templates/airflow_local_settings.py index 14ea5ae..b4fc27a 100644 --- a/cwl_airflow/config_templates/airflow_local_settings.py +++ b/cwl_airflow/config_templates/airflow_local_settings.py @@ -17,9 +17,6 @@ # under the License. """Airflow logging settings""" -# COPY of /airflow/config_templates/airflow_local_settings.py from Airflow 2.0.0 -# with added cwltool logger and handler - import os from pathlib import Path from typing import Any, Dict, Union @@ -57,6 +54,11 @@ PROCESSOR_FILENAME_TEMPLATE: str = conf.get('logging', 'LOG_PROCESSOR_FILENAME_TEMPLATE') + +# COPY of /airflow/config_templates/airflow_local_settings.py from Airflow 2.1.3 +# with added cwltool logger and handler + + DEFAULT_LOGGING_CONFIG: Dict[str, Any] = { 'version': 1, 'disable_existing_loggers': False, @@ -67,29 +69,38 @@ 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter', }, }, + 'filters': { + 'mask_secrets': { + '()': 'airflow.utils.log.secrets_masker.SecretsMasker', + }, + }, 'handlers': { 'console': { 'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler', 'formatter': 'airflow_coloured', 'stream': 'sys.stdout', + 'filters': ['mask_secrets'], }, 'task': { 'class': 'airflow.utils.log.file_task_handler.FileTaskHandler', 'formatter': 'airflow', 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), 'filename_template': FILENAME_TEMPLATE, + 'filters': ['mask_secrets'], }, 'processor': { 'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler', 'formatter': 'airflow', 'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER), 'filename_template': PROCESSOR_FILENAME_TEMPLATE, + 'filters': ['mask_secrets'], }, 'cwltool': { 'class': 'airflow.utils.log.file_task_handler.FileTaskHandler', 'formatter': 'airflow', 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), - 'filename_template': FILENAME_TEMPLATE + ".cwl" + 'filename_template': FILENAME_TEMPLATE + ".cwl", + 'filters': ['mask_secrets'] } }, 'loggers': { @@ -102,6 +113,7 @@ 'handlers': ['task'], 'level': LOG_LEVEL, 'propagate': False, + 'filters': ['mask_secrets'], }, 'flask_appbuilder': { 'handler': ['console'], @@ -111,12 +123,14 @@ 'cwltool': { 'handlers': ['cwltool'], 'level': LOG_LEVEL, - 'propagate': False + 'propagate': False, + 'filters': ['mask_secrets'] } }, 'root': { 'handlers': ['console'], 'level': LOG_LEVEL, + 'filters': ['mask_secrets'], }, } @@ -258,6 +272,8 @@ ELASTICSEARCH_WRITE_STDOUT: bool = conf.getboolean('elasticsearch', 'WRITE_STDOUT') ELASTICSEARCH_JSON_FORMAT: bool = conf.getboolean('elasticsearch', 'JSON_FORMAT') ELASTICSEARCH_JSON_FIELDS: str = conf.get('elasticsearch', 'JSON_FIELDS') + ELASTICSEARCH_HOST_FIELD: str = conf.get('elasticsearch', 'HOST_FIELD') + ELASTICSEARCH_OFFSET_FIELD: str = conf.get('elasticsearch', 'OFFSET_FIELD') ELASTIC_REMOTE_HANDLERS: Dict[str, Dict[str, Union[str, bool]]] = { 'task': { @@ -272,6 +288,8 @@ 'write_stdout': ELASTICSEARCH_WRITE_STDOUT, 'json_format': ELASTICSEARCH_JSON_FORMAT, 'json_fields': ELASTICSEARCH_JSON_FIELDS, + 'host_field': ELASTICSEARCH_HOST_FIELD, + 'offset_field': ELASTICSEARCH_OFFSET_FIELD, }, } @@ -280,5 +298,5 @@ raise AirflowException( "Incorrect remote log configuration. Please check the configuration of option 'host' in " "section 'elasticsearch' if you are using Elasticsearch. In the other case, " - "'remote_base_log_folder' option in 'core' section." + "'remote_base_log_folder' option in the 'logging' section." ) diff --git a/cwl_airflow/utilities/cwl.py b/cwl_airflow/utilities/cwl.py index 1262b20..e81ec48 100644 --- a/cwl_airflow/utilities/cwl.py +++ b/cwl_airflow/utilities/cwl.py @@ -26,7 +26,7 @@ from airflow.utils.state import State from airflow.utils.db import provide_session from airflow.exceptions import AirflowConfigException -from airflow.utils.dag_processing import list_py_file_paths +from airflow.utils.file import list_py_file_paths from airflow.api.common.experimental import delete_dag from cwltool.argparser import get_default_args from cwltool.main import ( diff --git a/docs/index.rst b/docs/index.rst index d24e30a..effeef6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ Welcome to CWL-Airflow's documentation! :target: https://pepy.tech/project/cwl-airflow -Python package to extend `Apache-Airflow 2.0.1 `_ functionality with `CWL v1.1 `_ support. +Python package to extend `Apache-Airflow 2.1.4 `_ functionality with `CWL v1.1 `_ support. .. raw:: html diff --git a/docs/readme/how_to_use.md b/docs/readme/how_to_use.md index 1a95fc1..71b0e92 100644 --- a/docs/readme/how_to_use.md +++ b/docs/readme/how_to_use.md @@ -21,7 +21,7 @@ optional arguments: **Init command will run the following steps** for the specified `--home` and `--config` parameters: - Call `airflow --help` to create a default `airflow.cfg` -- Update `airflow.cfg` to hide paused DAGs, skip loading example DAGs and connections and **do not** pause newly created DAGs. Also, we set our custom `logging_config_class` to split Airflow and CWL related logs into the separate files. In case of upgrading from the previous version of CWL-Airflow that used Airflow < 2.0.0 to the latest one, `airflow.cfg` will be backuped and upgraded to fit Airflow 2.0.1. You will have to manually make sure that all custom fields were properly copied to the new `airflow.cfg` +- Update `airflow.cfg` to hide paused DAGs, skip loading example DAGs and connections and **do not** pause newly created DAGs. Also, we set our custom `logging_config_class` to split Airflow and CWL related logs into the separate files. In case of upgrading from the previous version of CWL-Airflow that used Airflow < 2.0.0 to the latest one, `airflow.cfg` will be backuped and upgraded to fit Airflow 2.1.4. You will have to manually make sure that all custom fields were properly copied to the new `airflow.cfg` - Call `airflow db init` to init/upgrade Airflow metadata database. - If run with `--upgrade`, upgrade old CWLDAGs to correspond to the latest format, save original CWLDAGs into `deprecated_dags` folder. - Put **clean_dag_run.py** into the DAGs folder. diff --git a/packaging/constraints/constraints-3.6.txt b/packaging/constraints/constraints-3.6.txt index 631b6f6..14e9fc6 100644 --- a/packaging/constraints/constraints-3.6.txt +++ b/packaging/constraints/constraints-3.6.txt @@ -1,247 +1,337 @@ -# Editable install with no version control (apache-airflow==2.0.1) +# +# This constraints file was automatically generated on 2021-09-15T22:04:12Z +# via "eager-upgrade" mechanism of PIP. For the "v2-1-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# Editable install with no version control (apache-airflow==2.1.4) APScheduler==3.6.3 -Authlib==0.15.3 -Babel==2.9.0 -Flask-AppBuilder==3.1.1 +Authlib==0.15.4 +Babel==2.9.1 +Deprecated==1.2.13 +Flask-AppBuilder==3.3.3 Flask-Babel==1.0.0 Flask-Bcrypt==0.7.1 -Flask-Caching==1.9.0 -Flask-JWT-Extended==3.25.0 +Flask-Caching==1.10.1 +Flask-JWT-Extended==3.25.1 Flask-Login==0.4.1 -Flask-OAuthlib==0.9.5 -Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.4 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 Flask-WTF==0.14.3 -Flask==1.1.2 -GitPython==3.1.12 +Flask==1.1.4 +GitPython==3.1.18 HeapDict==1.0.1 -JPype1==1.2.1 +JPype1==1.3.0 JayDeBeApi==1.2.3 Jinja2==2.11.3 -Mako==1.1.4 -Markdown==3.3.3 +Mako==1.1.5 +Markdown==3.3.4 MarkupSafe==1.1.1 -PyHive==0.6.3 +PyGithub==1.54.1 +PyHive==0.6.4 PyJWT==1.7.1 PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.4.1 -Pygments==2.7.4 +Pygments==2.10.0 SQLAlchemy-JSONField==1.0.0 -SQLAlchemy-Utils==0.36.8 -SQLAlchemy==1.3.23 +SQLAlchemy-Utils==0.37.8 +SQLAlchemy==1.3.24 +SecretStorage==3.3.1 Sphinx==3.4.3 -Unidecode==1.2.0 +Unidecode==1.3.1 WTForms==2.3.3 Werkzeug==1.0.1 -adal==1.2.6 -aiohttp==3.7.3 +adal==1.2.7 +aiohttp==3.7.4.post0 alabaster==0.7.12 -alembic==1.5.4 +alembic==1.7.1 amqp==2.6.1 -analytics-python==1.2.9 +analytics-python==1.4.0 ansiwrap==0.8.4 -apipkg==1.5 +anyio==3.3.1 +apache-airflow-providers-airbyte==2.1.1 +apache-airflow-providers-amazon==2.2.0 +apache-airflow-providers-apache-beam==3.0.1 +apache-airflow-providers-apache-cassandra==2.0.1 +apache-airflow-providers-apache-drill==1.0.1 +apache-airflow-providers-apache-druid==2.0.2 +apache-airflow-providers-apache-hdfs==2.1.0 +apache-airflow-providers-apache-hive==2.0.2 +apache-airflow-providers-apache-kylin==2.0.1 +apache-airflow-providers-apache-livy==2.1.0 +apache-airflow-providers-apache-pig==2.0.1 +apache-airflow-providers-apache-pinot==2.0.1 +apache-airflow-providers-apache-spark==2.0.1 +apache-airflow-providers-apache-sqoop==2.0.2 +apache-airflow-providers-asana==1.1.0 +apache-airflow-providers-celery==2.0.0 +apache-airflow-providers-cloudant==2.0.1 +apache-airflow-providers-cncf-kubernetes==2.0.2 +apache-airflow-providers-databricks==2.0.1 +apache-airflow-providers-datadog==2.0.1 +apache-airflow-providers-dingding==2.0.1 +apache-airflow-providers-discord==2.0.1 +apache-airflow-providers-docker==2.1.1 +apache-airflow-providers-elasticsearch==2.0.3 +apache-airflow-providers-exasol==2.0.1 +apache-airflow-providers-facebook==2.0.1 +apache-airflow-providers-ftp==2.0.1 +apache-airflow-providers-google==5.1.0 +apache-airflow-providers-grpc==2.0.1 +apache-airflow-providers-hashicorp==2.1.0 +apache-airflow-providers-http==2.0.1 +apache-airflow-providers-imap==2.0.1 +apache-airflow-providers-jdbc==2.0.1 +apache-airflow-providers-jenkins==2.0.2 +apache-airflow-providers-jira==2.0.1 +apache-airflow-providers-microsoft-azure==3.1.1 +apache-airflow-providers-microsoft-mssql==2.0.1 +apache-airflow-providers-microsoft-winrm==2.0.1 +apache-airflow-providers-mongo==2.1.0 +apache-airflow-providers-mysql==2.1.1 +apache-airflow-providers-neo4j==2.0.1 +apache-airflow-providers-odbc==2.0.1 +apache-airflow-providers-openfaas==2.0.0 +apache-airflow-providers-opsgenie==2.0.1 +apache-airflow-providers-oracle==2.0.1 +apache-airflow-providers-pagerduty==2.0.1 +apache-airflow-providers-papermill==2.0.1 +apache-airflow-providers-plexus==2.0.1 +apache-airflow-providers-postgres==2.2.0 +apache-airflow-providers-presto==2.0.1 +apache-airflow-providers-qubole==2.0.1 +apache-airflow-providers-redis==2.0.1 +apache-airflow-providers-salesforce==3.2.0 +apache-airflow-providers-samba==3.0.0 +apache-airflow-providers-segment==2.0.1 +apache-airflow-providers-sendgrid==2.0.1 +apache-airflow-providers-sftp==2.1.1 +apache-airflow-providers-singularity==2.0.1 +apache-airflow-providers-slack==4.0.1 +apache-airflow-providers-snowflake==2.1.1 +apache-airflow-providers-sqlite==2.0.1 +apache-airflow-providers-ssh==2.1.1 +apache-airflow-providers-tableau==2.1.1 +apache-airflow-providers-telegram==2.0.1 +apache-airflow-providers-trino==2.0.1 +apache-airflow-providers-vertica==2.0.1 +apache-airflow-providers-yandex==2.1.0 +apache-airflow-providers-zendesk==2.0.1 +apache-beam==2.32.0 apispec==3.3.2 appdirs==1.4.4 -argcomplete==1.12.2 -arrow==0.17.0 +argcomplete==1.12.3 +arrow==1.1.1 +asana==0.10.3 asn1crypto==1.4.0 -astroid==2.4.2 +astroid==2.7.3 async-generator==1.10 async-timeout==3.0.1 atlasclient==1.0.0 attrs==20.3.0 -aws-sam-translator==1.34.0 -aws-xray-sdk==2.6.0 -azure-batch==10.0.0 -azure-common==1.1.26 -azure-core==1.11.0 +avro-python3==1.9.2.1 +aws-xray-sdk==2.8.0 +azure-batch==11.0.0 +azure-common==1.1.27 +azure-core==1.18.0 azure-cosmos==3.2.0 -azure-datalake-store==0.0.51 -azure-identity==1.5.0 -azure-keyvault-certificates==4.2.1 -azure-keyvault-keys==4.3.1 -azure-keyvault-secrets==4.2.0 +azure-datalake-store==0.0.52 +azure-identity==1.6.1 +azure-keyvault-certificates==4.3.0 +azure-keyvault-keys==4.4.0 +azure-keyvault-secrets==4.3.0 azure-keyvault==4.1.0 azure-kusto-data==0.0.45 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.2.2 +azure-mgmt-core==1.3.0 +azure-mgmt-datafactory==1.1.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==15.0.0 +azure-mgmt-resource==20.0.0 azure-nspkg==3.0.2 -azure-storage-blob==12.7.1 +azure-storage-blob==12.8.1 azure-storage-common==2.1.0 azure-storage-file==2.1.0 backcall==0.2.0 +backoff==1.10.0 +backports.entry-points-selectable==1.1.0 +backports.zoneinfo==0.2.1 bcrypt==3.2.0 beautifulsoup4==4.7.1 -billiard==3.6.3.0 -black==20.8b1 +billiard==3.6.4.0 +black==21.7b0 blinker==1.4 -boto3==1.15.18 +boto3==1.17.112 boto==2.49.0 -botocore==1.18.18 +botocore==1.20.112 bowler==0.9.0 cached-property==1.5.2 -cachetools==4.2.1 -cassandra-driver==3.20.2 +cachetools==4.2.2 +cassandra-driver==3.25.0 cattrs==1.0.0 celery==4.4.7 certifi==2020.12.5 -cffi==1.14.4 -cfgv==3.2.0 -cfn-lint==0.44.7 -cgroupspy==0.1.6 -chardet==3.0.4 +cffi==1.14.6 +cfgv==3.3.1 +cgroupspy==0.2.1 +chardet==4.0.0 +charset-normalizer==2.0.5 click==7.1.2 clickclick==20.10.2 -cloudant==2.14.0 +cloudant==2.15.0 cloudpickle==1.4.1 colorama==0.4.4 -colorlog==4.7.2 +colorlog==4.8.0 commonmark==0.9.1 -connexion==2.7.0 contextvars==2.4 -coverage==5.4 -croniter==0.3.37 -cryptography==3.4.3 +coverage==5.5 +crcmod==1.7 +croniter==1.0.15 +cryptography==3.4.8 curlify==2.2.1 -cx-Oracle==8.1.0 -dask==2021.2.0 -dataclasses==0.7 -datadog==0.39.0 -decorator==4.4.2 -defusedxml==0.6.0 -dill==0.3.3 -distlib==0.3.1 +cx-Oracle==8.2.1 +dask==2021.3.0 +dataclasses==0.8 +datadog==0.42.0 +decorator==5.1.0 +defusedxml==0.7.1 +dill==0.3.1.1 +distlib==0.3.2 distributed==2.19.0 dnspython==1.16.0 -docker-pycreds==0.4.0 -docker==3.7.3 +docker==5.0.2 docopt==0.6.2 docutils==0.16 -ecdsa==0.14.1 -elasticsearch-dbapi==0.1.0 -elasticsearch-dsl==7.3.0 -elasticsearch==7.5.1 -email-validator==1.1.2 +ecdsa==0.17.0 +elasticsearch-dbapi==0.2.5 +elasticsearch-dsl==7.4.0 +elasticsearch==7.14.1 +email-validator==1.1.3 entrypoints==0.3 -eventlet==0.30.1 -execnet==1.8.0 -facebook-business==9.0.2 -fastavro==1.3.1 +eventlet==0.32.0 +execnet==1.9.0 +facebook-business==11.0.0 +fastavro==1.4.4 filelock==3.0.12 -fissix==20.8.0 +fissix==21.6.6 flake8-colors==0.1.9 -flake8==3.8.4 +flake8==3.9.2 flaky==3.7.0 flower==0.9.7 freezegun==1.1.0 -fsspec==0.8.5 +fsspec==2021.8.1 future==0.18.2 -gcsfs==0.7.1 -gevent==21.1.2 -gitdb==4.0.5 -github3.py==1.3.0 -google-ads==4.0.0 -google-api-core==1.26.0 +gcsfs==2021.8.1 +geomet==0.2.1.post1 +gevent==21.8.0 +gitdb==4.0.7 +github3.py==2.0.0 +google-ads==12.0.0 +google-api-core==1.31.2 google-api-python-client==1.12.8 -google-auth-httplib2==0.0.4 -google-auth-oauthlib==0.4.2 -google-auth==1.25.0 -google-cloud-automl==1.0.1 -google-cloud-bigquery-datatransfer==1.1.1 -google-cloud-bigquery-storage==2.2.1 -google-cloud-bigquery==2.7.0 -google-cloud-bigtable==1.6.1 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.4.6 +google-auth==1.35.0 +google-cloud-appengine-logging==0.1.4 +google-cloud-audit-log==0.1.1 +google-cloud-automl==2.4.2 +google-cloud-bigquery-datatransfer==3.3.2 +google-cloud-bigquery-storage==2.8.0 +google-cloud-bigquery==2.26.0 +google-cloud-bigtable==1.7.0 google-cloud-container==1.0.1 -google-cloud-core==1.6.0 -google-cloud-datacatalog==0.7.0 -google-cloud-dataproc==1.1.1 +google-cloud-core==1.7.2 +google-cloud-datacatalog==3.4.1 +google-cloud-dataproc==2.5.0 google-cloud-dlp==1.0.0 -google-cloud-kms==1.4.0 +google-cloud-kms==2.6.0 google-cloud-language==1.3.0 -google-cloud-logging==1.15.1 -google-cloud-memcache==0.2.0 -google-cloud-monitoring==1.1.0 -google-cloud-os-login==1.0.0 -google-cloud-pubsub==1.7.0 -google-cloud-redis==1.0.0 +google-cloud-logging==2.6.0 +google-cloud-memcache==1.0.0 +google-cloud-monitoring==2.5.0 +google-cloud-os-login==2.3.1 +google-cloud-pubsub==2.8.0 +google-cloud-redis==2.2.2 google-cloud-secret-manager==1.0.0 google-cloud-spanner==1.19.1 google-cloud-speech==1.3.2 -google-cloud-storage==1.35.1 -google-cloud-tasks==1.5.0 +google-cloud-storage==1.42.1 +google-cloud-tasks==2.5.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==1.7.0 google-cloud-videointelligence==1.16.1 google-cloud-vision==1.0.0 +google-cloud-workflows==1.2.1 google-crc32c==1.1.2 -google-resumable-media==1.2.0 -googleapis-common-protos==1.52.0 -graphviz==0.16 -greenlet==1.0.0 +google-resumable-media==2.0.2 +googleapis-common-protos==1.53.0 +graphviz==0.17 +greenlet==1.1.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 -grpcio==1.35.0 -gunicorn==19.10.0 -hdfs==2.5.8 +grpcio==1.40.0 +gunicorn==20.1.0 +h11==0.12.0 +hdfs==2.6.0 hmsclient==0.1.1 -httplib2==0.19.0 -humanize==3.2.0 -hvac==0.10.8 -identify==1.5.13 +httpcore==0.13.7 +httplib2==0.19.1 +httpx==0.19.0 +humanize==3.11.0 +hvac==0.11.0 +identify==2.2.14 idna-ssl==1.1.0 -idna==2.10 +idna==3.2 +ijson==3.1.4 imagesize==1.2.0 -immutables==0.14 -importlib-metadata==1.7.0 +immutables==0.16 +importlib-metadata==4.8.1 importlib-resources==1.5.0 inflection==0.5.1 iniconfig==1.1.1 -ipdb==0.13.4 +ipdb==0.13.9 ipython-genutils==0.2.0 ipython==7.16.1 -iso8601==0.1.14 +iso8601==0.1.16 isodate==0.6.0 -isort==5.7.0 itsdangerous==1.1.0 jedi==0.18.0 -jira==2.0.0 +jeepney==0.7.1 +jira==3.0.1 jmespath==0.10.0 json-merge-patch==0.2 -jsondiff==1.2.0 -jsonpatch==1.28 -jsonpath-ng==1.5.2 -jsonpickle==2.0.0 -jsonpointer==2.0 +jsondiff==1.3.0 +jsonpath-ng==1.5.3 jsonschema==3.2.0 -junit-xml==1.9 -jupyter-client==6.1.11 +jupyter-client==7.0.2 jupyter-core==4.7.1 -jwcrypto==0.8 +jwcrypto==1.0 +keyring==23.2.1 kombu==4.6.11 kubernetes==11.0.0 kylinpy==2.8.4 lazy-object-proxy==1.4.3 -ldap3==2.9 -libcst==0.3.17 +ldap3==2.9.1 +libcst==0.3.20 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-oneofschema==2.1.0 +marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.23.1 -marshmallow==3.10.0 +marshmallow==3.13.0 mccabe==0.6.1 -mock==4.0.2 -mongomock==3.22.1 -more-itertools==8.7.0 -moreorless==0.3.0 -moto==1.3.16 +mongomock==3.23.0 +monotonic==1.6 +more-itertools==8.9.0 +moreorless==0.4.0 +moto==2.2.6 msal-extensions==0.3.0 -msal==1.9.0 +msal==1.14.0 msgpack==1.0.2 msrest==0.6.21 msrestazure==0.6.4 @@ -250,182 +340,195 @@ multidict==5.1.0 mypy-extensions==0.4.3 mypy==0.770 mysql-connector-python==8.0.22 -mysqlclient==1.3.14 -natsort==7.1.1 -nbclient==0.5.1 -nbformat==5.1.2 +mysqlclient==2.0.3 +nbclient==0.5.4 +nbformat==5.1.3 +neo4j==4.3.4 nest-asyncio==1.5.1 -networkx==2.5 -nodeenv==1.5.0 -nteract-scrapbook==0.4.2 +nodeenv==1.6.0 +nox==2020.12.31 ntlm-auth==1.5.0 numpy==1.19.5 -oauthlib==2.1.0 -openapi-spec-validator==0.2.9 +oauth2client==4.1.3 +oauthlib==3.1.1 +openapi-schema-validator==0.1.5 +openapi-spec-validator==0.3.1 +orjson==3.6.1 oscrypto==1.2.1 -packaging==20.9 +packaging==21.0 pandas-gbq==0.14.1 pandas==1.1.5 -papermill==2.3.2 +papermill==2.3.3 parameterized==0.8.1 paramiko==2.7.2 -parso==0.8.1 -pathspec==0.8.1 -pbr==5.5.1 -pdpyras==4.1.2 +parso==0.8.2 +pathspec==0.9.0 +pbr==5.6.0 +pdpyras==4.3.0 pendulum==2.1.2 -pep562==1.0 +pep562==1.1 pexpect==4.8.0 pickleshare==0.7.5 -pinotdb==0.3.3 -pipdeptree==2.0.0 -pluggy==0.13.1 +pinotdb==0.3.7 +pipdeptree==2.1.0 +platformdirs==2.3.0 +pluggy==1.0.0 ply==3.11 +plyvel==1.3.0 portalocker==1.7.1 -pre-commit==2.10.1 +pre-commit==2.15.0 presto-python-client==0.7.0 -prison==0.1.3 +prison==0.2.1 prometheus-client==0.8.0 -prompt-toolkit==3.0.14 -proto-plus==1.13.0 -protobuf==3.14.0 +prompt-toolkit==3.0.20 +proto-plus==1.19.0 +protobuf==3.18.0 psutil==5.8.0 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.1 ptyprocess==0.7.0 +pure-sasl==0.6.2 py4j==0.10.9 py==1.10.0 -pyOpenSSL==19.1.0 -pyarrow==3.0.0 +pyOpenSSL==20.0.1 +pyarrow==4.0.1 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycodestyle==2.6.0 +pycodestyle==2.7.0 pycountry==20.7.3 pycparser==2.20 -pycryptodomex==3.9.9 -pydata-google-auth==1.1.0 +pycryptodomex==3.10.1 +pydata-google-auth==1.2.0 +pydot==1.4.2 pydruid==0.6.2 -pyenchant==3.2.0 -pyexasol==0.17.0 -pyflakes==2.2.0 +pyenchant==3.2.1 +pyexasol==0.20.0 +pyflakes==2.3.1 pykerberos==1.2.1 -pylint==2.6.0 -pymongo==3.11.3 -pymssql==2.1.5 -pyodbc==4.0.30 +pymongo==3.12.0 +pymssql==2.2.2 +pyodbc==4.0.32 pyparsing==2.4.7 -pyrsistent==0.17.3 +pyrsistent==0.18.0 pysftp==0.2.9 -pyspark==3.0.1 -pytest-cov==2.11.1 +pyspark==3.1.2 +pyspnego==0.1.6 +pytest-cov==2.12.1 pytest-forked==1.3.0 +pytest-httpx==0.13.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.1.1 pytest-timeouts==1.2.1 -pytest-xdist==2.2.0 -pytest==6.2.2 -python-daemon==2.2.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-http-client==3.3.1 +pytest-xdist==2.3.0 +pytest==6.2.5 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-http-client==3.3.2 python-jenkins==1.7.0 -python-jose==3.2.0 +python-jose==3.3.0 python-ldap==3.3.1 python-nvd3==0.15.0 python-slugify==4.0.1 -python-telegram-bot==13.0 +python-telegram-bot==13.7 python3-openid==3.2.0 -pytz==2020.5 +pytz==2021.1 pytzdata==2020.1 -pywinrm==0.4.1 -pyzmq==22.0.2 +pywinrm==0.4.2 +pyzmq==22.2.1 qds-sdk==1.16.1 redis==3.5.3 -regex==2020.11.13 +regex==2021.8.28 requests-kerberos==0.12.0 -requests-mock==1.8.0 +requests-mock==1.9.3 requests-ntlm==1.1.0 -requests-oauthlib==1.1.0 +requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 -requests==2.25.1 -responses==0.12.1 -rich==9.2.0 -rsa==4.7 -s3transfer==0.3.4 -sasl==0.2.1 +requests==2.26.0 +responses==0.13.4 +rfc3986==1.5.0 +rich==10.9.0 +rsa==4.7.2 +s3transfer==0.4.2 +sasl==0.3.1 +scrapbook==0.5.0 semver==2.13.0 -sendgrid==6.5.0 +sendgrid==6.8.1 sentinels==1.0.0 -sentry-sdk==0.19.5 +sentry-sdk==1.3.1 setproctitle==1.2.2 -simple-salesforce==1.10.1 -six==1.15.0 -slackclient==2.9.3 -smmap==3.0.5 +simple-salesforce==1.11.4 +six==1.16.0 +slack-sdk==3.11.0 +smbprotocol==1.6.2 +smmap==4.0.0 +snakebite-py3==3.0.5 +sniffio==1.2.0 snowballstemmer==2.1.0 -snowflake-connector-python==2.3.10 -snowflake-sqlalchemy==1.2.4 -sortedcontainers==2.3.0 -soupsieve==2.1 -sphinx-airflow-theme==0.0.2 -sphinx-argparse==0.2.5 +snowflake-connector-python==2.6.1 +snowflake-sqlalchemy==1.2.5 +sortedcontainers==2.4.0 +soupsieve==2.2.1 +sphinx-airflow-theme==0.0.6 +sphinx-argparse==0.3.1 sphinx-autoapi==1.0.0 -sphinx-copybutton==0.3.1 +sphinx-copybutton==0.4.0 sphinx-jinja==1.1.1 -sphinx-rtd-theme==0.5.1 +sphinx-rtd-theme==0.5.2 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-dotnetdomain==0.4 sphinxcontrib-golangdomain==0.2.0.dev0 -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.4 -sphinxcontrib-spelling==5.2.1 -spython==0.0.85 -sshpubkeys==3.3.1 +sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-spelling==7.2.1 +spython==0.1.16 +sqlalchemy-drill==1.1.1 +sqlparse==0.4.2 sshtunnel==0.1.5 -starkbank-ecdsa==1.1.0 +starkbank-ecdsa==1.1.1 statsd==3.3.0 -swagger-ui-bundle==0.0.8 -tableauserverclient==0.14.1 -tabulate==0.8.7 +swagger-ui-bundle==0.0.9 +tableauserverclient==0.16.0 +tabulate==0.8.9 tblib==1.7.0 tenacity==6.2.0 termcolor==1.1.0 -testfixtures==6.17.1 text-unidecode==1.3 textwrap3==0.9.2 -thrift-sasl==0.4.2 +thrift-sasl==0.4.3 thrift==0.13.0 toml==0.10.2 +tomli==1.2.1 toolz==0.11.1 tornado==6.1 -tqdm==4.56.0 +tqdm==4.62.2 traitlets==4.3.3 -typed-ast==1.4.2 +trino==0.306.0 +typed-ast==1.4.3 typing-extensions==3.7.4.3 -typing-inspect==0.6.0 +typing-inspect==0.7.1 typing==3.7.4.3 -tzlocal==2.1 +tzlocal==3.0 unicodecsv==0.14.1 uritemplate==3.0.1 -urllib3==1.25.11 +urllib3==1.26.6 vertica-python==1.0.1 vine==1.3.0 -virtualenv==20.4.2 +virtualenv==20.7.2 volatile==2.1.0 -watchtower==0.7.3 +watchtower==1.0.6 wcwidth==0.2.5 -websocket-client==0.57.0 +websocket-client==1.2.1 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.26.0 -yandexcloud==0.72.0 +yamllint==1.26.3 +yandexcloud==0.105.0 yarl==1.6.3 zdesk==2.7.1 zict==2.0.0 -zipp==3.4.0 +zipp==3.5.0 zope.event==4.5.0 -zope.interface==5.2.0 \ No newline at end of file +zope.interface==5.4.0 \ No newline at end of file diff --git a/packaging/constraints/constraints-3.7.txt b/packaging/constraints/constraints-3.7.txt index 8469842..40c917e 100644 --- a/packaging/constraints/constraints-3.7.txt +++ b/packaging/constraints/constraints-3.7.txt @@ -1,243 +1,334 @@ -# Editable install with no version control (apache-airflow==2.0.1) +# +# This constraints file was automatically generated on 2021-09-15T22:04:11Z +# via "eager-upgrade" mechanism of PIP. For the "v2-1-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# Editable install with no version control (apache-airflow==2.1.4) APScheduler==3.6.3 -Authlib==0.15.3 -Babel==2.9.0 -Flask-AppBuilder==3.1.1 +Authlib==0.15.4 +Babel==2.9.1 +Deprecated==1.2.13 +Flask-AppBuilder==3.3.3 Flask-Babel==1.0.0 Flask-Bcrypt==0.7.1 -Flask-Caching==1.9.0 -Flask-JWT-Extended==3.25.0 +Flask-Caching==1.10.1 +Flask-JWT-Extended==3.25.1 Flask-Login==0.4.1 -Flask-OAuthlib==0.9.5 -Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.4 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 Flask-WTF==0.14.3 -Flask==1.1.2 -GitPython==3.1.12 +Flask==1.1.4 +GitPython==3.1.18 HeapDict==1.0.1 -JPype1==1.2.1 +JPype1==1.3.0 JayDeBeApi==1.2.3 Jinja2==2.11.3 -Mako==1.1.4 -Markdown==3.3.3 +Mako==1.1.5 +Markdown==3.3.4 MarkupSafe==1.1.1 -PyHive==0.6.3 +PyGithub==1.54.1 +PyHive==0.6.4 PyJWT==1.7.1 PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.4.1 -Pygments==2.7.4 +Pygments==2.10.0 SQLAlchemy-JSONField==1.0.0 -SQLAlchemy-Utils==0.36.8 -SQLAlchemy==1.3.23 +SQLAlchemy-Utils==0.37.8 +SQLAlchemy==1.3.24 +SecretStorage==3.3.1 Sphinx==3.4.3 -Unidecode==1.2.0 +Unidecode==1.3.1 WTForms==2.3.3 Werkzeug==1.0.1 -adal==1.2.6 -aiohttp==3.7.3 +adal==1.2.7 +aiohttp==3.7.4.post0 alabaster==0.7.12 -alembic==1.5.4 +alembic==1.7.1 amqp==2.6.1 -analytics-python==1.2.9 +analytics-python==1.4.0 ansiwrap==0.8.4 -apipkg==1.5 +anyio==3.3.1 +apache-airflow-providers-airbyte==2.1.1 +apache-airflow-providers-amazon==2.2.0 +apache-airflow-providers-apache-beam==3.0.1 +apache-airflow-providers-apache-cassandra==2.0.1 +apache-airflow-providers-apache-drill==1.0.1 +apache-airflow-providers-apache-druid==2.0.2 +apache-airflow-providers-apache-hdfs==2.1.0 +apache-airflow-providers-apache-hive==2.0.2 +apache-airflow-providers-apache-kylin==2.0.1 +apache-airflow-providers-apache-livy==2.1.0 +apache-airflow-providers-apache-pig==2.0.1 +apache-airflow-providers-apache-pinot==2.0.1 +apache-airflow-providers-apache-spark==2.0.1 +apache-airflow-providers-apache-sqoop==2.0.2 +apache-airflow-providers-asana==1.1.0 +apache-airflow-providers-celery==2.0.0 +apache-airflow-providers-cloudant==2.0.1 +apache-airflow-providers-cncf-kubernetes==2.0.2 +apache-airflow-providers-databricks==2.0.1 +apache-airflow-providers-datadog==2.0.1 +apache-airflow-providers-dingding==2.0.1 +apache-airflow-providers-discord==2.0.1 +apache-airflow-providers-docker==2.1.1 +apache-airflow-providers-elasticsearch==2.0.3 +apache-airflow-providers-exasol==2.0.1 +apache-airflow-providers-facebook==2.0.1 +apache-airflow-providers-ftp==2.0.1 +apache-airflow-providers-google==5.1.0 +apache-airflow-providers-grpc==2.0.1 +apache-airflow-providers-hashicorp==2.1.0 +apache-airflow-providers-http==2.0.1 +apache-airflow-providers-imap==2.0.1 +apache-airflow-providers-jdbc==2.0.1 +apache-airflow-providers-jenkins==2.0.2 +apache-airflow-providers-jira==2.0.1 +apache-airflow-providers-microsoft-azure==3.1.1 +apache-airflow-providers-microsoft-mssql==2.0.1 +apache-airflow-providers-microsoft-winrm==2.0.1 +apache-airflow-providers-mongo==2.1.0 +apache-airflow-providers-mysql==2.1.1 +apache-airflow-providers-neo4j==2.0.1 +apache-airflow-providers-odbc==2.0.1 +apache-airflow-providers-openfaas==2.0.0 +apache-airflow-providers-opsgenie==2.0.1 +apache-airflow-providers-oracle==2.0.1 +apache-airflow-providers-pagerduty==2.0.1 +apache-airflow-providers-papermill==2.0.1 +apache-airflow-providers-plexus==2.0.1 +apache-airflow-providers-postgres==2.2.0 +apache-airflow-providers-presto==2.0.1 +apache-airflow-providers-qubole==2.0.1 +apache-airflow-providers-redis==2.0.1 +apache-airflow-providers-salesforce==3.2.0 +apache-airflow-providers-samba==3.0.0 +apache-airflow-providers-segment==2.0.1 +apache-airflow-providers-sendgrid==2.0.1 +apache-airflow-providers-sftp==2.1.1 +apache-airflow-providers-singularity==2.0.1 +apache-airflow-providers-slack==4.0.1 +apache-airflow-providers-snowflake==2.1.1 +apache-airflow-providers-sqlite==2.0.1 +apache-airflow-providers-ssh==2.1.1 +apache-airflow-providers-tableau==2.1.1 +apache-airflow-providers-telegram==2.0.1 +apache-airflow-providers-trino==2.0.1 +apache-airflow-providers-vertica==2.0.1 +apache-airflow-providers-yandex==2.1.0 +apache-airflow-providers-zendesk==2.0.1 +apache-beam==2.32.0 apispec==3.3.2 appdirs==1.4.4 -argcomplete==1.12.2 -arrow==0.17.0 +argcomplete==1.12.3 +arrow==1.1.1 +asana==0.10.3 asn1crypto==1.4.0 -astroid==2.4.2 -async-generator==1.10 +astroid==2.7.3 async-timeout==3.0.1 atlasclient==1.0.0 attrs==20.3.0 -aws-sam-translator==1.34.0 -aws-xray-sdk==2.6.0 -azure-batch==10.0.0 -azure-common==1.1.26 -azure-core==1.11.0 +avro-python3==1.9.2.1 +aws-xray-sdk==2.8.0 +azure-batch==11.0.0 +azure-common==1.1.27 +azure-core==1.18.0 azure-cosmos==3.2.0 -azure-datalake-store==0.0.51 -azure-identity==1.5.0 -azure-keyvault-certificates==4.2.1 -azure-keyvault-keys==4.3.1 -azure-keyvault-secrets==4.2.0 +azure-datalake-store==0.0.52 +azure-identity==1.6.1 +azure-keyvault-certificates==4.3.0 +azure-keyvault-keys==4.4.0 +azure-keyvault-secrets==4.3.0 azure-keyvault==4.1.0 azure-kusto-data==0.0.45 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.2.2 +azure-mgmt-core==1.3.0 +azure-mgmt-datafactory==1.1.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==15.0.0 +azure-mgmt-resource==20.0.0 azure-nspkg==3.0.2 -azure-storage-blob==12.7.1 +azure-storage-blob==12.8.1 azure-storage-common==2.1.0 azure-storage-file==2.1.0 backcall==0.2.0 +backoff==1.10.0 +backports.entry-points-selectable==1.1.0 +backports.zoneinfo==0.2.1 bcrypt==3.2.0 beautifulsoup4==4.7.1 -billiard==3.6.3.0 -black==20.8b1 +billiard==3.6.4.0 +black==21.7b0 blinker==1.4 -boto3==1.15.18 +boto3==1.17.112 boto==2.49.0 -botocore==1.18.18 +botocore==1.20.112 bowler==0.9.0 cached-property==1.5.2 -cachetools==4.2.1 -cassandra-driver==3.20.2 -cattrs==1.2.0 +cachetools==4.2.2 +cassandra-driver==3.25.0 +cattrs==1.5.0 celery==4.4.7 certifi==2020.12.5 -cffi==1.14.4 -cfgv==3.2.0 -cfn-lint==0.44.7 -cgroupspy==0.1.6 -chardet==3.0.4 +cffi==1.14.6 +cfgv==3.3.1 +cgroupspy==0.2.1 +chardet==4.0.0 +charset-normalizer==2.0.5 click==7.1.2 clickclick==20.10.2 -cloudant==2.14.0 +cloudant==2.15.0 cloudpickle==1.4.1 colorama==0.4.4 -colorlog==4.7.2 +colorlog==4.8.0 commonmark==0.9.1 -connexion==2.7.0 -coverage==5.4 -croniter==0.3.37 -cryptography==3.4.3 +coverage==5.5 +crcmod==1.7 +croniter==1.0.15 +cryptography==3.4.8 curlify==2.2.1 -cx-Oracle==8.1.0 -dask==2021.2.0 -datadog==0.39.0 -decorator==4.4.2 -defusedxml==0.6.0 -dill==0.3.3 -distlib==0.3.1 +cx-Oracle==8.2.1 +dask==2021.6.0 +datadog==0.42.0 +decorator==5.1.0 +defusedxml==0.7.1 +dill==0.3.1.1 +distlib==0.3.2 distributed==2.19.0 dnspython==1.16.0 -docker-pycreds==0.4.0 -docker==3.7.3 +docker==5.0.2 docopt==0.6.2 docutils==0.16 -ecdsa==0.14.1 -elasticsearch-dbapi==0.1.0 -elasticsearch-dsl==7.3.0 -elasticsearch==7.5.1 -email-validator==1.1.2 +ecdsa==0.17.0 +elasticsearch-dbapi==0.2.5 +elasticsearch-dsl==7.4.0 +elasticsearch==7.14.1 +email-validator==1.1.3 entrypoints==0.3 -eventlet==0.30.1 -execnet==1.8.0 -facebook-business==9.0.2 -fastavro==1.3.1 +eventlet==0.32.0 +execnet==1.9.0 +facebook-business==11.0.0 +fastavro==1.4.4 filelock==3.0.12 -fissix==20.8.0 +fissix==21.6.6 flake8-colors==0.1.9 -flake8==3.8.4 +flake8==3.9.2 flaky==3.7.0 flower==0.9.7 freezegun==1.1.0 -fsspec==0.8.5 +fsspec==2021.8.1 future==0.18.2 -gcsfs==0.7.1 -gevent==21.1.2 -gitdb==4.0.5 -github3.py==1.3.0 -google-ads==7.0.0 -google-api-core==1.26.0 +gcsfs==2021.8.1 +geomet==0.2.1.post1 +gevent==21.8.0 +gitdb==4.0.7 +github3.py==2.0.0 +google-ads==14.0.0 +google-api-core==1.31.2 google-api-python-client==1.12.8 -google-auth-httplib2==0.0.4 -google-auth-oauthlib==0.4.2 -google-auth==1.25.0 -google-cloud-automl==1.0.1 -google-cloud-bigquery-datatransfer==1.1.1 -google-cloud-bigquery-storage==2.2.1 -google-cloud-bigquery==2.7.0 -google-cloud-bigtable==1.6.1 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.4.6 +google-auth==1.35.0 +google-cloud-appengine-logging==0.1.4 +google-cloud-audit-log==0.1.1 +google-cloud-automl==2.4.2 +google-cloud-bigquery-datatransfer==3.3.2 +google-cloud-bigquery-storage==2.8.0 +google-cloud-bigquery==2.26.0 +google-cloud-bigtable==1.7.0 google-cloud-container==1.0.1 -google-cloud-core==1.6.0 -google-cloud-datacatalog==0.7.0 -google-cloud-dataproc==1.1.1 +google-cloud-core==1.7.2 +google-cloud-datacatalog==3.4.1 +google-cloud-dataproc==2.5.0 google-cloud-dlp==1.0.0 -google-cloud-kms==1.4.0 +google-cloud-kms==2.6.0 google-cloud-language==1.3.0 -google-cloud-logging==1.15.1 -google-cloud-memcache==0.2.0 -google-cloud-monitoring==1.1.0 -google-cloud-os-login==1.0.0 -google-cloud-pubsub==1.7.0 -google-cloud-redis==1.0.0 +google-cloud-logging==2.6.0 +google-cloud-memcache==1.0.0 +google-cloud-monitoring==2.5.0 +google-cloud-os-login==2.3.1 +google-cloud-pubsub==2.8.0 +google-cloud-redis==2.2.2 google-cloud-secret-manager==1.0.0 google-cloud-spanner==1.19.1 google-cloud-speech==1.3.2 -google-cloud-storage==1.35.1 -google-cloud-tasks==1.5.0 +google-cloud-storage==1.42.1 +google-cloud-tasks==2.5.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==1.7.0 google-cloud-videointelligence==1.16.1 google-cloud-vision==1.0.0 +google-cloud-workflows==1.2.1 google-crc32c==1.1.2 -google-resumable-media==1.2.0 -googleapis-common-protos==1.52.0 -graphviz==0.16 -greenlet==1.0.0 +google-resumable-media==2.0.2 +googleapis-common-protos==1.53.0 +graphviz==0.17 +greenlet==1.1.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 -grpcio==1.35.0 -gunicorn==19.10.0 -hdfs==2.5.8 +grpcio==1.40.0 +gunicorn==20.1.0 +h11==0.12.0 +hdfs==2.6.0 hmsclient==0.1.1 -httplib2==0.19.0 -humanize==3.2.0 -hvac==0.10.8 -identify==1.5.13 -idna==2.10 +httpcore==0.13.7 +httplib2==0.19.1 +httpx==0.19.0 +humanize==3.11.0 +hvac==0.11.0 +identify==2.2.14 +idna==3.2 +ijson==3.1.4 imagesize==1.2.0 -importlib-metadata==1.7.0 +importlib-metadata==4.8.1 importlib-resources==1.5.0 inflection==0.5.1 iniconfig==1.1.1 -ipdb==0.13.4 +ipdb==0.13.9 ipython-genutils==0.2.0 -ipython==7.20.0 -iso8601==0.1.14 +ipython==7.27.0 +iso8601==0.1.16 isodate==0.6.0 -isort==5.7.0 itsdangerous==1.1.0 jedi==0.18.0 -jira==2.0.0 +jeepney==0.7.1 +jira==3.0.1 jmespath==0.10.0 json-merge-patch==0.2 -jsondiff==1.2.0 -jsonpatch==1.28 -jsonpath-ng==1.5.2 -jsonpickle==2.0.0 -jsonpointer==2.0 +jsondiff==1.3.0 +jsonpath-ng==1.5.3 jsonschema==3.2.0 -junit-xml==1.9 -jupyter-client==6.1.11 +jupyter-client==7.0.2 jupyter-core==4.7.1 -jwcrypto==0.8 +jwcrypto==1.0 +keyring==23.2.1 kombu==4.6.11 kubernetes==11.0.0 kylinpy==2.8.4 lazy-object-proxy==1.4.3 -ldap3==2.9 -libcst==0.3.17 +ldap3==2.9.1 +libcst==0.3.20 +locket==0.2.1 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-oneofschema==2.1.0 +marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.23.1 -marshmallow==3.10.0 +marshmallow==3.13.0 +matplotlib-inline==0.1.3 mccabe==0.6.1 -mock==4.0.2 -mongomock==3.22.1 -more-itertools==8.7.0 -moreorless==0.3.0 -moto==1.3.16 +mongomock==3.23.0 +monotonic==1.6 +more-itertools==8.9.0 +moreorless==0.4.0 +moto==2.2.6 msal-extensions==0.3.0 -msal==1.9.0 +msal==1.14.0 msgpack==1.0.2 msrest==0.6.21 msrestazure==0.6.4 @@ -246,180 +337,194 @@ multidict==5.1.0 mypy-extensions==0.4.3 mypy==0.770 mysql-connector-python==8.0.22 -mysqlclient==1.3.14 -natsort==7.1.1 -nbclient==0.5.1 -nbformat==5.1.2 +mysqlclient==2.0.3 +nbclient==0.5.4 +nbformat==5.1.3 +neo4j==4.3.4 nest-asyncio==1.5.1 -networkx==2.5 -nodeenv==1.5.0 -nteract-scrapbook==0.4.2 +nodeenv==1.6.0 +nox==2020.12.31 ntlm-auth==1.5.0 -numpy==1.20.1 -oauthlib==2.1.0 -openapi-spec-validator==0.2.9 +numpy==1.20.3 +oauth2client==4.1.3 +oauthlib==3.1.1 +openapi-schema-validator==0.1.5 +openapi-spec-validator==0.3.1 +orjson==3.6.3 oscrypto==1.2.1 -packaging==20.9 +packaging==21.0 pandas-gbq==0.14.1 -pandas==1.2.1 -papermill==2.3.2 +pandas==1.3.3 +papermill==2.3.3 parameterized==0.8.1 paramiko==2.7.2 -parso==0.8.1 -pathspec==0.8.1 -pbr==5.5.1 -pdpyras==4.1.2 +parso==0.8.2 +partd==1.2.0 +pathspec==0.9.0 +pbr==5.6.0 +pdpyras==4.3.0 pendulum==2.1.2 pexpect==4.8.0 pickleshare==0.7.5 -pinotdb==0.3.3 -pipdeptree==2.0.0 -pluggy==0.13.1 +pinotdb==0.3.7 +pipdeptree==2.1.0 +platformdirs==2.3.0 +pluggy==1.0.0 ply==3.11 +plyvel==1.3.0 portalocker==1.7.1 -pre-commit==2.10.1 +pre-commit==2.15.0 presto-python-client==0.7.0 -prison==0.1.3 +prison==0.2.1 prometheus-client==0.8.0 -prompt-toolkit==3.0.14 -proto-plus==1.13.0 -protobuf==3.14.0 +prompt-toolkit==3.0.20 +proto-plus==1.18.1 +protobuf==3.18.0 psutil==5.8.0 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.1 ptyprocess==0.7.0 +pure-sasl==0.6.2 py4j==0.10.9 py==1.10.0 -pyOpenSSL==19.1.0 -pyarrow==3.0.0 +pyOpenSSL==20.0.1 +pyarrow==4.0.1 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycodestyle==2.6.0 +pycodestyle==2.7.0 pycountry==20.7.3 pycparser==2.20 -pycryptodomex==3.9.9 -pydata-google-auth==1.1.0 +pycryptodomex==3.10.1 +pydata-google-auth==1.2.0 +pydot==1.4.2 pydruid==0.6.2 -pyenchant==3.2.0 -pyexasol==0.17.0 -pyflakes==2.2.0 +pyenchant==3.2.1 +pyexasol==0.20.0 +pyflakes==2.3.1 pykerberos==1.2.1 -pylint==2.6.0 -pymongo==3.11.3 -pymssql==2.1.5 -pyodbc==4.0.30 +pymongo==3.12.0 +pymssql==2.2.2 +pyodbc==4.0.32 pyparsing==2.4.7 -pyrsistent==0.17.3 +pyrsistent==0.18.0 pysftp==0.2.9 -pyspark==3.0.1 -pytest-cov==2.11.1 +pyspark==3.1.2 +pyspnego==0.1.6 +pytest-cov==2.12.1 pytest-forked==1.3.0 +pytest-httpx==0.13.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.1.1 pytest-timeouts==1.2.1 -pytest-xdist==2.2.0 -pytest==6.2.2 -python-daemon==2.2.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-http-client==3.3.1 +pytest-xdist==2.3.0 +pytest==6.2.5 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-http-client==3.3.2 python-jenkins==1.7.0 -python-jose==3.2.0 +python-jose==3.3.0 python-ldap==3.3.1 python-nvd3==0.15.0 python-slugify==4.0.1 -python-telegram-bot==13.0 +python-telegram-bot==13.7 python3-openid==3.2.0 -pytz==2020.5 +pytz==2021.1 pytzdata==2020.1 -pywinrm==0.4.1 -pyzmq==22.0.2 +pywinrm==0.4.2 +pyzmq==22.2.1 qds-sdk==1.16.1 redis==3.5.3 -regex==2020.11.13 +regex==2021.8.28 requests-kerberos==0.12.0 -requests-mock==1.8.0 +requests-mock==1.9.3 requests-ntlm==1.1.0 -requests-oauthlib==1.1.0 +requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 -requests==2.25.1 -responses==0.12.1 -rich==9.2.0 -rsa==4.7 -s3transfer==0.3.4 -sasl==0.2.1 +requests==2.26.0 +responses==0.13.4 +rfc3986==1.5.0 +rich==10.9.0 +rsa==4.7.2 +s3transfer==0.4.2 +sasl==0.3.1 +scrapbook==0.5.0 semver==2.13.0 -sendgrid==6.5.0 +sendgrid==6.8.1 sentinels==1.0.0 -sentry-sdk==0.19.5 +sentry-sdk==1.3.1 setproctitle==1.2.2 -simple-salesforce==1.10.1 -six==1.15.0 -slackclient==2.9.3 -smmap==3.0.5 +simple-salesforce==1.11.4 +six==1.16.0 +slack-sdk==3.11.0 +smbprotocol==1.6.2 +smmap==4.0.0 +snakebite-py3==3.0.5 +sniffio==1.2.0 snowballstemmer==2.1.0 -snowflake-connector-python==2.3.10 -snowflake-sqlalchemy==1.2.4 -sortedcontainers==2.3.0 -soupsieve==2.1 -sphinx-airflow-theme==0.0.2 -sphinx-argparse==0.2.5 +snowflake-connector-python==2.6.1 +snowflake-sqlalchemy==1.2.5 +sortedcontainers==2.4.0 +soupsieve==2.2.1 +sphinx-airflow-theme==0.0.6 +sphinx-argparse==0.3.1 sphinx-autoapi==1.0.0 -sphinx-copybutton==0.3.1 +sphinx-copybutton==0.4.0 sphinx-jinja==1.1.1 -sphinx-rtd-theme==0.5.1 +sphinx-rtd-theme==0.5.2 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-dotnetdomain==0.4 sphinxcontrib-golangdomain==0.2.0.dev0 -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.4 -sphinxcontrib-spelling==5.2.1 -spython==0.0.85 -sshpubkeys==3.3.1 +sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-spelling==7.2.1 +spython==0.1.16 +sqlalchemy-drill==1.1.1 +sqlparse==0.4.2 sshtunnel==0.1.5 -starkbank-ecdsa==1.1.0 +starkbank-ecdsa==1.1.1 statsd==3.3.0 -swagger-ui-bundle==0.0.8 -tableauserverclient==0.14.1 -tabulate==0.8.7 +swagger-ui-bundle==0.0.9 +tableauserverclient==0.16.0 +tabulate==0.8.9 tblib==1.7.0 tenacity==6.2.0 termcolor==1.1.0 -testfixtures==6.17.1 text-unidecode==1.3 textwrap3==0.9.2 -thrift-sasl==0.4.2 +thrift-sasl==0.4.3 thrift==0.13.0 toml==0.10.2 +tomli==1.2.1 toolz==0.11.1 tornado==6.1 -tqdm==4.56.0 -traitlets==5.0.5 -typed-ast==1.4.2 +tqdm==4.62.2 +traitlets==5.1.0 +trino==0.306.0 +typed-ast==1.4.3 typing-extensions==3.7.4.3 -typing-inspect==0.6.0 -tzlocal==2.1 +typing-inspect==0.7.1 +tzlocal==3.0 unicodecsv==0.14.1 uritemplate==3.0.1 -urllib3==1.25.11 +urllib3==1.26.6 vertica-python==1.0.1 vine==1.3.0 -virtualenv==20.4.2 +virtualenv==20.7.2 volatile==2.1.0 -watchtower==0.7.3 +watchtower==1.0.6 wcwidth==0.2.5 -websocket-client==0.57.0 +websocket-client==1.2.1 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.26.0 -yandexcloud==0.72.0 +yamllint==1.26.3 +yandexcloud==0.105.0 yarl==1.6.3 zdesk==2.7.1 zict==2.0.0 -zipp==3.4.0 +zipp==3.5.0 zope.event==4.5.0 -zope.interface==5.2.0 \ No newline at end of file +zope.interface==5.4.0 \ No newline at end of file diff --git a/packaging/constraints/constraints-3.8.txt b/packaging/constraints/constraints-3.8.txt index 8469842..e09a206 100644 --- a/packaging/constraints/constraints-3.8.txt +++ b/packaging/constraints/constraints-3.8.txt @@ -1,243 +1,334 @@ -# Editable install with no version control (apache-airflow==2.0.1) +# +# This constraints file was automatically generated on 2021-09-15T22:04:03Z +# via "eager-upgrade" mechanism of PIP. For the "v2-1-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# Editable install with no version control (apache-airflow==2.1.4) APScheduler==3.6.3 -Authlib==0.15.3 -Babel==2.9.0 -Flask-AppBuilder==3.1.1 +Authlib==0.15.4 +Babel==2.9.1 +Deprecated==1.2.13 +Flask-AppBuilder==3.3.3 Flask-Babel==1.0.0 Flask-Bcrypt==0.7.1 -Flask-Caching==1.9.0 -Flask-JWT-Extended==3.25.0 +Flask-Caching==1.10.1 +Flask-JWT-Extended==3.25.1 Flask-Login==0.4.1 -Flask-OAuthlib==0.9.5 -Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.4 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 Flask-WTF==0.14.3 -Flask==1.1.2 -GitPython==3.1.12 +Flask==1.1.4 +GitPython==3.1.18 HeapDict==1.0.1 -JPype1==1.2.1 +JPype1==1.3.0 JayDeBeApi==1.2.3 Jinja2==2.11.3 -Mako==1.1.4 -Markdown==3.3.3 +Mako==1.1.5 +Markdown==3.3.4 MarkupSafe==1.1.1 -PyHive==0.6.3 +PyGithub==1.54.1 +PyHive==0.6.4 PyJWT==1.7.1 PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.4.1 -Pygments==2.7.4 +Pygments==2.10.0 SQLAlchemy-JSONField==1.0.0 -SQLAlchemy-Utils==0.36.8 -SQLAlchemy==1.3.23 +SQLAlchemy-Utils==0.37.8 +SQLAlchemy==1.3.24 +SecretStorage==3.3.1 Sphinx==3.4.3 -Unidecode==1.2.0 +Unidecode==1.3.1 WTForms==2.3.3 Werkzeug==1.0.1 -adal==1.2.6 -aiohttp==3.7.3 +adal==1.2.7 +aiohttp==3.7.4.post0 alabaster==0.7.12 -alembic==1.5.4 +alembic==1.7.1 amqp==2.6.1 -analytics-python==1.2.9 +analytics-python==1.4.0 ansiwrap==0.8.4 -apipkg==1.5 +anyio==3.3.1 +apache-airflow-providers-airbyte==2.1.1 +apache-airflow-providers-amazon==2.2.0 +apache-airflow-providers-apache-beam==3.0.1 +apache-airflow-providers-apache-cassandra==2.0.1 +apache-airflow-providers-apache-drill==1.0.1 +apache-airflow-providers-apache-druid==2.0.2 +apache-airflow-providers-apache-hdfs==2.1.0 +apache-airflow-providers-apache-hive==2.0.2 +apache-airflow-providers-apache-kylin==2.0.1 +apache-airflow-providers-apache-livy==2.1.0 +apache-airflow-providers-apache-pig==2.0.1 +apache-airflow-providers-apache-pinot==2.0.1 +apache-airflow-providers-apache-spark==2.0.1 +apache-airflow-providers-apache-sqoop==2.0.2 +apache-airflow-providers-asana==1.1.0 +apache-airflow-providers-celery==2.0.0 +apache-airflow-providers-cloudant==2.0.1 +apache-airflow-providers-cncf-kubernetes==2.0.2 +apache-airflow-providers-databricks==2.0.1 +apache-airflow-providers-datadog==2.0.1 +apache-airflow-providers-dingding==2.0.1 +apache-airflow-providers-discord==2.0.1 +apache-airflow-providers-docker==2.1.1 +apache-airflow-providers-elasticsearch==2.0.3 +apache-airflow-providers-exasol==2.0.1 +apache-airflow-providers-facebook==2.0.1 +apache-airflow-providers-ftp==2.0.1 +apache-airflow-providers-google==5.1.0 +apache-airflow-providers-grpc==2.0.1 +apache-airflow-providers-hashicorp==2.1.0 +apache-airflow-providers-http==2.0.1 +apache-airflow-providers-imap==2.0.1 +apache-airflow-providers-jdbc==2.0.1 +apache-airflow-providers-jenkins==2.0.2 +apache-airflow-providers-jira==2.0.1 +apache-airflow-providers-microsoft-azure==3.1.1 +apache-airflow-providers-microsoft-mssql==2.0.1 +apache-airflow-providers-microsoft-winrm==2.0.1 +apache-airflow-providers-mongo==2.1.0 +apache-airflow-providers-mysql==2.1.1 +apache-airflow-providers-neo4j==2.0.1 +apache-airflow-providers-odbc==2.0.1 +apache-airflow-providers-openfaas==2.0.0 +apache-airflow-providers-opsgenie==2.0.1 +apache-airflow-providers-oracle==2.0.1 +apache-airflow-providers-pagerduty==2.0.1 +apache-airflow-providers-papermill==2.0.1 +apache-airflow-providers-plexus==2.0.1 +apache-airflow-providers-postgres==2.2.0 +apache-airflow-providers-presto==2.0.1 +apache-airflow-providers-qubole==2.0.1 +apache-airflow-providers-redis==2.0.1 +apache-airflow-providers-salesforce==3.2.0 +apache-airflow-providers-samba==3.0.0 +apache-airflow-providers-segment==2.0.1 +apache-airflow-providers-sendgrid==2.0.1 +apache-airflow-providers-sftp==2.1.1 +apache-airflow-providers-singularity==2.0.1 +apache-airflow-providers-slack==4.0.1 +apache-airflow-providers-snowflake==2.1.1 +apache-airflow-providers-sqlite==2.0.1 +apache-airflow-providers-ssh==2.1.1 +apache-airflow-providers-tableau==2.1.1 +apache-airflow-providers-telegram==2.0.1 +apache-airflow-providers-trino==2.0.1 +apache-airflow-providers-vertica==2.0.1 +apache-airflow-providers-yandex==2.1.0 +apache-airflow-providers-zendesk==2.0.1 +apache-beam==2.32.0 apispec==3.3.2 appdirs==1.4.4 -argcomplete==1.12.2 -arrow==0.17.0 +argcomplete==1.12.3 +arrow==1.1.1 +asana==0.10.3 asn1crypto==1.4.0 -astroid==2.4.2 -async-generator==1.10 +astroid==2.7.3 async-timeout==3.0.1 atlasclient==1.0.0 attrs==20.3.0 -aws-sam-translator==1.34.0 -aws-xray-sdk==2.6.0 -azure-batch==10.0.0 -azure-common==1.1.26 -azure-core==1.11.0 +avro-python3==1.9.2.1 +aws-xray-sdk==2.8.0 +azure-batch==11.0.0 +azure-common==1.1.27 +azure-core==1.18.0 azure-cosmos==3.2.0 -azure-datalake-store==0.0.51 -azure-identity==1.5.0 -azure-keyvault-certificates==4.2.1 -azure-keyvault-keys==4.3.1 -azure-keyvault-secrets==4.2.0 +azure-datalake-store==0.0.52 +azure-identity==1.6.1 +azure-keyvault-certificates==4.3.0 +azure-keyvault-keys==4.4.0 +azure-keyvault-secrets==4.3.0 azure-keyvault==4.1.0 azure-kusto-data==0.0.45 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.2.2 +azure-mgmt-core==1.3.0 +azure-mgmt-datafactory==1.1.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==15.0.0 +azure-mgmt-resource==20.0.0 azure-nspkg==3.0.2 -azure-storage-blob==12.7.1 +azure-storage-blob==12.8.1 azure-storage-common==2.1.0 azure-storage-file==2.1.0 backcall==0.2.0 +backoff==1.10.0 +backports.entry-points-selectable==1.1.0 +backports.zoneinfo==0.2.1 bcrypt==3.2.0 beautifulsoup4==4.7.1 -billiard==3.6.3.0 -black==20.8b1 +billiard==3.6.4.0 +black==21.7b0 blinker==1.4 -boto3==1.15.18 +boto3==1.17.112 boto==2.49.0 -botocore==1.18.18 +botocore==1.20.112 bowler==0.9.0 cached-property==1.5.2 -cachetools==4.2.1 -cassandra-driver==3.20.2 -cattrs==1.2.0 +cachetools==4.2.2 +cassandra-driver==3.25.0 +cattrs==1.5.0 celery==4.4.7 certifi==2020.12.5 -cffi==1.14.4 -cfgv==3.2.0 -cfn-lint==0.44.7 -cgroupspy==0.1.6 -chardet==3.0.4 +cffi==1.14.6 +cfgv==3.3.1 +cgroupspy==0.2.1 +chardet==4.0.0 +charset-normalizer==2.0.5 click==7.1.2 clickclick==20.10.2 -cloudant==2.14.0 +cloudant==2.15.0 cloudpickle==1.4.1 colorama==0.4.4 -colorlog==4.7.2 +colorlog==4.8.0 commonmark==0.9.1 -connexion==2.7.0 -coverage==5.4 -croniter==0.3.37 -cryptography==3.4.3 +coverage==5.5 +crcmod==1.7 +croniter==1.0.15 +cryptography==3.4.8 curlify==2.2.1 -cx-Oracle==8.1.0 -dask==2021.2.0 -datadog==0.39.0 -decorator==4.4.2 -defusedxml==0.6.0 -dill==0.3.3 -distlib==0.3.1 +cx-Oracle==8.2.1 +dask==2021.6.0 +datadog==0.42.0 +decorator==5.1.0 +defusedxml==0.7.1 +dill==0.3.1.1 +distlib==0.3.2 distributed==2.19.0 dnspython==1.16.0 -docker-pycreds==0.4.0 -docker==3.7.3 +docker==5.0.2 docopt==0.6.2 docutils==0.16 -ecdsa==0.14.1 -elasticsearch-dbapi==0.1.0 -elasticsearch-dsl==7.3.0 -elasticsearch==7.5.1 -email-validator==1.1.2 +ecdsa==0.17.0 +elasticsearch-dbapi==0.2.5 +elasticsearch-dsl==7.4.0 +elasticsearch==7.14.1 +email-validator==1.1.3 entrypoints==0.3 -eventlet==0.30.1 -execnet==1.8.0 -facebook-business==9.0.2 -fastavro==1.3.1 +eventlet==0.32.0 +execnet==1.9.0 +facebook-business==11.0.0 +fastavro==1.4.4 filelock==3.0.12 -fissix==20.8.0 +fissix==21.6.6 flake8-colors==0.1.9 -flake8==3.8.4 +flake8==3.9.2 flaky==3.7.0 flower==0.9.7 freezegun==1.1.0 -fsspec==0.8.5 +fsspec==2021.8.1 future==0.18.2 -gcsfs==0.7.1 -gevent==21.1.2 -gitdb==4.0.5 -github3.py==1.3.0 -google-ads==7.0.0 -google-api-core==1.26.0 +gcsfs==2021.8.1 +geomet==0.2.1.post1 +gevent==21.8.0 +gitdb==4.0.7 +github3.py==2.0.0 +google-ads==14.0.0 +google-api-core==1.31.2 google-api-python-client==1.12.8 -google-auth-httplib2==0.0.4 -google-auth-oauthlib==0.4.2 -google-auth==1.25.0 -google-cloud-automl==1.0.1 -google-cloud-bigquery-datatransfer==1.1.1 -google-cloud-bigquery-storage==2.2.1 -google-cloud-bigquery==2.7.0 -google-cloud-bigtable==1.6.1 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.4.6 +google-auth==1.35.0 +google-cloud-appengine-logging==0.1.4 +google-cloud-audit-log==0.1.1 +google-cloud-automl==2.4.2 +google-cloud-bigquery-datatransfer==3.3.2 +google-cloud-bigquery-storage==2.8.0 +google-cloud-bigquery==2.26.0 +google-cloud-bigtable==1.7.0 google-cloud-container==1.0.1 -google-cloud-core==1.6.0 -google-cloud-datacatalog==0.7.0 -google-cloud-dataproc==1.1.1 +google-cloud-core==1.7.2 +google-cloud-datacatalog==3.4.1 +google-cloud-dataproc==2.5.0 google-cloud-dlp==1.0.0 -google-cloud-kms==1.4.0 +google-cloud-kms==2.6.0 google-cloud-language==1.3.0 -google-cloud-logging==1.15.1 -google-cloud-memcache==0.2.0 -google-cloud-monitoring==1.1.0 -google-cloud-os-login==1.0.0 -google-cloud-pubsub==1.7.0 -google-cloud-redis==1.0.0 +google-cloud-logging==2.6.0 +google-cloud-memcache==1.0.0 +google-cloud-monitoring==2.5.0 +google-cloud-os-login==2.3.1 +google-cloud-pubsub==2.8.0 +google-cloud-redis==2.2.2 google-cloud-secret-manager==1.0.0 google-cloud-spanner==1.19.1 google-cloud-speech==1.3.2 -google-cloud-storage==1.35.1 -google-cloud-tasks==1.5.0 +google-cloud-storage==1.42.1 +google-cloud-tasks==2.5.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==1.7.0 google-cloud-videointelligence==1.16.1 google-cloud-vision==1.0.0 +google-cloud-workflows==1.2.1 google-crc32c==1.1.2 -google-resumable-media==1.2.0 -googleapis-common-protos==1.52.0 -graphviz==0.16 -greenlet==1.0.0 +google-resumable-media==2.0.2 +googleapis-common-protos==1.53.0 +graphviz==0.17 +greenlet==1.1.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 -grpcio==1.35.0 -gunicorn==19.10.0 -hdfs==2.5.8 +grpcio==1.40.0 +gunicorn==20.1.0 +h11==0.12.0 +hdfs==2.6.0 hmsclient==0.1.1 -httplib2==0.19.0 -humanize==3.2.0 -hvac==0.10.8 -identify==1.5.13 -idna==2.10 +httpcore==0.13.7 +httplib2==0.19.1 +httpx==0.19.0 +humanize==3.11.0 +hvac==0.11.0 +identify==2.2.14 +idna==3.2 +ijson==3.1.4 imagesize==1.2.0 -importlib-metadata==1.7.0 +importlib-metadata==4.8.1 importlib-resources==1.5.0 inflection==0.5.1 iniconfig==1.1.1 -ipdb==0.13.4 +ipdb==0.13.9 ipython-genutils==0.2.0 -ipython==7.20.0 -iso8601==0.1.14 +ipython==7.27.0 +iso8601==0.1.16 isodate==0.6.0 -isort==5.7.0 itsdangerous==1.1.0 jedi==0.18.0 -jira==2.0.0 +jeepney==0.7.1 +jira==3.0.1 jmespath==0.10.0 json-merge-patch==0.2 -jsondiff==1.2.0 -jsonpatch==1.28 -jsonpath-ng==1.5.2 -jsonpickle==2.0.0 -jsonpointer==2.0 +jsondiff==1.3.0 +jsonpath-ng==1.5.3 jsonschema==3.2.0 -junit-xml==1.9 -jupyter-client==6.1.11 +jupyter-client==7.0.2 jupyter-core==4.7.1 -jwcrypto==0.8 +jwcrypto==1.0 +keyring==23.2.1 kombu==4.6.11 kubernetes==11.0.0 kylinpy==2.8.4 lazy-object-proxy==1.4.3 -ldap3==2.9 -libcst==0.3.17 +ldap3==2.9.1 +libcst==0.3.20 +locket==0.2.1 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-oneofschema==2.1.0 +marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.23.1 -marshmallow==3.10.0 +marshmallow==3.13.0 +matplotlib-inline==0.1.3 mccabe==0.6.1 -mock==4.0.2 -mongomock==3.22.1 -more-itertools==8.7.0 -moreorless==0.3.0 -moto==1.3.16 +mongomock==3.23.0 +monotonic==1.6 +more-itertools==8.9.0 +moreorless==0.4.0 +moto==2.2.6 msal-extensions==0.3.0 -msal==1.9.0 +msal==1.14.0 msgpack==1.0.2 msrest==0.6.21 msrestazure==0.6.4 @@ -246,180 +337,194 @@ multidict==5.1.0 mypy-extensions==0.4.3 mypy==0.770 mysql-connector-python==8.0.22 -mysqlclient==1.3.14 -natsort==7.1.1 -nbclient==0.5.1 -nbformat==5.1.2 +mysqlclient==2.0.3 +nbclient==0.5.4 +nbformat==5.1.3 +neo4j==4.3.4 nest-asyncio==1.5.1 -networkx==2.5 -nodeenv==1.5.0 -nteract-scrapbook==0.4.2 +nodeenv==1.6.0 +nox==2020.12.31 ntlm-auth==1.5.0 -numpy==1.20.1 -oauthlib==2.1.0 -openapi-spec-validator==0.2.9 +numpy==1.20.3 +oauth2client==4.1.3 +oauthlib==3.1.1 +openapi-schema-validator==0.1.5 +openapi-spec-validator==0.3.1 +orjson==3.6.3 oscrypto==1.2.1 -packaging==20.9 +packaging==21.0 pandas-gbq==0.14.1 -pandas==1.2.1 -papermill==2.3.2 +pandas==1.3.3 +papermill==2.3.3 parameterized==0.8.1 paramiko==2.7.2 -parso==0.8.1 -pathspec==0.8.1 -pbr==5.5.1 -pdpyras==4.1.2 +parso==0.8.2 +partd==1.2.0 +pathspec==0.9.0 +pbr==5.6.0 +pdpyras==4.3.0 pendulum==2.1.2 pexpect==4.8.0 pickleshare==0.7.5 -pinotdb==0.3.3 -pipdeptree==2.0.0 -pluggy==0.13.1 +pinotdb==0.3.7 +pipdeptree==2.1.0 +platformdirs==2.3.0 +pluggy==1.0.0 ply==3.11 +plyvel==1.3.0 portalocker==1.7.1 -pre-commit==2.10.1 +pre-commit==2.15.0 presto-python-client==0.7.0 -prison==0.1.3 +prison==0.2.1 prometheus-client==0.8.0 -prompt-toolkit==3.0.14 -proto-plus==1.13.0 -protobuf==3.14.0 +prompt-toolkit==3.0.20 +proto-plus==1.18.1 +protobuf==3.18.0 psutil==5.8.0 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.1 ptyprocess==0.7.0 +pure-sasl==0.6.2 py4j==0.10.9 py==1.10.0 -pyOpenSSL==19.1.0 -pyarrow==3.0.0 +pyOpenSSL==20.0.1 +pyarrow==4.0.1 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycodestyle==2.6.0 +pycodestyle==2.7.0 pycountry==20.7.3 pycparser==2.20 -pycryptodomex==3.9.9 -pydata-google-auth==1.1.0 +pycryptodomex==3.10.1 +pydata-google-auth==1.2.0 +pydot==1.4.2 pydruid==0.6.2 -pyenchant==3.2.0 -pyexasol==0.17.0 -pyflakes==2.2.0 +pyenchant==3.2.1 +pyexasol==0.20.0 +pyflakes==2.3.1 pykerberos==1.2.1 -pylint==2.6.0 -pymongo==3.11.3 -pymssql==2.1.5 -pyodbc==4.0.30 +pymongo==3.12.0 +pymssql==2.2.2 +pyodbc==4.0.32 pyparsing==2.4.7 -pyrsistent==0.17.3 +pyrsistent==0.18.0 pysftp==0.2.9 -pyspark==3.0.1 -pytest-cov==2.11.1 +pyspark==3.1.2 +pyspnego==0.1.6 +pytest-cov==2.12.1 pytest-forked==1.3.0 +pytest-httpx==0.13.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.1.1 pytest-timeouts==1.2.1 -pytest-xdist==2.2.0 -pytest==6.2.2 -python-daemon==2.2.4 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-http-client==3.3.1 +pytest-xdist==2.3.0 +pytest==6.2.5 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-http-client==3.3.2 python-jenkins==1.7.0 -python-jose==3.2.0 +python-jose==3.3.0 python-ldap==3.3.1 python-nvd3==0.15.0 python-slugify==4.0.1 -python-telegram-bot==13.0 +python-telegram-bot==13.7 python3-openid==3.2.0 -pytz==2020.5 +pytz==2021.1 pytzdata==2020.1 -pywinrm==0.4.1 -pyzmq==22.0.2 +pywinrm==0.4.2 +pyzmq==22.2.1 qds-sdk==1.16.1 redis==3.5.3 -regex==2020.11.13 +regex==2021.8.28 requests-kerberos==0.12.0 -requests-mock==1.8.0 +requests-mock==1.9.3 requests-ntlm==1.1.0 -requests-oauthlib==1.1.0 +requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 -requests==2.25.1 -responses==0.12.1 -rich==9.2.0 -rsa==4.7 -s3transfer==0.3.4 -sasl==0.2.1 +requests==2.26.0 +responses==0.13.4 +rfc3986==1.5.0 +rich==10.9.0 +rsa==4.7.2 +s3transfer==0.4.2 +sasl==0.3.1 +scrapbook==0.5.0 semver==2.13.0 -sendgrid==6.5.0 +sendgrid==6.8.1 sentinels==1.0.0 -sentry-sdk==0.19.5 +sentry-sdk==1.3.1 setproctitle==1.2.2 -simple-salesforce==1.10.1 -six==1.15.0 -slackclient==2.9.3 -smmap==3.0.5 +simple-salesforce==1.11.4 +six==1.16.0 +slack-sdk==3.11.0 +smbprotocol==1.6.2 +smmap==4.0.0 +snakebite-py3==3.0.5 +sniffio==1.2.0 snowballstemmer==2.1.0 -snowflake-connector-python==2.3.10 -snowflake-sqlalchemy==1.2.4 -sortedcontainers==2.3.0 -soupsieve==2.1 -sphinx-airflow-theme==0.0.2 -sphinx-argparse==0.2.5 +snowflake-connector-python==2.6.1 +snowflake-sqlalchemy==1.2.5 +sortedcontainers==2.4.0 +soupsieve==2.2.1 +sphinx-airflow-theme==0.0.6 +sphinx-argparse==0.3.1 sphinx-autoapi==1.0.0 -sphinx-copybutton==0.3.1 +sphinx-copybutton==0.4.0 sphinx-jinja==1.1.1 -sphinx-rtd-theme==0.5.1 +sphinx-rtd-theme==0.5.2 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-dotnetdomain==0.4 sphinxcontrib-golangdomain==0.2.0.dev0 -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.4 -sphinxcontrib-spelling==5.2.1 -spython==0.0.85 -sshpubkeys==3.3.1 +sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-spelling==7.2.1 +spython==0.1.16 +sqlalchemy-drill==1.1.1 +sqlparse==0.4.2 sshtunnel==0.1.5 -starkbank-ecdsa==1.1.0 +starkbank-ecdsa==1.1.1 statsd==3.3.0 -swagger-ui-bundle==0.0.8 -tableauserverclient==0.14.1 -tabulate==0.8.7 +swagger-ui-bundle==0.0.9 +tableauserverclient==0.16.0 +tabulate==0.8.9 tblib==1.7.0 tenacity==6.2.0 termcolor==1.1.0 -testfixtures==6.17.1 text-unidecode==1.3 textwrap3==0.9.2 -thrift-sasl==0.4.2 +thrift-sasl==0.4.3 thrift==0.13.0 toml==0.10.2 +tomli==1.2.1 toolz==0.11.1 tornado==6.1 -tqdm==4.56.0 -traitlets==5.0.5 -typed-ast==1.4.2 +tqdm==4.62.2 +traitlets==5.1.0 +trino==0.306.0 +typed-ast==1.4.3 typing-extensions==3.7.4.3 -typing-inspect==0.6.0 -tzlocal==2.1 +typing-inspect==0.7.1 +tzlocal==3.0 unicodecsv==0.14.1 uritemplate==3.0.1 -urllib3==1.25.11 +urllib3==1.26.6 vertica-python==1.0.1 vine==1.3.0 -virtualenv==20.4.2 +virtualenv==20.7.2 volatile==2.1.0 -watchtower==0.7.3 +watchtower==1.0.6 wcwidth==0.2.5 -websocket-client==0.57.0 +websocket-client==1.2.1 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.26.0 -yandexcloud==0.72.0 +yamllint==1.26.3 +yandexcloud==0.105.0 yarl==1.6.3 zdesk==2.7.1 zict==2.0.0 -zipp==3.4.0 +zipp==3.5.0 zope.event==4.5.0 -zope.interface==5.2.0 \ No newline at end of file +zope.interface==5.4.0 \ No newline at end of file diff --git a/packaging/constraints/constraints-3.9.txt b/packaging/constraints/constraints-3.9.txt new file mode 100644 index 0000000..9d4f75a --- /dev/null +++ b/packaging/constraints/constraints-3.9.txt @@ -0,0 +1,526 @@ +# +# This constraints file was automatically generated on 2021-09-15T22:04:04Z +# via "eager-upgrade" mechanism of PIP. For the "v2-1-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# Editable install with no version control (apache-airflow==2.1.4) +APScheduler==3.6.3 +Authlib==0.15.4 +Babel==2.9.1 +Deprecated==1.2.13 +Flask-AppBuilder==3.3.3 +Flask-Babel==1.0.0 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.10.1 +Flask-JWT-Extended==3.25.1 +Flask-Login==0.4.1 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.14.3 +Flask==1.1.4 +GitPython==3.1.18 +HeapDict==1.0.1 +JPype1==1.3.0 +JayDeBeApi==1.2.3 +Jinja2==2.11.3 +Mako==1.1.5 +Markdown==3.3.4 +MarkupSafe==1.1.1 +PyGithub==1.54.1 +PyJWT==1.7.1 +PyNaCl==1.4.0 +PySmbClient==0.1.5 +PyYAML==5.4.1 +Pygments==2.10.0 +SQLAlchemy-JSONField==1.0.0 +SQLAlchemy-Utils==0.37.8 +SQLAlchemy==1.3.24 +SecretStorage==3.3.1 +Sphinx==3.4.3 +Unidecode==1.3.1 +WTForms==2.3.3 +Werkzeug==1.0.1 +adal==1.2.7 +aiohttp==3.7.4.post0 +alabaster==0.7.12 +alembic==1.7.1 +amqp==2.6.1 +analytics-python==1.4.0 +ansiwrap==0.8.4 +anyio==3.3.1 +apache-airflow-providers-airbyte==2.1.1 +apache-airflow-providers-amazon==2.2.0 +apache-airflow-providers-apache-beam==3.0.1 +apache-airflow-providers-apache-cassandra==2.0.1 +apache-airflow-providers-apache-drill==1.0.1 +apache-airflow-providers-apache-druid==2.0.2 +apache-airflow-providers-apache-hdfs==2.1.0 +apache-airflow-providers-apache-kylin==2.0.1 +apache-airflow-providers-apache-livy==2.1.0 +apache-airflow-providers-apache-pig==2.0.1 +apache-airflow-providers-apache-pinot==2.0.1 +apache-airflow-providers-apache-spark==2.0.1 +apache-airflow-providers-apache-sqoop==2.0.2 +apache-airflow-providers-asana==1.1.0 +apache-airflow-providers-celery==2.0.0 +apache-airflow-providers-cloudant==2.0.1 +apache-airflow-providers-cncf-kubernetes==2.0.2 +apache-airflow-providers-databricks==2.0.1 +apache-airflow-providers-datadog==2.0.1 +apache-airflow-providers-dingding==2.0.1 +apache-airflow-providers-discord==2.0.1 +apache-airflow-providers-docker==2.1.1 +apache-airflow-providers-elasticsearch==2.0.3 +apache-airflow-providers-exasol==2.0.1 +apache-airflow-providers-facebook==2.0.1 +apache-airflow-providers-ftp==2.0.1 +apache-airflow-providers-google==5.1.0 +apache-airflow-providers-grpc==2.0.1 +apache-airflow-providers-hashicorp==2.1.0 +apache-airflow-providers-http==2.0.1 +apache-airflow-providers-imap==2.0.1 +apache-airflow-providers-jdbc==2.0.1 +apache-airflow-providers-jenkins==2.0.2 +apache-airflow-providers-jira==2.0.1 +apache-airflow-providers-microsoft-azure==3.1.1 +apache-airflow-providers-microsoft-mssql==2.0.1 +apache-airflow-providers-microsoft-winrm==2.0.1 +apache-airflow-providers-mongo==2.1.0 +apache-airflow-providers-mysql==2.1.1 +apache-airflow-providers-neo4j==2.0.1 +apache-airflow-providers-odbc==2.0.1 +apache-airflow-providers-openfaas==2.0.0 +apache-airflow-providers-opsgenie==2.0.1 +apache-airflow-providers-oracle==2.0.1 +apache-airflow-providers-pagerduty==2.0.1 +apache-airflow-providers-papermill==2.0.1 +apache-airflow-providers-plexus==2.0.1 +apache-airflow-providers-postgres==2.2.0 +apache-airflow-providers-presto==2.0.1 +apache-airflow-providers-qubole==2.0.1 +apache-airflow-providers-redis==2.0.1 +apache-airflow-providers-salesforce==3.2.0 +apache-airflow-providers-samba==3.0.0 +apache-airflow-providers-segment==2.0.1 +apache-airflow-providers-sendgrid==2.0.1 +apache-airflow-providers-sftp==2.1.1 +apache-airflow-providers-singularity==2.0.1 +apache-airflow-providers-slack==4.0.1 +apache-airflow-providers-snowflake==2.1.1 +apache-airflow-providers-sqlite==2.0.1 +apache-airflow-providers-ssh==2.1.1 +apache-airflow-providers-tableau==2.1.1 +apache-airflow-providers-telegram==2.0.1 +apache-airflow-providers-trino==2.0.1 +apache-airflow-providers-vertica==2.0.1 +apache-airflow-providers-yandex==2.1.0 +apache-airflow-providers-zendesk==2.0.1 +apache-beam==2.32.0 +apispec==3.3.2 +appdirs==1.4.4 +argcomplete==1.12.3 +arrow==1.1.1 +asana==0.10.3 +asn1crypto==1.4.0 +astroid==2.7.3 +async-timeout==3.0.1 +atlasclient==1.0.0 +attrs==20.3.0 +avro-python3==1.9.2.1 +aws-xray-sdk==2.8.0 +azure-batch==11.0.0 +azure-common==1.1.27 +azure-core==1.18.0 +azure-cosmos==3.2.0 +azure-datalake-store==0.0.52 +azure-identity==1.6.1 +azure-keyvault-certificates==4.3.0 +azure-keyvault-keys==4.4.0 +azure-keyvault-secrets==4.3.0 +azure-keyvault==4.1.0 +azure-kusto-data==0.0.45 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-core==1.3.0 +azure-mgmt-datafactory==1.1.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==20.0.0 +azure-nspkg==3.0.2 +azure-storage-blob==12.8.1 +azure-storage-common==2.1.0 +azure-storage-file==2.1.0 +backcall==0.2.0 +backoff==1.10.0 +backports.entry-points-selectable==1.1.0 +bcrypt==3.2.0 +beautifulsoup4==4.7.1 +billiard==3.6.4.0 +black==21.7b0 +blinker==1.4 +boto3==1.17.112 +boto==2.49.0 +botocore==1.20.112 +bowler==0.9.0 +cached-property==1.5.2 +cachetools==4.2.2 +cassandra-driver==3.25.0 +cattrs==1.5.0 +celery==4.4.7 +certifi==2020.12.5 +cffi==1.14.6 +cfgv==3.3.1 +cgroupspy==0.2.1 +chardet==4.0.0 +charset-normalizer==2.0.5 +click==7.1.2 +clickclick==20.10.2 +cloudant==2.15.0 +cloudpickle==1.4.1 +colorama==0.4.4 +colorlog==4.8.0 +commonmark==0.9.1 +coverage==5.5 +crcmod==1.7 +croniter==1.0.15 +cryptography==3.4.8 +curlify==2.2.1 +cx-Oracle==8.2.1 +dask==2021.6.0 +datadog==0.42.0 +decorator==5.1.0 +defusedxml==0.7.1 +dill==0.3.1.1 +distlib==0.3.2 +distributed==2.19.0 +dnspython==1.16.0 +docker==5.0.2 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.17.0 +elasticsearch-dbapi==0.2.5 +elasticsearch-dsl==7.4.0 +elasticsearch==7.14.1 +email-validator==1.1.3 +entrypoints==0.3 +eventlet==0.32.0 +execnet==1.9.0 +facebook-business==11.0.0 +fastavro==1.4.4 +filelock==3.0.12 +fissix==21.6.6 +flake8-colors==0.1.9 +flake8==3.9.2 +flaky==3.7.0 +flower==0.9.7 +freezegun==1.1.0 +fsspec==2021.8.1 +future==0.18.2 +gcsfs==2021.8.1 +geomet==0.2.1.post1 +gevent==21.8.0 +gitdb==4.0.7 +github3.py==2.0.0 +google-ads==14.0.0 +google-api-core==1.31.2 +google-api-python-client==1.12.8 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.4.6 +google-auth==1.35.0 +google-cloud-appengine-logging==0.1.4 +google-cloud-audit-log==0.1.1 +google-cloud-automl==2.4.2 +google-cloud-bigquery-datatransfer==3.3.2 +google-cloud-bigquery-storage==2.8.0 +google-cloud-bigquery==2.26.0 +google-cloud-bigtable==1.7.0 +google-cloud-container==1.0.1 +google-cloud-core==1.7.2 +google-cloud-datacatalog==3.4.1 +google-cloud-dataproc==2.5.0 +google-cloud-dlp==1.0.0 +google-cloud-kms==2.6.0 +google-cloud-language==1.3.0 +google-cloud-logging==2.6.0 +google-cloud-memcache==1.0.0 +google-cloud-monitoring==2.5.0 +google-cloud-os-login==2.3.1 +google-cloud-pubsub==2.8.0 +google-cloud-redis==2.2.2 +google-cloud-secret-manager==1.0.0 +google-cloud-spanner==1.19.1 +google-cloud-speech==1.3.2 +google-cloud-storage==1.42.1 +google-cloud-tasks==2.5.1 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==1.7.0 +google-cloud-videointelligence==1.16.1 +google-cloud-vision==1.0.0 +google-cloud-workflows==1.2.1 +google-crc32c==1.1.2 +google-resumable-media==2.0.2 +googleapis-common-protos==1.53.0 +graphviz==0.17 +greenlet==1.1.1 +grpc-google-iam-v1==0.12.3 +grpcio-gcp==0.2.2 +grpcio==1.40.0 +gunicorn==20.1.0 +h11==0.12.0 +hdfs==2.6.0 +hmsclient==0.1.1 +httpcore==0.13.7 +httplib2==0.19.1 +httpx==0.19.0 +humanize==3.11.0 +hvac==0.11.0 +identify==2.2.14 +idna==3.2 +ijson==3.1.4 +imagesize==1.2.0 +importlib-metadata==4.8.1 +importlib-resources==1.5.0 +inflection==0.5.1 +iniconfig==1.1.1 +ipdb==0.13.9 +ipython-genutils==0.2.0 +ipython==7.27.0 +iso8601==0.1.16 +isodate==0.6.0 +itsdangerous==1.1.0 +jedi==0.18.0 +jeepney==0.7.1 +jira==3.0.1 +jmespath==0.10.0 +json-merge-patch==0.2 +jsondiff==1.3.0 +jsonpath-ng==1.5.3 +jsonschema==3.2.0 +jupyter-client==7.0.2 +jupyter-core==4.7.1 +jwcrypto==1.0 +keyring==23.2.1 +kombu==4.6.11 +kubernetes==11.0.0 +kylinpy==2.8.4 +lazy-object-proxy==1.4.3 +ldap3==2.9.1 +libcst==0.3.20 +locket==0.2.1 +lockfile==0.12.2 +marshmallow-enum==1.5.1 +marshmallow-oneofschema==3.0.1 +marshmallow-sqlalchemy==0.23.1 +marshmallow==3.13.0 +matplotlib-inline==0.1.3 +mccabe==0.6.1 +mongomock==3.23.0 +monotonic==1.6 +more-itertools==8.9.0 +moreorless==0.4.0 +moto==2.2.6 +msal-extensions==0.3.0 +msal==1.14.0 +msgpack==1.0.2 +msrest==0.6.21 +msrestazure==0.6.4 +multi-key-dict==2.0.3 +multidict==5.1.0 +mypy-extensions==0.4.3 +mypy==0.770 +mysql-connector-python==8.0.22 +mysqlclient==2.0.3 +nbclient==0.5.4 +nbformat==5.1.3 +neo4j==4.3.4 +nest-asyncio==1.5.1 +nodeenv==1.6.0 +nox==2020.12.31 +ntlm-auth==1.5.0 +numpy==1.20.3 +oauth2client==4.1.3 +oauthlib==3.1.1 +openapi-schema-validator==0.1.5 +openapi-spec-validator==0.3.1 +orjson==3.6.3 +oscrypto==1.2.1 +packaging==21.0 +pandas-gbq==0.14.1 +pandas==1.3.3 +papermill==2.3.3 +parameterized==0.8.1 +paramiko==2.7.2 +parso==0.8.2 +partd==1.2.0 +pathspec==0.9.0 +pbr==5.6.0 +pdpyras==4.3.0 +pendulum==2.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.3.7 +pipdeptree==2.1.0 +platformdirs==2.3.0 +pluggy==1.0.0 +ply==3.11 +plyvel==1.3.0 +portalocker==1.7.1 +pre-commit==2.15.0 +presto-python-client==0.7.0 +prison==0.2.1 +prometheus-client==0.8.0 +prompt-toolkit==3.0.20 +proto-plus==1.18.1 +protobuf==3.18.0 +psutil==5.8.0 +psycopg2-binary==2.9.1 +ptyprocess==0.7.0 +pure-sasl==0.6.2 +py4j==0.10.9 +py==1.10.0 +pyOpenSSL==20.0.1 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pycodestyle==2.7.0 +pycountry==20.7.3 +pycparser==2.20 +pycryptodomex==3.10.1 +pydata-google-auth==1.2.0 +pydot==1.4.2 +pydruid==0.6.2 +pyenchant==3.2.1 +pyexasol==0.20.0 +pyflakes==2.3.1 +pykerberos==1.2.1 +pymongo==3.12.0 +pymssql==2.2.2 +pyodbc==4.0.32 +pyparsing==2.4.7 +pyrsistent==0.18.0 +pysftp==0.2.9 +pyspark==3.1.2 +pyspnego==0.1.6 +pytest-cov==2.12.1 +pytest-forked==1.3.0 +pytest-httpx==0.13.0 +pytest-instafail==0.4.2 +pytest-rerunfailures==9.1.1 +pytest-timeouts==1.2.1 +pytest-xdist==2.3.0 +pytest==6.2.5 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-http-client==3.3.2 +python-jenkins==1.7.0 +python-jose==3.3.0 +python-ldap==3.3.1 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python-telegram-bot==13.7 +python3-openid==3.2.0 +pytz==2021.1 +pytzdata==2020.1 +pywinrm==0.4.2 +pyzmq==22.2.1 +qds-sdk==1.16.1 +redis==3.5.3 +regex==2021.8.28 +requests-kerberos==0.12.0 +requests-mock==1.9.3 +requests-ntlm==1.1.0 +requests-oauthlib==1.3.0 +requests-toolbelt==0.9.1 +requests==2.26.0 +responses==0.13.4 +rfc3986==1.5.0 +rich==10.9.0 +rsa==4.7.2 +s3transfer==0.4.2 +scrapbook==0.5.0 +semver==2.13.0 +sendgrid==6.8.1 +sentinels==1.0.0 +sentry-sdk==1.3.1 +setproctitle==1.2.2 +simple-salesforce==1.11.4 +six==1.16.0 +slack-sdk==3.11.0 +smbprotocol==1.6.2 +smmap==4.0.0 +snakebite-py3==3.0.5 +sniffio==1.2.0 +snowballstemmer==2.1.0 +snowflake-connector-python==2.6.1 +snowflake-sqlalchemy==1.2.5 +sortedcontainers==2.4.0 +soupsieve==2.2.1 +sphinx-airflow-theme==0.0.6 +sphinx-argparse==0.3.1 +sphinx-autoapi==1.0.0 +sphinx-copybutton==0.4.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.5.2 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-redoc==1.6.0 +sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-spelling==7.2.1 +spython==0.1.16 +sqlalchemy-drill==1.1.1 +sqlparse==0.4.2 +sshtunnel==0.1.5 +starkbank-ecdsa==1.1.1 +statsd==3.3.0 +swagger-ui-bundle==0.0.9 +tableauserverclient==0.16.0 +tabulate==0.8.9 +tblib==1.7.0 +tenacity==6.2.0 +termcolor==1.1.0 +text-unidecode==1.3 +textwrap3==0.9.2 +thrift-sasl==0.4.3 +thrift==0.13.0 +toml==0.10.2 +tomli==1.2.1 +toolz==0.11.1 +tornado==6.1 +tqdm==4.62.2 +traitlets==5.1.0 +trino==0.306.0 +typed-ast==1.4.3 +typing-extensions==3.7.4.3 +typing-inspect==0.7.1 +tzlocal==3.0 +unicodecsv==0.14.1 +uritemplate==3.0.1 +urllib3==1.26.6 +vertica-python==1.0.1 +vine==1.3.0 +virtualenv==20.7.2 +volatile==2.1.0 +watchtower==1.0.6 +wcwidth==0.2.5 +websocket-client==1.2.1 +wrapt==1.12.1 +xmltodict==0.12.0 +yamllint==1.26.3 +yandexcloud==0.105.0 +yarl==1.6.3 +zdesk==2.7.1 +zict==2.0.0 +zipp==3.5.0 +zope.event==4.5.0 +zope.interface==5.4.0 \ No newline at end of file diff --git a/packaging/docker_compose/local_executor/.env b/packaging/docker_compose/local_executor/.env index c022f2f..de431d8 100644 --- a/packaging/docker_compose/local_executor/.env +++ b/packaging/docker_compose/local_executor/.env @@ -35,6 +35,10 @@ # somehow, because docker-cli inside container can use a different API # version from what the docker daemon on the host machine uses. +# By default we build latest commit from master branch of cwl-airflow. +# Optionally it can be controlled through the CWL_AIRFLOW_VERSION parameter, +# that will be passed to the docker build command with --build-arg. +# The same way we can pass UBUNTU_VERSION and PYTHON_VERSION parameters. # All volumes required by CWL-Airflow will be mounted with exactly the same # locations inside docker containers as they are on the host machine. # CWL_TMP_FOLDER and CWL_INPUTS_FOLDER should be accessible from the inner @@ -54,7 +58,14 @@ # AIRFLOW__CORE__PARALLELISM=1 # AIRFLOW__CORE__DAG_CONCURRENCY=1 + +# Ubuntu and Python versions +UBUNTU_VERSION=20.04 +PYTHON_VERSION=3.8.10 + + # CWL-Airflow +CWL_AIRFLOW_VERSION=master AIRFLOW_HOME=/Users/tester/compose_airflow CWL_TMP_FOLDER=/Users/tester/compose_airflow/cwl_tmp_folder CWL_INPUTS_FOLDER=/Users/tester/compose_airflow/cwl_inputs_folder diff --git a/packaging/docker_compose/local_executor/cwl_airflow/Dockerfile b/packaging/docker_compose/local_executor/cwl_airflow/Dockerfile index b036c1c..5766f99 100644 --- a/packaging/docker_compose/local_executor/cwl_airflow/Dockerfile +++ b/packaging/docker_compose/local_executor/cwl_airflow/Dockerfile @@ -1,49 +1,72 @@ -############################################################################### -# Software: CWL-Airflow -# Software Version: always latest commit from master -# Image Version: latest -# Description: CWL-Airflow image for LocalExecutor and MYSQL backend -# using default Python 3.8 -# Website: https://cwl-airflow.readthedocs.io/en/latest/ -# Provides: Airflow, CWL-Airflow, cwltool -# Base Image: ubuntu:20.04 -# Build Cmd: docker build --rm -t biowardrobe2/cwl-airflow:latest . -############################################################################### - - -FROM ubuntu:20.04 +############################################################################################################################ +# Docker image for running CWL-Airflow conformance tests # +############################################################################################################################ +# Software: CWL-Airflow # +# Software Version: CWL_AIRFLOW_VERSION # +# Image Version: latest # +# Description: CWL-Airflow image for LocalExecutor and MYSQL backend # +# Website: https://cwl-airflow.readthedocs.io/en/latest/ # +# Provides: Airflow, CWL-Airflow, cwltool # +# Python: PYTHON_VERSION # +# Base Image: ubuntu:UBUNTU_VERSION # +# Build Cmd: docker build --no-cache --rm -t biowardrobe2/cwl-airflow:latest . # +# for tag: docker build --no-cache --build-arg CWL_AIRFLOW_VERSION=[tag] --rm -t biowardrobe2/cwl-airflow:[tag] . # +############################################################################################################################ + +# can be provided through --build-arg PARAM=value +ARG UBUNTU_VERSION="20.04" +ARG PYTHON_VERSION="3.8.10" +ARG CWL_AIRFLOW_VERSION="master" + +FROM ubuntu:${UBUNTU_VERSION} LABEL maintainer="misha.kotliar@gmail.com" ENV DEBIAN_FRONTEND noninteractive WORKDIR /tmp -ENV CWL_AIRFLOW_VERSION "master" +# ARG need to be declared after FROM statement again. See details below +# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG PYTHON_VERSION +ARG CWL_AIRFLOW_VERSION + ENV CWL_AIRFLOW_URL "https://github.com/Barski-lab/cwl-airflow" +ENV PYTHON_URL "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" + COPY ./scripts/start_webserver.sh /usr/local/bin/start_webserver.sh COPY ./scripts/start_scheduler.sh /usr/local/bin/start_scheduler.sh COPY ./scripts/start_apiserver.sh /usr/local/bin/start_apiserver.sh -RUN apt-get update && \ - # installing dependencies (including docker dependencies) - apt-get install -y python3-dev python3-pip nodejs gcc git wget curl \ - libmysqlclient-dev mysql-client apt-transport-https \ - ca-certificates gnupg-agent software-properties-common && \ - # installing docker-ce-cli +RUN echo "Installing dependencies" && \ + apt-get update && \ + apt-get install -y gcc build-essential \ + git wget curl zlib1g-dev libmysqlclient-dev libffi-dev libssl-dev \ + ca-certificates \ + nodejs mysql-client apt-transport-https libsqlite3-dev \ + gnupg-agent software-properties-common && \ + echo "Installing Python" && \ + wget ${PYTHON_URL} && \ + tar xzf Python-${PYTHON_VERSION}.tgz && \ + cd Python-${PYTHON_VERSION} && \ + ./configure && \ + make && \ + make install && \ + cd .. && \ + echo "Installing docker-ce-cli" && \ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ apt-get update && \ apt-get -y install docker-ce-cli && \ - # installing cwl-airflow - pip3 install -U pip && \ - git clone ${CWL_AIRFLOW_URL} && \ - cd cwl-airflow && \ - git checkout ${CWL_AIRFLOW_VERSION} && \ - pip3 install ".[mysql]" --constraint ./packaging/constraints/constraints-3.8.txt && \ - cd .. && \ - rm -rf cwl-airflow && \ - # cleaning up - apt-get clean && \ - apt-get purge && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* && \ + echo "Installing cwl-airflow" && \ + pip3 install -U pip && \ + git clone ${CWL_AIRFLOW_URL} && \ + cd cwl-airflow && \ + git checkout ${CWL_AIRFLOW_VERSION} && \ + export SHORT_PYTHON_VERSION=$(echo ${PYTHON_VERSION} | cut -d "." -f 1,2) && \ + pip3 install ".[mysql]" --constraint ./packaging/constraints/constraints-${SHORT_PYTHON_VERSION}.txt && \ + cd .. && \ + echo "Cleaning up" && \ + apt-get clean && \ + apt-get purge && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/* && \ strip /usr/local/bin/*; true \ No newline at end of file diff --git a/packaging/docker_compose/local_executor/docker-compose.yml b/packaging/docker_compose/local_executor/docker-compose.yml index 0383221..aee21c9 100644 --- a/packaging/docker_compose/local_executor/docker-compose.yml +++ b/packaging/docker_compose/local_executor/docker-compose.yml @@ -30,7 +30,13 @@ services: # starts after mysql was launched, then waits until airflow gets access to the database, # then runs cwl-airflow init creating all required tables in the database and starts scheduler scheduler: - build: ./cwl_airflow + build: + context: ./cwl_airflow + dockerfile: Dockerfile + args: + UBUNTU_VERSION: ${UBUNTU_VERSION} + PYTHON_VERSION: ${PYTHON_VERSION} + CWL_AIRFLOW_VERSION: ${CWL_AIRFLOW_VERSION} container_name: scheduler volumes: # can't reuse *airflow_volumes as YAML doesn't support sequence merging - /var/run/docker.sock:/var/run/docker.sock @@ -50,7 +56,13 @@ services: # starts after scheduler was launched, then waits untill all required for airflow tables have been # created (we just check dag_run table) and starts webserver webserver: - build: ./cwl_airflow + build: + context: ./cwl_airflow + dockerfile: Dockerfile + args: + UBUNTU_VERSION: ${UBUNTU_VERSION} + PYTHON_VERSION: ${PYTHON_VERSION} + CWL_AIRFLOW_VERSION: ${CWL_AIRFLOW_VERSION} container_name: webserver ports: - ${AIRFLOW_WEBSERVER_PORT}:8080 @@ -66,7 +78,13 @@ services: # created (we just check dag_run table) and starts apiserver # the default API --host 127.0.0.1 won't work from inside the docker container, so we used 0.0.0.0 apiserver: - build: ./cwl_airflow + build: + context: ./cwl_airflow + dockerfile: Dockerfile + args: + UBUNTU_VERSION: ${UBUNTU_VERSION} + PYTHON_VERSION: ${PYTHON_VERSION} + CWL_AIRFLOW_VERSION: ${CWL_AIRFLOW_VERSION} container_name: apiserver ports: - ${CWL_AIRFLOW_API_PORT}:8081 diff --git a/setup.py b/setup.py index c1531ab..1ca534f 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def get_version(): ], "mysql": [ "mysql-connector-python>=8.0.11, <=8.0.22", - "mysqlclient>=1.3.6,<1.4" + "mysqlclient>=1.3.6,<3", ], "statsd": [ "statsd>=3.3.0, <4.0" @@ -95,9 +95,9 @@ def get_version(): ), extras_require=EXTRAS_REQUIRE, install_requires=[ - "apache-airflow==2.0.1", - "cwltool==3.0.20200710214758", - "cwltest==2.0.20200626112502", + "apache-airflow==2.1.4", + "cwltool==3.1.20210816212154", + "cwltest==2.1.20210626101542", "jsonmerge", "connexion", "tornado", diff --git a/test_requirements.txt b/test_requirements.txt index e717dba..47198a5 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,4 @@ -pytest==6.1.1 -pytest-xdist==2.1.0 -pytest-cov==2.10.1 -coveralls==2.1.2 \ No newline at end of file +pytest==6.2.5 +pytest-xdist==2.3.0 +pytest-cov==2.12.1 +coveralls==3.2.0 \ No newline at end of file diff --git a/tests/test_conformance.sh b/tests/conformance_tests/run_conformance_tests.sh similarity index 84% rename from tests/test_conformance.sh rename to tests/conformance_tests/run_conformance_tests.sh index 525da20..1283c63 100755 --- a/tests/test_conformance.sh +++ b/tests/conformance_tests/run_conformance_tests.sh @@ -12,16 +12,19 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -REPO_URL=$1 -SUITE=$2 -PARAMS=${@:3} +UBUNTU_VERSION=$1 +PYTHON_VERSION=$2 +CWL_AIRFLOW_VERSION=$3 +REPO_URL=$4 +SUITE=$5 +PARAMS=${@:6} if [ $# -lt 2 ]; then echo "Usage: run_conformance_tests.sh https://github.com/repository.git ./location/within/repository/conformance.yaml [other params for cwl-airflow test]" exit 1 fi -echo "Running conformance tests from ${REPO_URL} repository. File ${SUITE}" +echo "Running tests for ${REPO_URL} from file ${SUITE} with CWL-Airflow==${CWL_AIRFLOW_VERSION}" TEMP="${DIR}/temp" echo "Cleaning temporary directory ${TEMP}" @@ -33,7 +36,12 @@ echo "AIRFLOW__CORE__PARALLELISM=1" >> ${AIRFLOW_ENV_FILE} echo "AIRFLOW__CORE__DAG_CONCURRENCY=1" >> ${AIRFLOW_ENV_FILE} echo "AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=60" >> ${AIRFLOW_ENV_FILE} echo "AIRFLOW__CORE__HOSTNAME_CALLABLE=socket.gethostname" >> ${AIRFLOW_ENV_FILE} +echo "AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT=120" >> ${AIRFLOW_ENV_FILE} + +export UBUNTU_VERSION="${UBUNTU_VERSION}" +export PYTHON_VERSION="${PYTHON_VERSION}" +export CWL_AIRFLOW_VERSION="${CWL_AIRFLOW_VERSION}" export AIRFLOW_HOME="${TEMP}/airflow" export CWL_TMP_FOLDER="${TEMP}/airflow/cwl_tmp_folder" export CWL_INPUTS_FOLDER="${TEMP}/airflow/cwl_inputs_folder" @@ -57,20 +65,21 @@ echo "Cleaning old images" # image names are based on the docker-compose file a docker rmi --force local_executor_apiserver local_executor_scheduler local_executor_webserver echo "Building and starting docker-compose as daemon" -DOCKER_COMPOSE_FILE="${DIR}/../packaging/docker_compose/local_executor/docker-compose.yml" +DOCKER_COMPOSE_FILE="${DIR}/../../packaging/docker_compose/local_executor/docker-compose.yml" docker-compose -f ${DOCKER_COMPOSE_FILE} build --no-cache # need --no-cache as we want to have the latest commit for CWL-Airflow, but its Dockerfile is not changed docker-compose -f ${DOCKER_COMPOSE_FILE} up -d +echo "Sleeping 60 sec to let all services start" +sleep 60 + echo "Cloning repository with tests ${REPO_URL}" +mkdir -p ${AIRFLOW_HOME} cd ${AIRFLOW_HOME} git clone ${REPO_URL} --recursive REPO_FOLDER=`basename ${REPO_URL}` REPO_FOLDER="${REPO_FOLDER%.*}" # to exclude possible .git in the url cd - -echo "Sleeping 30 sec to let all services start" -sleep 30 - echo "Starting docker container to run tests from ${SUITE}" docker run --rm \ --name "${PROCESS_REPORT_HOST}" \ diff --git a/tests/run_conformance_tests_docker.sh b/tests/run_conformance_tests_docker.sh new file mode 100755 index 0000000..bbfd907 --- /dev/null +++ b/tests/run_conformance_tests_docker.sh @@ -0,0 +1,10 @@ +#!/bin/bash +UBUNTU_VERSION=${1:-"20.04"} +PYTHON_VERSION=${2:-"3.8.10"} +CWL_AIRFLOW_VERSION=${3:-`git rev-parse --abbrev-ref HEAD`} +REPO_URL=${4:-"https://github.com/datirium/workflows.git"} +SUITE=${5:-"tests/conformance_tests.yaml"} + +echo "Running tests for ${REPO_URL} from the file ${SUITE} with CWL-Airflow (${CWL_AIRFLOW_VERSION})" +echo "using Ubuntu ${UBUNTU_VERSION} and Python ${PYTHON_VERSION}" +./conformance_tests/run_conformance_tests.sh ${UBUNTU_VERSION} ${PYTHON_VERSION} ${CWL_AIRFLOW_VERSION} ${REPO_URL} ${SUITE} \ No newline at end of file diff --git a/tests/run_cwl_v1.1_tests.sh b/tests/run_cwl_v1.1_tests.sh deleted file mode 100755 index c8ee8d0..0000000 --- a/tests/run_cwl_v1.1_tests.sh +++ /dev/null @@ -1,19 +0,0 @@ - -REPO_URL="https://github.com/common-workflow-language/cwl-v1.1.git" -SUITE="conformance_tests.yaml" -RANGE="--range 2-999" - - -# SKIP test N1 -# - job: tests/bwa-mem-job.json -# tool: tests/bwa-mem-tool.cwl -# BECAUSE -# In this test they create cwl.output.json that doesn’t include sam output. -# Therefore when placed as a step of the workflow the bwa-mem-tool.cwl tool -# won’t produce a required output and we will get an error "Output is missing -# expected field". This is not a bug that we should fix, as this test interfere -# the normal flow of things. - - -echo "Running ${RANGE} tests for ${REPO_URL} from file ${SUITE}" -./test_conformance.sh ${REPO_URL} ${SUITE} ${RANGE} \ No newline at end of file diff --git a/tests/run_cwl_v1.2_tests.sh b/tests/run_cwl_v1.2_tests.sh deleted file mode 100755 index ff39e47..0000000 --- a/tests/run_cwl_v1.2_tests.sh +++ /dev/null @@ -1,6 +0,0 @@ - -REPO_URL="https://github.com/common-workflow-language/cwl-v1.2.git" -SUITE="conformance_tests.yaml" - -echo "Running tests for ${REPO_URL} from file ${SUITE}" -./test_conformance.sh ${REPO_URL} ${SUITE} \ No newline at end of file diff --git a/tests/run_unit_tests_docker.sh b/tests/run_unit_tests_docker.sh new file mode 100755 index 0000000..ca2d4ab --- /dev/null +++ b/tests/run_unit_tests_docker.sh @@ -0,0 +1,19 @@ +#!/bin/bash +UBUNTU_VERSION=${1:-"20.04"} +PYTHON_VERSION=${2:-"3.8.10"} +CWL_AIRFLOW_VERSION=${3:-`git rev-parse --abbrev-ref HEAD`} + +WORKING_DIR=$( cd ../"$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +echo "Running unit tests with Python ${PYTHON_VERSION} in dockerized Ubuntu $UBUNTU_VERSION" +echo "Using CWL-Airflow (${CWL_AIRFLOW_VERSION})" +echo "Working directory $WORKING_DIR" +docker rmi cwl_airflow:latest --force +docker build --no-cache --build-arg UBUNTU_VERSION=$UBUNTU_VERSION \ + --build-arg PYTHON_VERSION=$PYTHON_VERSION \ + --build-arg CWL_AIRFLOW_VERSION=$CWL_AIRFLOW_VERSION \ + --rm -t cwl_airflow:latest $WORKING_DIR/packaging/docker_compose/local_executor/cwl_airflow +docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock \ + -v ${WORKING_DIR}:${WORKING_DIR} \ + --workdir ${WORKING_DIR} \ + cwl_airflow:latest \ + ${WORKING_DIR}/tests/unit_tests/run_unit_tests.sh \ No newline at end of file diff --git a/tests/run_workflows_tests.sh b/tests/run_workflows_tests.sh deleted file mode 100755 index 6e310cf..0000000 --- a/tests/run_workflows_tests.sh +++ /dev/null @@ -1,6 +0,0 @@ - -REPO_URL="https://github.com/datirium/workflows.git" -SUITE="tests/conformance_tests.yaml" - -echo "Running tests for ${REPO_URL} from file ${SUITE}" -./test_conformance.sh ${REPO_URL} ${SUITE} \ No newline at end of file diff --git a/tests/__init__.py b/tests/unit_tests/__init__.py similarity index 100% rename from tests/__init__.py rename to tests/unit_tests/__init__.py diff --git a/tests/run_unit_tests.sh b/tests/unit_tests/run_unit_tests.sh similarity index 61% rename from tests/run_unit_tests.sh rename to tests/unit_tests/run_unit_tests.sh index 2d1f08c..615321e 100755 --- a/tests/run_unit_tests.sh +++ b/tests/unit_tests/run_unit_tests.sh @@ -2,8 +2,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -pip3 install -r $DIR/../test_requirements.txt +pip3 install -r $DIR/../../test_requirements.txt +pytest --cov=cwl_airflow --cov-append --forked $DIR/test_conformance.py && \ pytest --cov=cwl_airflow --cov-append --forked $DIR/test_helpers.py && \ pytest --cov=cwl_airflow --cov-append --forked $DIR/test_parser.py && \ pytest --cov=cwl_airflow --cov-append --forked $DIR/test_init_config.py && \ diff --git a/tests/test_conformance.py b/tests/unit_tests/test_conformance.py similarity index 95% rename from tests/test_conformance.py rename to tests/unit_tests/test_conformance.py index b0fb9fe..5533580 100644 --- a/tests/test_conformance.py +++ b/tests/unit_tests/test_conformance.py @@ -12,7 +12,7 @@ from cwl_airflow.utilities.parser import parse_arguments -DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "data")) +DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "../data")) if sys.platform == "darwin": # docker has troubles of mounting /var/private on macOs tempfile.tempdir = "/private/tmp" diff --git a/tests/test_cwl.py b/tests/unit_tests/test_cwl.py similarity index 99% rename from tests/test_cwl.py rename to tests/unit_tests/test_cwl.py index 3be1d67..30b04da 100644 --- a/tests/test_cwl.py +++ b/tests/unit_tests/test_cwl.py @@ -44,7 +44,7 @@ ) -DATA_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) +DATA_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data")) if sys.platform == "darwin": # docker has troubles of mounting /var/private on macOs tempfile.tempdir = "/private/tmp" diff --git a/tests/test_helpers.py b/tests/unit_tests/test_helpers.py similarity index 99% rename from tests/test_helpers.py rename to tests/unit_tests/test_helpers.py index ac3ac90..bf5794a 100644 --- a/tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -22,7 +22,7 @@ ) -DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "data")) +DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "../data")) if sys.platform == "darwin": # docker has troubles of mounting /var/private on macOs tempfile.tempdir = "/private/tmp" diff --git a/tests/test_init_config.py b/tests/unit_tests/test_init_config.py similarity index 98% rename from tests/test_init_config.py rename to tests/unit_tests/test_init_config.py index 57f4ec1..fb09f21 100644 --- a/tests/test_init_config.py +++ b/tests/unit_tests/test_init_config.py @@ -17,7 +17,7 @@ ) -DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "data")) +DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "../data")) if sys.platform == "darwin": # docker has troubles of mounting /var/private on macOs tempfile.tempdir = "/private/tmp" diff --git a/tests/test_parser.py b/tests/unit_tests/test_parser.py similarity index 98% rename from tests/test_parser.py rename to tests/unit_tests/test_parser.py index 1c8a603..ec97e4e 100644 --- a/tests/test_parser.py +++ b/tests/unit_tests/test_parser.py @@ -9,7 +9,7 @@ from cwl_airflow.utilities.helpers import get_absolute_path -DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "data")) +DATA_FOLDER = path.abspath(path.join(path.dirname(__file__), "../data")) if sys.platform == "darwin": # docker has troubles of mounting /var/private on macOs tempfile.tempdir = "/private/tmp"