diff --git a/Dockerfile.ci b/Dockerfile.ci index 3b99c5b6baa97..9a2ea82880f45 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -349,7 +349,7 @@ COPY requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt \ # But in cron job we will install latest versions matching setup.py to see if there is no breaking change RUN \ if [[ "${UPGRADE_TO_LATEST_REQUIREMENTS}" == "true" ]]; then \ - pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade; \ + pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \ else \ pip install -e ".[${AIRFLOW_EXTRAS}]" \ --constraint ${AIRFLOW_SOURCES}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt ; \ diff --git a/airflow/models/dag.py b/airflow/models/dag.py index abdd7fbfce91f..c6b9171cdde2d 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -262,7 +262,8 @@ def __init__( self._description = description # set file location to caller source path - self.fileloc = sys._getframe().f_back.f_code.co_filename + back = sys._getframe().f_back + self.fileloc = back.f_code.co_filename if back else "" self.task_dict = dict() # type: Dict[str, BaseOperator] # set timezone from start_date diff --git a/airflow/security/kerberos.py b/airflow/security/kerberos.py index 35fb671d70195..6c04081456465 100644 --- a/airflow/security/kerberos.py +++ b/airflow/security/kerberos.py @@ -83,7 +83,9 @@ def renew_from_kt(principal, keytab): if subp.returncode != 0: log.error( "Couldn't reinit from keytab! `kinit' exited with %s.\n%s\n%s", - subp.returncode, "\n".join(subp.stdout.readlines()), "\n".join(subp.stderr.readlines()) + subp.returncode, + "\n".join(subp.stdout.readlines() if subp.stdout else []), + "\n".join(subp.stderr.readlines() if subp.stderr else []) ) sys.exit(subp.returncode) diff --git a/requirements/requirements-python2.7.txt b/requirements/requirements-python2.7.txt index 5ffb7e5df2e01..9058478be7de3 100644 --- a/requirements/requirements-python2.7.txt +++ b/requirements/requirements-python2.7.txt @@ -45,7 +45,7 @@ atlasclient==1.0.0 atomicwrites==1.3.0 attrs==19.3.0 aws-sam-translator==1.22.0 -aws-xray-sdk==2.4.3 +aws-xray-sdk==2.5.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 @@ -79,7 +79,7 @@ celery==4.4.2 certifi==2020.4.5.1 cffi==1.14.0 cfgv==2.0.1 -cfn-lint==0.29.4 +cfn-lint==0.29.5 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -90,7 +90,7 @@ configparser==3.5.3 contextdecorator==0.10.0 contextlib2==0.6.0.post1 cookies==2.2.1 -coverage==5.0.4 +coverage==5.1 croniter==0.3.31 cryptography==2.9 cx-Oracle==7.3.0 @@ -123,19 +123,19 @@ future-fstrings==1.2.0 future==0.18.2 futures==3.3.0 gcsfs==0.2.3 -google-api-core==1.16.0 +google-api-core==1.17.0 google-api-python-client==1.8.0 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.13.1 +google-auth==1.14.0 google-cloud-bigquery==1.24.0 google-cloud-bigtable==1.2.1 -google-cloud-container==0.4.0 +google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.13.0 google-cloud-language==1.3.0 google-cloud-secret-manager==0.2.0 -google-cloud-spanner==1.15.0 +google-cloud-spanner==1.15.1 google-cloud-speech==1.3.2 google-cloud-storage==1.27.0 google-cloud-texttospeech==1.0.1 @@ -151,7 +151,7 @@ grpcio==1.28.1 gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 -httplib2==0.17.1 +httplib2==0.17.2 humanize==0.5.1 hvac==0.10.1 identify==1.4.14 @@ -173,7 +173,7 @@ jmespath==0.9.5 json-merge-patch==0.2 jsondiff==1.1.2 jsonpatch==1.25 -jsonpickle==1.3 +jsonpickle==1.4 jsonpointer==2.0 jsonschema==3.2.0 jupyter-client==5.3.4 @@ -212,7 +212,7 @@ pandas-gbq==0.13.1 pandas==0.24.2 pandocfilters==1.4.2 papermill==1.2.1 -parameterized==0.7.1 +parameterized==0.7.4 paramiko==2.7.1 pathlib2==2.3.5 pathspec==0.8.0 @@ -260,7 +260,7 @@ pytz==2019.3 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.0 -qds-sdk==1.15.0 +qds-sdk==1.15.1 redis==3.4.1 requests-futures==0.9.4 requests-kerberos==0.12.0 @@ -326,7 +326,7 @@ webencodings==0.5.1 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.21.0 +yamllint==1.22.1 zdesk==2.7.1 zipp==1.2.0 zope.deprecation==4.4.0 diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt index 6304f14597d85..5c2a5da301b29 100644 --- a/requirements/requirements-python3.5.txt +++ b/requirements/requirements-python3.5.txt @@ -25,7 +25,7 @@ PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 SQLAlchemy==1.3.16 -Sphinx==3.0.0 +Sphinx==3.0.1 Unidecode==1.1.1 WTForms==2.2.1 Werkzeug==0.16.1 @@ -44,7 +44,7 @@ astroid==2.3.3 atlasclient==1.0.0 attrs==19.3.0 aws-sam-translator==1.22.0 -aws-xray-sdk==2.4.3 +aws-xray-sdk==2.5.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 @@ -70,7 +70,7 @@ celery==4.4.2 certifi==2020.4.5.1 cffi==1.14.0 cfgv==2.0.1 -cfn-lint==0.29.4 +cfn-lint==0.29.5 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -78,7 +78,7 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.0.4 +coverage==5.1 croniter==0.3.31 cryptography==2.9 cx-Oracle==7.3.0 @@ -109,19 +109,19 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.1 -google-api-core==1.16.0 +google-api-core==1.17.0 google-api-python-client==1.8.0 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.13.1 +google-auth==1.14.0 google-cloud-bigquery==1.24.0 google-cloud-bigtable==1.2.1 -google-cloud-container==0.4.0 +google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.13.0 google-cloud-language==1.3.0 google-cloud-secret-manager==0.2.0 -google-cloud-spanner==1.15.0 +google-cloud-spanner==1.15.1 google-cloud-speech==1.3.2 google-cloud-storage==1.27.0 google-cloud-texttospeech==1.0.1 @@ -137,7 +137,7 @@ grpcio==1.28.1 gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 -httplib2==0.17.1 +httplib2==0.17.2 humanize==0.5.1 hvac==0.10.1 identify==1.4.14 @@ -153,16 +153,16 @@ ipython==7.9.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 -jedi==0.16.0 +jedi==0.17.0 jira==2.0.0 jmespath==0.9.5 json-merge-patch==0.2 jsondiff==1.1.2 jsonpatch==1.25 -jsonpickle==1.3 +jsonpickle==1.4 jsonpointer==2.0 jsonschema==3.2.0 -jupyter-client==6.1.2 +jupyter-client==6.1.3 jupyter-core==4.6.3 kombu==4.6.8 kubernetes==11.0.0 @@ -187,7 +187,7 @@ nbclient==0.1.0 nbformat==5.0.5 networkx==2.4 nodeenv==1.3.5 -nteract-scrapbook==0.3.1 +nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 numpy==1.18.2 oauthlib==3.1.0 @@ -196,9 +196,9 @@ packaging==20.3 pandas-gbq==0.13.1 pandas==0.25.3 papermill==2.0.0 -parameterized==0.7.1 +parameterized==0.7.4 paramiko==2.7.1 -parso==0.6.2 +parso==0.7.0 pathlib2==2.3.5 pathspec==0.8.0 pbr==5.4.5 @@ -246,7 +246,7 @@ pytz==2019.3 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.0 -qds-sdk==1.15.0 +qds-sdk==1.15.1 redis==3.4.1 requests-futures==0.9.4 requests-kerberos==0.12.0 @@ -266,7 +266,7 @@ setproctitle==1.1.10 six==1.14.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.3 +snowflake-connector-python==2.2.4 snowflake-sqlalchemy==1.2.3 soupsieve==2.0 sphinx-argparse==0.2.5 @@ -310,7 +310,7 @@ wcwidth==0.1.9 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.21.0 +yamllint==1.22.1 zdesk==2.7.1 zipp==1.2.0 zope.deprecation==4.4.0 diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt index f2cf1bbe81ec5..5170d49300466 100644 --- a/requirements/requirements-python3.6.txt +++ b/requirements/requirements-python3.6.txt @@ -45,7 +45,7 @@ async-generator==1.10 atlasclient==1.0.0 attrs==19.3.0 aws-sam-translator==1.22.0 -aws-xray-sdk==2.4.3 +aws-xray-sdk==2.5.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 @@ -80,7 +80,7 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.0.4 +coverage==5.1 croniter==0.3.31 cryptography==2.9 cx-Oracle==7.3.0 @@ -111,19 +111,19 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.1 -google-api-core==1.16.0 +google-api-core==1.17.0 google-api-python-client==1.8.0 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.13.1 +google-auth==1.14.0 google-cloud-bigquery==1.24.0 google-cloud-bigtable==1.2.1 -google-cloud-container==0.4.0 +google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.13.0 google-cloud-language==1.3.0 google-cloud-secret-manager==0.2.0 -google-cloud-spanner==1.15.0 +google-cloud-spanner==1.15.1 google-cloud-speech==1.3.2 google-cloud-storage==1.27.0 google-cloud-texttospeech==1.0.1 @@ -139,7 +139,7 @@ grpcio==1.28.1 gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 -httplib2==0.17.1 +httplib2==0.17.2 humanize==0.5.1 hvac==0.10.1 identify==1.4.14 @@ -155,16 +155,16 @@ ipython==7.13.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 -jedi==0.16.0 +jedi==0.17.0 jira==2.0.0 jmespath==0.9.5 json-merge-patch==0.2 jsondiff==1.1.2 jsonpatch==1.25 -jsonpickle==1.3 +jsonpickle==1.4 jsonpointer==2.0 jsonschema==3.2.0 -jupyter-client==6.1.2 +jupyter-client==6.1.3 jupyter-core==4.6.3 kombu==4.6.8 kubernetes==11.0.0 @@ -190,7 +190,7 @@ nbformat==5.0.5 nest-asyncio==1.3.2 networkx==2.4 nodeenv==1.3.5 -nteract-scrapbook==0.3.1 +nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 numpy==1.18.2 oauthlib==3.1.0 @@ -199,9 +199,9 @@ packaging==20.3 pandas-gbq==0.13.1 pandas==0.25.3 papermill==2.1.0 -parameterized==0.7.1 +parameterized==0.7.4 paramiko==2.7.1 -parso==0.6.2 +parso==0.7.0 pathspec==0.8.0 pbr==5.4.5 pendulum==1.4.4 @@ -248,7 +248,7 @@ pytz==2019.3 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.0 -qds-sdk==1.15.0 +qds-sdk==1.15.1 redis==3.4.1 regex==2020.4.4 requests-futures==0.9.4 @@ -269,7 +269,7 @@ setproctitle==1.1.10 six==1.14.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.3 +snowflake-connector-python==2.2.4 snowflake-sqlalchemy==1.2.3 soupsieve==2.0 sphinx-argparse==0.2.5 @@ -312,7 +312,7 @@ wcwidth==0.1.9 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.21.0 +yamllint==1.22.1 zdesk==2.7.1 zipp==3.1.0 zope.deprecation==4.4.0 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt index 6e8e201f1f478..b5b5bf589adc5 100644 --- a/requirements/requirements-python3.7.txt +++ b/requirements/requirements-python3.7.txt @@ -26,7 +26,7 @@ Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 SQLAlchemy-Utils==0.36.3 SQLAlchemy==1.3.16 -Sphinx==3.0.0 +Sphinx==3.0.1 Unidecode==1.1.1 WTForms==2.2.1 Werkzeug==0.16.1 @@ -45,7 +45,7 @@ async-generator==1.10 atlasclient==1.0.0 attrs==19.3.0 aws-sam-translator==1.22.0 -aws-xray-sdk==2.4.3 +aws-xray-sdk==2.5.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 @@ -72,7 +72,7 @@ celery==4.4.2 certifi==2020.4.5.1 cffi==1.14.0 cfgv==3.1.0 -cfn-lint==0.29.4 +cfn-lint==0.29.5 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -80,7 +80,7 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.0.4 +coverage==5.1 croniter==0.3.31 cryptography==2.9 cx-Oracle==7.3.0 @@ -111,19 +111,19 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.1 -google-api-core==1.16.0 +google-api-core==1.17.0 google-api-python-client==1.8.0 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.13.1 +google-auth==1.14.0 google-cloud-bigquery==1.24.0 google-cloud-bigtable==1.2.1 -google-cloud-container==0.4.0 +google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.13.0 google-cloud-language==1.3.0 google-cloud-secret-manager==0.2.0 -google-cloud-spanner==1.15.0 +google-cloud-spanner==1.15.1 google-cloud-speech==1.3.2 google-cloud-storage==1.27.0 google-cloud-texttospeech==1.0.1 @@ -139,7 +139,7 @@ grpcio==1.28.1 gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 -httplib2==0.17.1 +httplib2==0.17.2 humanize==0.5.1 hvac==0.10.1 identify==1.4.14 @@ -154,16 +154,16 @@ ipython==7.13.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 -jedi==0.16.0 +jedi==0.17.0 jira==2.0.0 jmespath==0.9.5 json-merge-patch==0.2 jsondiff==1.1.2 jsonpatch==1.25 -jsonpickle==1.3 +jsonpickle==1.4 jsonpointer==2.0 jsonschema==3.2.0 -jupyter-client==6.1.2 +jupyter-client==6.1.3 jupyter-core==4.6.3 kombu==4.6.8 kubernetes==11.0.0 @@ -189,7 +189,7 @@ nbformat==5.0.5 nest-asyncio==1.3.2 networkx==2.4 nodeenv==1.3.5 -nteract-scrapbook==0.3.1 +nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 numpy==1.18.2 oauthlib==3.1.0 @@ -198,9 +198,9 @@ packaging==20.3 pandas-gbq==0.13.1 pandas==0.25.3 papermill==2.1.0 -parameterized==0.7.1 +parameterized==0.7.4 paramiko==2.7.1 -parso==0.6.2 +parso==0.7.0 pathspec==0.8.0 pbr==5.4.5 pendulum==1.4.4 @@ -247,7 +247,7 @@ pytz==2019.3 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.0 -qds-sdk==1.15.0 +qds-sdk==1.15.1 redis==3.4.1 regex==2020.4.4 requests-futures==0.9.4 @@ -268,7 +268,7 @@ setproctitle==1.1.10 six==1.14.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.3 +snowflake-connector-python==2.2.4 snowflake-sqlalchemy==1.2.3 soupsieve==2.0 sphinx-argparse==0.2.5 @@ -310,7 +310,7 @@ wcwidth==0.1.9 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.21.0 +yamllint==1.22.1 zdesk==2.7.1 zipp==3.1.0 zope.deprecation==4.4.0 diff --git a/requirements/setup-2.7.md5 b/requirements/setup-2.7.md5 new file mode 100644 index 0000000000000..bdce3ec4b52b2 --- /dev/null +++ b/requirements/setup-2.7.md5 @@ -0,0 +1 @@ +1c562adbde19d08c7d112bdfbf8b3d86 /opt/airflow/setup.py diff --git a/requirements/setup-3.5.md5 b/requirements/setup-3.5.md5 new file mode 100644 index 0000000000000..bdce3ec4b52b2 --- /dev/null +++ b/requirements/setup-3.5.md5 @@ -0,0 +1 @@ +1c562adbde19d08c7d112bdfbf8b3d86 /opt/airflow/setup.py diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5 new file mode 100644 index 0000000000000..bdce3ec4b52b2 --- /dev/null +++ b/requirements/setup-3.6.md5 @@ -0,0 +1 @@ +1c562adbde19d08c7d112bdfbf8b3d86 /opt/airflow/setup.py diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5 new file mode 100644 index 0000000000000..bdce3ec4b52b2 --- /dev/null +++ b/requirements/setup-3.7.md5 @@ -0,0 +1 @@ +1c562adbde19d08c7d112bdfbf8b3d86 /opt/airflow/setup.py diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 0fd9c6e2e7341..4ca64aaa2283b 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -163,7 +163,7 @@ function initialize_common_environment { # upgrade while generating requirements should only happen in localy run # pre-commits or in cron job - if [[ ${LOCAL_RUN} == "true" || "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then + if [[ ${LOCAL_RUN} == "true" ]]; then export UPGRADE_WHILE_GENERATING_REQUIREMENTS="true" else export UPGRADE_WHILE_GENERATING_REQUIREMENTS="false" diff --git a/scripts/ci/in_container/run_generate_requirements.sh b/scripts/ci/in_container/run_generate_requirements.sh index fd4cf78660daa..19f8c595953fd 100755 --- a/scripts/ci/in_container/run_generate_requirements.sh +++ b/scripts/ci/in_container/run_generate_requirements.sh @@ -23,6 +23,20 @@ HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" # shellcheck disable=SC2064 trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT +STORED_SETUP_PY_HASH_FILE="${AIRFLOW_SOURCES}/requirements/setup-${PYTHON_MAJOR_MINOR_VERSION}.md5" + +CURRENT_SETUP_PY_HASH=$(md5sum "${AIRFLOW_SOURCES}/setup.py") +STORED_SETUP_PY_HASH=$(cat "${STORED_SETUP_PY_HASH_FILE}" 2>/dev/null || true) + +if [[ ${STORED_SETUP_PY_HASH} != "${CURRENT_SETUP_PY_HASH}" ]]; then + echo + echo "Setup.py changed since last time requirements were generated" + echo + echo "Switching to eager update strategy for the requirements" + echo + UPGRADE_WHILE_GENERATING_REQUIREMENTS="true" +fi + # Upgrading requirements will happen only in CRON job to see that we have some # new requirements released if [[ ${UPGRADE_WHILE_GENERATING_REQUIREMENTS} == "true" ]]; then @@ -52,6 +66,11 @@ echo echo "Requirements generated in ${GENERATED_REQUIREMENTS_FILE}" echo +echo +echo "Storing setup.py hash in ${STORED_SETUP_PY_HASH_FILE}" +echo +echo "${CURRENT_SETUP_PY_HASH}" > "${STORED_SETUP_PY_HASH_FILE}" + set +e # Fail in case diff shows difference diff --color=always "${OLD_REQUIREMENTS_FILE}" "${GENERATED_REQUIREMENTS_FILE}"