Less aggressive eager upgrade of requirements (#8267)

With this change requirements are only eagerly upgraded when
generating requirements when setup.py changes. They are also
eagerly upgraded when you run ./breeze generate-requirements
locally. Still the cron job will use the eager update mechanism
when building the docker image which means that CRON jobs will
still detect cases where upgrede of requirements causes failure
either at the installation time or during tests.

(cherry picked from commit 45c8983)

Dockerfile.ci

@@ -349,7 +349,7 @@ COPY requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt \
 # But in cron job we will install latest versions matching setup.py to see if there is no breaking change
 RUN \
     if [[ "${UPGRADE_TO_LATEST_REQUIREMENTS}" == "true" ]]; then \
-        pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade; \
+        pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade --upgrade-strategy eager; \
     else \
         pip install -e ".[${AIRFLOW_EXTRAS}]" \
             --constraint  ${AIRFLOW_SOURCES}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt ; \

airflow/models/dag.py

@@ -262,7 +262,8 @@ def __init__(
 
         self._description = description
         # set file location to caller source path
-        self.fileloc = sys._getframe().f_back.f_code.co_filename
+        back = sys._getframe().f_back
+        self.fileloc = back.f_code.co_filename if back else ""
         self.task_dict = dict()  # type: Dict[str, BaseOperator]
 
         # set timezone from start_date

airflow/security/kerberos.py

@@ -83,7 +83,9 @@ def renew_from_kt(principal, keytab):
     if subp.returncode != 0:
         log.error(
             "Couldn't reinit from keytab! `kinit' exited with %s.\n%s\n%s",
-            subp.returncode, "\n".join(subp.stdout.readlines()), "\n".join(subp.stderr.readlines())
+            subp.returncode,
+            "\n".join(subp.stdout.readlines() if subp.stdout else []),
+            "\n".join(subp.stderr.readlines() if subp.stderr else [])
         )
         sys.exit(subp.returncode)
 

requirements/requirements-python2.7.txt

@@ -45,7 +45,7 @@ atlasclient==1.0.0
 atomicwrites==1.3.0
 attrs==19.3.0
 aws-sam-translator==1.22.0
-aws-xray-sdk==2.4.3
+aws-xray-sdk==2.5.0
 azure-common==1.1.25
 azure-cosmos==3.1.2
 azure-datalake-store==0.0.48
@@ -79,7 +79,7 @@ celery==4.4.2
 certifi==2020.4.5.1
 cffi==1.14.0
 cfgv==2.0.1
-cfn-lint==0.29.4
+cfn-lint==0.29.5
 cgroupspy==0.1.6
 chardet==3.0.4
 click==6.7
@@ -90,7 +90,7 @@ configparser==3.5.3
 contextdecorator==0.10.0
 contextlib2==0.6.0.post1
 cookies==2.2.1
-coverage==5.0.4
+coverage==5.1
 croniter==0.3.31
 cryptography==2.9
 cx-Oracle==7.3.0
@@ -123,19 +123,19 @@ future-fstrings==1.2.0
 future==0.18.2
 futures==3.3.0
 gcsfs==0.2.3
-google-api-core==1.16.0
+google-api-core==1.17.0
 google-api-python-client==1.8.0
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
-google-auth==1.13.1
+google-auth==1.14.0
 google-cloud-bigquery==1.24.0
 google-cloud-bigtable==1.2.1
-google-cloud-container==0.4.0
+google-cloud-container==0.5.0
 google-cloud-core==1.3.0
 google-cloud-dlp==0.13.0
 google-cloud-language==1.3.0
 google-cloud-secret-manager==0.2.0
-google-cloud-spanner==1.15.0
+google-cloud-spanner==1.15.1
 google-cloud-speech==1.3.2
 google-cloud-storage==1.27.0
 google-cloud-texttospeech==1.0.1
@@ -151,7 +151,7 @@ grpcio==1.28.1
 gunicorn==19.10.0
 hdfs==2.5.8
 hmsclient==0.1.1
-httplib2==0.17.1
+httplib2==0.17.2
 humanize==0.5.1
 hvac==0.10.1
 identify==1.4.14
@@ -173,7 +173,7 @@ jmespath==0.9.5
 json-merge-patch==0.2
 jsondiff==1.1.2
 jsonpatch==1.25
-jsonpickle==1.3
+jsonpickle==1.4
 jsonpointer==2.0
 jsonschema==3.2.0
 jupyter-client==5.3.4
@@ -212,7 +212,7 @@ pandas-gbq==0.13.1
 pandas==0.24.2
 pandocfilters==1.4.2
 papermill==1.2.1
-parameterized==0.7.1
+parameterized==0.7.4
 paramiko==2.7.1
 pathlib2==2.3.5
 pathspec==0.8.0
@@ -260,7 +260,7 @@ pytz==2019.3
 pytzdata==2019.3
 pywinrm==0.4.1
 pyzmq==19.0.0
-qds-sdk==1.15.0
+qds-sdk==1.15.1
 redis==3.4.1
 requests-futures==0.9.4
 requests-kerberos==0.12.0
@@ -326,7 +326,7 @@ webencodings==0.5.1
 websocket-client==0.57.0
 wrapt==1.12.1
 xmltodict==0.12.0
-yamllint==1.21.0
+yamllint==1.22.1
 zdesk==2.7.1
 zipp==1.2.0
 zope.deprecation==4.4.0

requirements/requirements-python3.5.txt

@@ -25,7 +25,7 @@ PyYAML==5.3.1
 Pygments==2.6.1
 SQLAlchemy-JSONField==0.9.0
 SQLAlchemy==1.3.16
-Sphinx==3.0.0
+Sphinx==3.0.1
 Unidecode==1.1.1
 WTForms==2.2.1
 Werkzeug==0.16.1
@@ -44,7 +44,7 @@ astroid==2.3.3
 atlasclient==1.0.0
 attrs==19.3.0
 aws-sam-translator==1.22.0
-aws-xray-sdk==2.4.3
+aws-xray-sdk==2.5.0
 azure-common==1.1.25
 azure-cosmos==3.1.2
 azure-datalake-store==0.0.48
@@ -70,15 +70,15 @@ celery==4.4.2
 certifi==2020.4.5.1
 cffi==1.14.0
 cfgv==2.0.1
-cfn-lint==0.29.4
+cfn-lint==0.29.5
 cgroupspy==0.1.6
 chardet==3.0.4
 click==6.7
 cloudant==0.5.10
 colorama==0.4.3
 colorlog==4.0.2
 configparser==3.5.3
-coverage==5.0.4
+coverage==5.1
 croniter==0.3.31
 cryptography==2.9
 cx-Oracle==7.3.0
@@ -109,19 +109,19 @@ funcsigs==1.0.2
 future-fstrings==1.2.0
 future==0.18.2
 gcsfs==0.6.1
-google-api-core==1.16.0
+google-api-core==1.17.0
 google-api-python-client==1.8.0
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
-google-auth==1.13.1
+google-auth==1.14.0
 google-cloud-bigquery==1.24.0
 google-cloud-bigtable==1.2.1
-google-cloud-container==0.4.0
+google-cloud-container==0.5.0
 google-cloud-core==1.3.0
 google-cloud-dlp==0.13.0
 google-cloud-language==1.3.0
 google-cloud-secret-manager==0.2.0
-google-cloud-spanner==1.15.0
+google-cloud-spanner==1.15.1
 google-cloud-speech==1.3.2
 google-cloud-storage==1.27.0
 google-cloud-texttospeech==1.0.1
@@ -137,7 +137,7 @@ grpcio==1.28.1
 gunicorn==19.10.0
 hdfs==2.5.8
 hmsclient==0.1.1
-httplib2==0.17.1
+httplib2==0.17.2
 humanize==0.5.1
 hvac==0.10.1
 identify==1.4.14
@@ -153,16 +153,16 @@ ipython==7.9.0
 iso8601==0.1.12
 isodate==0.6.0
 itsdangerous==1.1.0
-jedi==0.16.0
+jedi==0.17.0
 jira==2.0.0
 jmespath==0.9.5
 json-merge-patch==0.2
 jsondiff==1.1.2
 jsonpatch==1.25
-jsonpickle==1.3
+jsonpickle==1.4
 jsonpointer==2.0
 jsonschema==3.2.0
-jupyter-client==6.1.2
+jupyter-client==6.1.3
 jupyter-core==4.6.3
 kombu==4.6.8
 kubernetes==11.0.0
@@ -187,7 +187,7 @@ nbclient==0.1.0
 nbformat==5.0.5
 networkx==2.4
 nodeenv==1.3.5
-nteract-scrapbook==0.3.1
+nteract-scrapbook==0.4.1
 ntlm-auth==1.4.0
 numpy==1.18.2
 oauthlib==3.1.0
@@ -196,9 +196,9 @@ packaging==20.3
 pandas-gbq==0.13.1
 pandas==0.25.3
 papermill==2.0.0
-parameterized==0.7.1
+parameterized==0.7.4
 paramiko==2.7.1
-parso==0.6.2
+parso==0.7.0
 pathlib2==2.3.5
 pathspec==0.8.0
 pbr==5.4.5
@@ -246,7 +246,7 @@ pytz==2019.3
 pytzdata==2019.3
 pywinrm==0.4.1
 pyzmq==19.0.0
-qds-sdk==1.15.0
+qds-sdk==1.15.1
 redis==3.4.1
 requests-futures==0.9.4
 requests-kerberos==0.12.0
@@ -266,7 +266,7 @@ setproctitle==1.1.10
 six==1.14.0
 slackclient==1.3.2
 snowballstemmer==2.0.0
-snowflake-connector-python==2.2.3
+snowflake-connector-python==2.2.4
 snowflake-sqlalchemy==1.2.3
 soupsieve==2.0
 sphinx-argparse==0.2.5
@@ -310,7 +310,7 @@ wcwidth==0.1.9
 websocket-client==0.57.0
 wrapt==1.12.1
 xmltodict==0.12.0
-yamllint==1.21.0
+yamllint==1.22.1
 zdesk==2.7.1
 zipp==1.2.0
 zope.deprecation==4.4.0

requirements/requirements-python3.6.txt

@@ -45,7 +45,7 @@ async-generator==1.10
 atlasclient==1.0.0
 attrs==19.3.0
 aws-sam-translator==1.22.0
-aws-xray-sdk==2.4.3
+aws-xray-sdk==2.5.0
 azure-common==1.1.25
 azure-cosmos==3.1.2
 azure-datalake-store==0.0.48
@@ -80,7 +80,7 @@ cloudant==0.5.10
 colorama==0.4.3
 colorlog==4.0.2
 configparser==3.5.3
-coverage==5.0.4
+coverage==5.1
 croniter==0.3.31
 cryptography==2.9
 cx-Oracle==7.3.0
@@ -111,19 +111,19 @@ funcsigs==1.0.2
 future-fstrings==1.2.0
 future==0.18.2
 gcsfs==0.6.1
-google-api-core==1.16.0
+google-api-core==1.17.0
 google-api-python-client==1.8.0
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
-google-auth==1.13.1
+google-auth==1.14.0
 google-cloud-bigquery==1.24.0
 google-cloud-bigtable==1.2.1
-google-cloud-container==0.4.0
+google-cloud-container==0.5.0
 google-cloud-core==1.3.0
 google-cloud-dlp==0.13.0
 google-cloud-language==1.3.0
 google-cloud-secret-manager==0.2.0
-google-cloud-spanner==1.15.0
+google-cloud-spanner==1.15.1
 google-cloud-speech==1.3.2
 google-cloud-storage==1.27.0
 google-cloud-texttospeech==1.0.1
@@ -139,7 +139,7 @@ grpcio==1.28.1
 gunicorn==19.10.0
 hdfs==2.5.8
 hmsclient==0.1.1
-httplib2==0.17.1
+httplib2==0.17.2
 humanize==0.5.1
 hvac==0.10.1
 identify==1.4.14
@@ -155,16 +155,16 @@ ipython==7.13.0
 iso8601==0.1.12
 isodate==0.6.0
 itsdangerous==1.1.0
-jedi==0.16.0
+jedi==0.17.0
 jira==2.0.0
 jmespath==0.9.5
 json-merge-patch==0.2
 jsondiff==1.1.2
 jsonpatch==1.25
-jsonpickle==1.3
+jsonpickle==1.4
 jsonpointer==2.0
 jsonschema==3.2.0
-jupyter-client==6.1.2
+jupyter-client==6.1.3
 jupyter-core==4.6.3
 kombu==4.6.8
 kubernetes==11.0.0
@@ -190,7 +190,7 @@ nbformat==5.0.5
 nest-asyncio==1.3.2
 networkx==2.4
 nodeenv==1.3.5
-nteract-scrapbook==0.3.1
+nteract-scrapbook==0.4.1
 ntlm-auth==1.4.0
 numpy==1.18.2
 oauthlib==3.1.0
@@ -199,9 +199,9 @@ packaging==20.3
 pandas-gbq==0.13.1
 pandas==0.25.3
 papermill==2.1.0
-parameterized==0.7.1
+parameterized==0.7.4
 paramiko==2.7.1
-parso==0.6.2
+parso==0.7.0
 pathspec==0.8.0
 pbr==5.4.5
 pendulum==1.4.4
@@ -248,7 +248,7 @@ pytz==2019.3
 pytzdata==2019.3
 pywinrm==0.4.1
 pyzmq==19.0.0
-qds-sdk==1.15.0
+qds-sdk==1.15.1
 redis==3.4.1
 regex==2020.4.4
 requests-futures==0.9.4
@@ -269,7 +269,7 @@ setproctitle==1.1.10
 six==1.14.0
 slackclient==1.3.2
 snowballstemmer==2.0.0
-snowflake-connector-python==2.2.3
+snowflake-connector-python==2.2.4
 snowflake-sqlalchemy==1.2.3
 soupsieve==2.0
 sphinx-argparse==0.2.5
@@ -312,7 +312,7 @@ wcwidth==0.1.9
 websocket-client==0.57.0
 wrapt==1.12.1
 xmltodict==0.12.0
-yamllint==1.21.0
+yamllint==1.22.1
 zdesk==2.7.1
 zipp==3.1.0
 zope.deprecation==4.4.0