chore: bump majors on celery and Flask (#19168)

* chore: bump celery, Flask, flask-jwt-extended, pyJWT * fix pyJWT breaking change * fix pyJWT breaking change 2 * test * fixed test * fixed test * fixed test * revert since mypy won't pick the correct signature * lint 1 * fix test * fix test * docs and celery config migration * bump FAB to 4.0.0rc3, remove AUTH_STRICT_RESPONSE_CODES * update docs for new celery config keys * downgrade celery to 5.2.2 * ref FAB to final 4.0.0 release * remove conflict left over
apache · Mar 24, 2022 · f37fc1a · f37fc1a
1 parent 2af2d00
commit f37fc1a
Show file tree

Hide file tree

Showing 21 changed files with 117 additions and 111 deletions.
diff --git a/UPDATING.md b/UPDATING.md
@@ -29,6 +29,9 @@ assists people when migrating to a new version.
 
 ### Breaking Changes
 
+- [19168](https://github.com/apache/superset/pull/19168): Celery upgrade to 5.X has breaking changes on it's command line invocation.
+  Please follow: https://docs.celeryq.dev/en/stable/whatsnew-5.2.html#step-1-adjust-your-command-line-invocation
+  Consider migrating you celery config if you haven't already: https://docs.celeryq.dev/en/stable/userguide/configuration.html#conf-old-settings-map
 - [19049](https://github.com/apache/superset/pull/19049): APP_ICON_WIDTH has been removed from the config. Superset should now be able to handle different logo sizes without having to explicitly set an APP_ICON_WIDTH. This might affect the size of existing custom logos as the UI will now resize them according to the specified space of maximum 148px and not according to the value of APP_ICON_WIDTH.
 - [19274](https://github.com/apache/superset/pull/19274): The `PUBLIC_ROLE_LIKE_GAMMA` config key has been removed, set `PUBLIC_ROLE_LIKE` = "Gamma" to have the same functionality.
 - [19273](https://github.com/apache/superset/pull/19273): The `SUPERSET_CELERY_WORKERS` and `SUPERSET_WORKERS` config keys has been removed. Configure celery directly using `CELERY_CONFIG` on Superset

diff --git a/docs/docs/installation/alerts-reports.mdx b/docs/docs/installation/alerts-reports.mdx
@@ -89,12 +89,12 @@ REDIS_HOST = "redis-superset"
 REDIS_PORT = "6379"
 
 class CeleryConfig:
-    BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
-    CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
-    CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
-    CELERYD_PREFETCH_MULTIPLIER = 10
-    CELERY_ACKS_LATE = True
-    CELERY_ANNOTATIONS = {
+    broker_url = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
+    imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
+    result_backend = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
+    worker_prefetch_multiplier = 10
+    task_acks_late = True
+    task_annotations = {
         'sql_lab.get_sql_results': {
             'rate_limit': '100/s',
         },
@@ -105,7 +105,7 @@ class CeleryConfig:
             'ignore_result': True,
         },
     }
-    CELERYBEAT_SCHEDULE = {
+    beat_schedule = {
         'reports.scheduler': {
             'task': 'reports.scheduler',
             'schedule': crontab(minute='*', hour='*'),

diff --git a/docs/docs/installation/async-queries-celery.mdx b/docs/docs/installation/async-queries-celery.mdx
@@ -23,16 +23,16 @@ and web server processes should have the same configuration.
 
 ```python
 class CeleryConfig(object):
-    BROKER_URL = 'redis://localhost:6379/0'
-    CELERY_IMPORTS = (
+    broker_url = 'redis://localhost:6379/0'
+    imports = (
         'superset.sql_lab',
         'superset.tasks',
     )
-    CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
-    CELERYD_LOG_LEVEL = 'DEBUG'
-    CELERYD_PREFETCH_MULTIPLIER = 10
-    CELERY_ACKS_LATE = True
-    CELERY_ANNOTATIONS = {
+    result_backend = 'redis://localhost:6379/0'
+    worker_log_level = 'DEBUG'
+    worker_prefetch_multiplier = 10
+    task_acks_late = True
+    task_annotations = {
         'sql_lab.get_sql_results': {
             'rate_limit': '100/s',
         },
@@ -43,7 +43,7 @@ class CeleryConfig(object):
             'ignore_result': True,
         },
     }
-    CELERYBEAT_SCHEDULE = {
+    beat_schedule = {
         'email_reports.schedule_hourly': {
             'task': 'email_reports.schedule_hourly',
             'schedule': crontab(minute=1, hour='*'),

diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx
@@ -85,11 +85,11 @@ from s3cache.s3cache import S3Cache
 ...
 
 class CeleryConfig(object):
-    BROKER_URL = "redis://localhost:6379/0"
-    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks", "superset.tasks.thumbnails")
-    CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
-    CELERYD_PREFETCH_MULTIPLIER = 10
-    CELERY_ACKS_LATE = True
+    broker_url = "redis://localhost:6379/0"
+    imports = ("superset.sql_lab", "superset.tasks", "superset.tasks.thumbnails")
+    result_backend = "redis://localhost:6379/0"
+    worker_prefetch_multiplier = 10
+    task_acks_late = True
 
 
 CELERY_CONFIG = CeleryConfig

diff --git a/docs/docs/installation/running-on-kubernetes.mdx b/docs/docs/installation/running-on-kubernetes.mdx
@@ -344,12 +344,10 @@ configOverrides:
     from celery.schedules import crontab
 
     class CeleryConfig(object):
-      BROKER_URL = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
-      CELERY_IMPORTS = ('superset.sql_lab', )
-      CELERY_RESULT_BACKEND = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
-      CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
-      CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
-      CELERY_ANNOTATIONS = {
+      broker_url = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+      imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
+      result_backend = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+      task_annotations = {
           'sql_lab.get_sql_results': {
               'rate_limit': '100/s',
           },
@@ -360,7 +358,7 @@ configOverrides:
               'ignore_result': True,
           },
       }
-      CELERYBEAT_SCHEDULE = {
+      beat_schedule = {
           'reports.scheduler': {
               'task': 'reports.scheduler',
               'schedule': crontab(minute='*', hour='*'),

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -11,7 +11,7 @@ aiohttp==3.7.4.post0
     # via slackclient
 alembic==1.6.5
     # via flask-migrate
-amqp==2.6.1
+amqp==5.1.0
     # via kombu
 apispec[yaml]==3.3.2
     # via flask-appbuilder
@@ -33,17 +33,27 @@ brotli==1.0.9
     # via flask-compress
 cachelib==0.4.1
     # via apache-superset
-celery==4.4.7
+celery==5.2.2
     # via apache-superset
 cffi==1.14.6
     # via cryptography
 chardet==4.0.0
     # via aiohttp
-click==7.1.2
+click==8.0.4
     # via
     #   apache-superset
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
     #   flask
     #   flask-appbuilder
+click-didyoumean==0.3.0
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.2.0
+    # via celery
 colorama==0.4.4
     # via
     #   apache-superset
@@ -56,15 +66,13 @@ croniter==1.0.15
     # via apache-superset
 cryptography==3.4.7
     # via apache-superset
-defusedxml==0.7.1
-    # via python3-openid
 deprecation==2.1.0
     # via apache-superset
 dnspython==2.1.0
     # via email-validator
 email-validator==1.1.3
     # via flask-appbuilder
-flask==1.1.4
+flask==2.0.3
     # via
     #   apache-superset
     #   flask-appbuilder
@@ -74,25 +82,22 @@ flask==1.1.4
     #   flask-jwt-extended
     #   flask-login
     #   flask-migrate
-    #   flask-openid
     #   flask-sqlalchemy
     #   flask-wtf
-flask-appbuilder==3.4.5
+flask-appbuilder==4.0.0
     # via apache-superset
 flask-babel==1.0.0
     # via flask-appbuilder
 flask-caching==1.10.1
     # via apache-superset
 flask-compress==1.10.1
     # via apache-superset
-flask-jwt-extended==3.25.1
+flask-jwt-extended==4.3.1
     # via flask-appbuilder
 flask-login==0.4.1
     # via flask-appbuilder
 flask-migrate==3.1.0
     # via apache-superset
-flask-openid==1.3.0
-    # via flask-appbuilder
 flask-sqlalchemy==2.5.1
     # via
     #   flask-appbuilder
@@ -123,18 +128,17 @@ idna==3.2
     #   yarl
 isodate==0.6.0
     # via apache-superset
-itsdangerous==1.1.0
+itsdangerous==2.1.1
     # via
-    #   apache-superset
     #   flask
     #   flask-wtf
-jinja2==2.11.3
+jinja2==3.0.3
     # via
     #   flask
     #   flask-babel
 jsonschema==3.2.0
     # via flask-appbuilder
-kombu==4.6.11
+kombu==5.2.4
     # via celery
 korean-lunar-calendar==0.2.1
     # via holidays
@@ -180,11 +184,13 @@ polyline==1.4.0
     # via apache-superset
 prison==0.2.1
     # via flask-appbuilder
+prompt-toolkit==3.0.28
+    # via click-repl
 pyarrow==5.0.0
     # via apache-superset
 pycparser==2.20
     # via cffi
-pyjwt==1.7.1
+pyjwt==2.2.0
     # via
     #   apache-superset
     #   flask-appbuilder
@@ -213,9 +219,7 @@ python-editor==1.0.4
     # via alembic
 python-geohash==0.8.5
     # via apache-superset
-python3-openid==3.2.0
-    # via flask-openid
-pytz==2021.1
+pytz==2021.3
     # via
     #   babel
     #   celery
@@ -237,7 +241,7 @@ simplejson==3.17.3
 six==1.16.0
     # via
     #   bleach
-    #   flask-jwt-extended
+    #   click-repl
     #   flask-talisman
     #   holidays
     #   isodate
@@ -273,13 +277,16 @@ typing-extensions==3.10.0.0
     #   apache-superset
 urllib3==1.26.6
     # via selenium
-vine==1.3.0
+vine==5.0.0
     # via
     #   amqp
     #   celery
+    #   kombu
+wcwidth==0.2.5
+    # via prompt-toolkit
 webencodings==0.5.1
     # via bleach
-werkzeug==1.0.1
+werkzeug==2.0.3
     # via
     #   flask
     #   flask-jwt-extended

diff --git a/requirements/integration.in b/requirements/integration.in
@@ -18,4 +18,4 @@ pip-compile-multi!=1.5.9
 pre-commit
 tox
 py>=1.10.0
-click==7.1.2
+click
diff --git a/requirements/integration.txt b/requirements/integration.txt
@@ -1,4 +1,4 @@
-# SHA1:03eb2d96afe21f1bda1ab33b4cf84e670a1efe21
+# SHA1:8e2dd1e795bcad7451376b3653eb03465e4f05d3
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -9,7 +9,7 @@ backports.entry-points-selectable==1.1.0
     # via virtualenv
 cfgv==3.3.0
     # via pre-commit
-click==7.1.2
+click==8.0.4
     # via
     #   -r requirements/integration.in
     #   pip-compile-multi

diff --git a/requirements/testing.txt b/requirements/testing.txt
@@ -116,8 +116,6 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-prompt-toolkit==3.0.19
-    # via ipython
 proto-plus==1.19.7
     # via
     #   google-cloud-bigquery
@@ -178,8 +176,6 @@ trino==0.306
     # via sqlalchemy-trino
 typing-inspect==0.7.1
     # via libcst
-wcwidth==0.2.5
-    # via prompt-toolkit
 websocket-client==1.2.0
     # via docker
 

diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
@@ -138,5 +138,5 @@ fi
 
 if [ $RUN_TESTS -eq 1 ]
 then
-  pytest -x -s "${TEST_MODULE}"
+  pytest --durations=0 --maxfail=1 "${TEST_MODULE}"
 fi
diff --git a/setup.cfg b/setup.cfg
@@ -41,7 +41,7 @@ disallow_untyped_calls = true
 disallow_untyped_defs = true
 ignore_missing_imports = true
 no_implicit_optional = true
-warn_unused_ignores = true
+warn_unused_ignores = false
 
 [mypy-superset.migrations.versions.*]
 ignore_errors = true

diff --git a/setup.py b/setup.py
@@ -70,15 +70,15 @@ def get_git_sha() -> str:
         "backoff>=1.8.0",
         "bleach>=3.0.2, <4.0.0",
         "cachelib>=0.4.1,<0.5",
-        "celery>=4.3.0, <5.0.0, !=4.4.1",
-        "click<8",
+        "celery>=5.2.2, <6.0.0",
+        "click>=8.0.3",
         "colorama",
         "croniter>=0.3.28",
         "cron-descriptor",
         "cryptography>=3.3.2",
         "deprecation>=2.1.0, <2.2.0",
-        "flask>=1.1.0, <2.0.0",
-        "flask-appbuilder>=3.4.5, <4.0.0",
+        "flask>=2.0.0, <3.0.0",
+        "flask-appbuilder>=4.0.0, <5.0.0",
         "flask-caching>=1.10.0",
         "flask-compress",
         "flask-talisman",
@@ -90,7 +90,6 @@ def get_git_sha() -> str:
         "gunicorn>=20.1.0",
         "holidays==0.10.3",  # PINNED! https://github.com/dr-prodigy/python-holidays/issues/406
         "humanize",
-        "itsdangerous>=1.0.0, <2.0.0",  # https://github.com/apache/superset/pull/14627
         "isodate",
         "markdown>=3.0",
         "msgpack>=1.0.0, <1.1",
@@ -104,7 +103,7 @@ def get_git_sha() -> str:
         "python-geohash",
         "pyarrow>=5.0.0, <6.0",
         "pyyaml>=5.4",
-        "PyJWT>=1.7.1, <2",
+        "PyJWT>=2.0.0, <2.3.0",
         "redis",
         "selenium>=3.141.0",
         "simplejson>=3.15.0",