Re-add deferred catchall routing (#4015)

docs/Developing/Queues-and-defer.md

@@ -37,6 +37,21 @@ app = Flask(__name__)
 app.wsgi_app = wrap_wsgi_app(app.wsgi_app, use_deferred=True)
 ```
 
+Additionally, the service `yaml` file should be configured to only execute authenticated deferred tasks.
+
+```yaml
+service: service
+
+...
+
+handlers:
+  - url: /_ah/queue/deferred.*
+    script: auto
+    login: admin
+  - url: .*
+    script: auto
+```
+
 ### Using `deferred.defer`
 
 ```python
@@ -55,7 +70,26 @@ See the [`google.appengine.ext.deferred.deferred` module documentation](https://
 
 ### Using custom `deferred.defer` routes
 
-By default, tasks will dispatch to the `/_ah/queue/deferred` route via the `default` queue. A custom route can be used if an endpoint to handle the route is built and calls to `deferred.application`.
+By default, tasks will dispatch to the `/_ah/queue/deferred` route via the `default` queue. To use a vanity URL starting with `/_ah/queue/deferred` but having a custom suffix (ex: `/_ah/queue/deferred_manipulator_clearCache`), we can install a custom catch-all regex handler on the service.
+
+```python
+from flask import Flask
+from google.appengine.api import wrap_wsgi_app
+from google.appengine.ext import deferred
+
+from backend.common.deferred import install_defer_routes
+
+app = Flask(__name__)
+app.wsgi_app = wrap_wsgi_app(app.wsgi_app, use_deferred=True)
+install_defer_routes(app)
+
+@app.route("/do_later")
+def do_later():
+    deferred.defer(clear_the_cache, _url="/_ah/queue/deferred_manipulator_clearCache")
+    return "Done! deferred expensive work for later."
+```
+
+Custom URLs can be supported as well using a more manual process. A route must be created that calls to `deferred.application` to handle the original request. This is not recommended, but can be done.
 
 ```python
 @app.route("/do_the_thing")

src/backend/common/deferred/__init__.py

@@ -0,0 +1 @@
+from .defer_handler import install_defer_routes  # noqa: F401

src/backend/common/deferred/defer_handler.py

@@ -0,0 +1,23 @@
+from flask import Flask, request, Response
+from google.appengine.ext import deferred
+
+from backend.common.url_converters import (
+    has_regex_url_converter,
+    install_regex_url_converter,
+)
+
+
+def handle_defer(path: str) -> Response:
+    return deferred.application.post(request.environ)
+
+
+def install_defer_routes(app: Flask) -> None:
+    # Requires regex URL converter
+    if not has_regex_url_converter(app):
+        install_regex_url_converter(app)
+
+    app.add_url_rule(
+        '/_ah/queue/<regex("deferred.*?"):path>',
+        view_func=handle_defer,
+        methods=["POST"],
+    )

src/backend/common/deferred/tests/defer_handler_test.py

@@ -0,0 +1,59 @@
+from unittest.mock import patch
+
+from flask import Flask
+
+from backend.common.deferred.defer_handler import handle_defer, install_defer_routes
+from backend.common.url_converters import install_regex_url_converter
+
+
+def test_install_defer_routes():
+    route = '/_ah/queue/<regex("deferred.*?"):path>'
+    app = Flask(__name__)
+    rules = [r for r in app.url_map.iter_rules() if str(r) == route]
+    assert len(rules) == 0
+    install_defer_routes(app)
+    rules = [r for r in app.url_map.iter_rules() if str(r) == route]
+    assert len(rules) == 1
+    rule = rules[0]
+    assert rule.methods == {"OPTIONS", "POST"}
+    assert rule.endpoint == "handle_defer"
+
+
+def test_install_defer_routes_regex():
+    app = Flask(__name__)
+
+    with patch(
+        "backend.common.deferred.defer_handler.install_regex_url_converter",
+        wraps=install_regex_url_converter,
+    ) as mock_install_regex_url_converter:
+        install_defer_routes(app)
+
+    mock_install_regex_url_converter.assert_called()
+
+
+def test_install_defer_routes_regex_already_installed():
+    app = Flask(__name__)
+
+    install_regex_url_converter(app)
+
+    with patch(
+        "backend.common.deferred.defer_handler.has_regex_url_converter",
+        return_value=True,
+    ), patch(
+        "backend.common.deferred.defer_handler.install_regex_url_converter",
+        wraps=install_regex_url_converter,
+    ) as mock_install_regex_url_converter:
+        install_defer_routes(app)
+
+    mock_install_regex_url_converter.assert_not_called()
+
+
+def test_handle_defer():
+    app = Flask(__name__)
+    with app.test_request_context() as request_context, patch(
+        "google.appengine.ext.deferred.application.post"
+    ) as mock_post:
+        environ = request_context.request.environ
+        handle_defer("/some/path")
+
+    assert mock_post.called_with(environ)

src/backend/tasks_io/main.py

@@ -1,6 +1,7 @@
 from flask import Flask
 from google.appengine.api import wrap_wsgi_app
 
+from backend.common.deferred import install_defer_routes
 from backend.common.logging import configure_logging
 from backend.common.middleware import install_middleware
 
@@ -10,3 +11,4 @@
 app = Flask(__name__)
 app.wsgi_app = wrap_wsgi_app(app.wsgi_app, use_deferred=True)
 install_middleware(app)
+install_defer_routes(app)