Changes for 3.9 along with nit fixes (pydoc comments, flaky tests) (#751

)

- Fixing `'''` to `"""` string for PyDocs
- Adding critical logs to hit that code path
- Marking some tests as flaky to allow it to be rerun.
- Adding some sleep when starting the test host/worker to allow it to start on mac.
- With `asycio.Task.creat_task()` now removed from Python 3.9, refactored it to work for 3.6-3.9.
- Bump `grpcio` and `grpcio-tools` from `1.26.0` to `1.32.0`.

azure_functions_worker/dispatcher.py

@@ -37,6 +37,14 @@
 
 _TRUE = "true"
 
+"""In Python 3.6, the current_task method was in the Task class, but got moved
+out in 3.7+ and fully removed in 3.9. Thus, to support 3.6 and 3.9 together, we
+need to switch the implementation of current_task for 3.6.
+"""
+_CURRENT_TASK = asyncio.Task.current_task \
+    if (sys.version_info[0] == 3 and sys.version_info[1] == 6) \
+    else asyncio.current_task
+
 
 class DispatcherMeta(type):
 
@@ -302,7 +310,7 @@ async def _handle__invocation_request(self, req):
             invoc_request.trace_context.attributes)
         # Set the current `invocation_id` to the current task so
         # that our logging handler can find it.
-        current_task = asyncio.Task.current_task(self._loop)
+        current_task = _CURRENT_TASK(self._loop)
         assert isinstance(current_task, ContextEnabledTask)
         current_task.set_azure_invocation_id(invocation_id)
 
@@ -576,7 +584,7 @@ class ContextEnabledTask(asyncio.Task):
     def __init__(self, coro, loop):
         super().__init__(coro, loop=loop)
 
-        current_task = asyncio.Task.current_task(loop)
+        current_task = _CURRENT_TASK(loop)
         if current_task is not None:
             invocation_id = getattr(
                 current_task, self.AZURE_INVOCATION_ID, None)
@@ -590,7 +598,7 @@ def set_azure_invocation_id(self, invocation_id: str) -> None:
 def get_current_invocation_id() -> Optional[str]:
     loop = asyncio._get_running_loop()
     if loop is not None:
-        current_task = asyncio.Task.current_task(loop)
+        current_task = _CURRENT_TASK(loop)
         if current_task is not None:
             task_invocation_id = getattr(current_task,
                                          ContextEnabledTask.AZURE_INVOCATION_ID,

azure_functions_worker/testutils.py

@@ -677,6 +677,7 @@ def start_webhost(*, script_dir=None, stdout=None):
     port = _find_open_port()
     proc = popen_webhost(stdout=stdout, stderr=subprocess.STDOUT,
                          script_root=script_root, port=port)
+    time.sleep(3)  # Giving host some time to start fully.
 
     addr = f'http://{LOCALHOST}:{port}'
     for _ in range(10):
@@ -690,7 +691,8 @@ def start_webhost(*, script_dir=None, stdout=None):
         except requests.exceptions.ConnectionError:
             pass
 
-        time.sleep(1)
+        time.sleep(2)
+
     else:
         proc.terminate()
         try:

python/prodV2/worker.py

@@ -17,20 +17,20 @@
 
 
 def is_azure_environment():
-    '''Check if the function app is running on the cloud'''
+    """Check if the function app is running on the cloud"""
     return (AZURE_CONTAINER_NAME in os.environ
             or AZURE_WEBSITE_INSTANCE_ID in os.environ)
 
 
 def add_script_root_to_sys_path():
-    '''Append function project root to module finding sys.path'''
+    """Append function project root to module finding sys.path"""
     functions_script_root = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT)
     if functions_script_root is not None:
         sys.path.append(functions_script_root)
 
 
 def determine_user_pkg_paths():
-    '''This finds the user packages when function apps are running on the cloud
+    """This finds the user packages when function apps are running on the cloud
 
     For Python 3.6 app, the third-party packages can live in any of the paths:
         /home/site/wwwroot/.python_packages/lib/site-packages
@@ -39,7 +39,7 @@ def determine_user_pkg_paths():
 
     For Python 3.7, we only accept:
         /home/site/wwwroot/.python_packages/lib/site-packages
-    '''
+    """
     minor_version = sys.version_info[1]
 
     home = Path.home()

python/prodV3/worker.config.json

@@ -1,9 +1,9 @@
 {
     "description":{
         "language":"python",
-        "defaultRuntimeVersion":"3.6",
+        "defaultRuntimeVersion":"3.8",
         "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"],
-        "supportedRuntimeVersions":["3.6", "3.7", "3.8"],
+        "supportedRuntimeVersions":["3.6", "3.7", "3.8", "3.9"],
         "supportedArchitectures":["X64", "X86"],
         "extensions":[".py"],
         "defaultExecutablePath":"python",

python/prodV3/worker.py

@@ -17,20 +17,20 @@
 
 
 def is_azure_environment():
-    '''Check if the function app is running on the cloud'''
+    """Check if the function app is running on the cloud"""
     return (AZURE_CONTAINER_NAME in os.environ
             or AZURE_WEBSITE_INSTANCE_ID in os.environ)
 
 
 def add_script_root_to_sys_path():
-    '''Append function project root to module finding sys.path'''
+    """Append function project root to module finding sys.path"""
     functions_script_root = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT)
     if functions_script_root is not None:
         sys.path.append(functions_script_root)
 
 
 def determine_user_pkg_paths():
-    '''This finds the user packages when function apps are running on the cloud
+    """This finds the user packages when function apps are running on the cloud
 
     For Python 3.6 app, the third-party packages can live in any of the paths:
         /home/site/wwwroot/.python_packages/lib/site-packages
@@ -39,7 +39,7 @@ def determine_user_pkg_paths():
 
     For Python 3.7 and Python 3.8, we only accept:
         /home/site/wwwroot/.python_packages/lib/site-packages
-    '''
+    """
     minor_version = sys.version_info[1]
 
     home = Path.home()
@@ -51,7 +51,7 @@ def determine_user_pkg_paths():
         user_pkg_paths.append(os.path.join(venv_pkgs_path, PKGS_36))
         user_pkg_paths.append(os.path.join(pkgs_path, PKGS_36))
         user_pkg_paths.append(os.path.join(pkgs_path, PKGS))
-    elif minor_version in (7, 8):
+    elif minor_version in (7, 8, 9):
         user_pkg_paths.append(os.path.join(pkgs_path, PKGS))
     else:
         raise RuntimeError(f'Unsupported Python version: 3.{minor_version}')

setup.py

@@ -282,8 +282,8 @@ def run(self):
               'azure_functions_worker.utils',
               'azure_functions_worker._thirdparty'],
     install_requires=[
-        'grpcio~=1.26.0',
-        'grpcio-tools~=1.26.0',
+        'grpcio~=1.32.0',
+        'grpcio-tools~=1.32.0',
     ],
     extras_require={
         'dev': [
@@ -299,7 +299,8 @@ def run(self):
             'pytest-cov',
             'pytest-xdist',
             'pytest-randomly',
-            'pytest-instafail'
+            'pytest-instafail',
+            'pytest-rerunfailures'
         ]
     },
     include_package_data=True,

tests/endtoend/test_eventhub_batch_functions.py

@@ -10,13 +10,13 @@
 
 
 class TestEventHubFunctions(testutils.WebHostTestCase):
-    '''Test EventHub Trigger and Output Bindings (cardinality: many).
+    """Test EventHub Trigger and Output Bindings (cardinality: many).
 
     Each testcase consists of 3 part:
     1. An eventhub_output_batch HTTP trigger for generating EventHub event
     2. An eventhub_multiple EventHub trigger for converting event into blob
     3. A get_eventhub_batch_triggered HTTP trigger for the event body
-    '''
+    """
 
     @classmethod
     def get_script_dir(cls):

tests/endtoend/test_eventhub_functions.py

@@ -9,13 +9,13 @@
 
 
 class TestEventHubFunctions(testutils.WebHostTestCase):
-    '''Test EventHub Trigger and Output Bindings (cardinality: one).
+    """Test EventHub Trigger and Output Bindings (cardinality: one).
 
     Each testcase consists of 3 part:
     1. An eventhub_output HTTP trigger for generating EventHub event
     2. An actual eventhub_trigger EventHub trigger for storing event into blob
     3. A get_eventhub_triggered HTTP trigger for retrieving event info blob
-    '''
+    """
 
     @classmethod
     def get_script_dir(cls):

tests/unittests/http_functions/debug_logging/main.py

@@ -6,6 +6,7 @@
 
 
 def main(req: azure.functions.HttpRequest):
+    logging.critical('logging critical', exc_info=True)
     logging.info('logging info', exc_info=True)
     logging.warning('logging warning', exc_info=True)
     logging.debug('logging debug', exc_info=True)

tests/unittests/http_functions/debug_user_logging/main.py

@@ -9,6 +9,7 @@
 
 
 def main(req: azure.functions.HttpRequest):
+    logging.critical('logging critical', exc_info=True)
     logger.info('logging info', exc_info=True)
     logger.warning('logging warning', exc_info=True)
     logger.debug('logging debug', exc_info=True)

tests/unittests/http_functions/sync_logging/main.py

@@ -14,5 +14,6 @@ def main(req: azure.functions.HttpRequest):
         1 / 0
     except ZeroDivisionError:
         logger.error('a gracefully handled error', exc_info=True)
+        logger.error('a gracefully handled critical error', exc_info=True)
     time.sleep(0.05)
     return 'OK-sync'

tests/unittests/load_functions/pytest/__init__.py

@@ -2,6 +2,6 @@
 # Licensed under the MIT License.
 
 
-'''This module pytest is provided inside customer's code,
-used for checking module name collision'''
+"""This module pytest is provided inside customer's code,
+used for checking module name collision"""
 __version__ = 'from.customer.code'

tests/unittests/test_dispatcher.py

@@ -18,38 +18,32 @@ def tearDown(self):
         os.environ.update(self._pre_env)
 
     async def test_dispatcher_sync_threadpool_default_worker(self):
-        '''Test if the sync threadpool has maximum worker count set to 1
+        """Test if the sync threadpool has maximum worker count set to 1
         by default
-        '''
+        """
         ctrl = testutils.start_mockhost(script_root=self.dispatcher_funcs_dir)
 
         async with ctrl as host:
             await self._check_if_function_is_ok(host)
-
-            # Ensure the dispatcher sync threadpool count is set to 1
             self.assertEqual(ctrl._worker._sync_tp_max_workers, 1)
 
     async def test_dispatcher_sync_threadpool_set_worker(self):
-        '''Test if the sync threadpool maximum worker can be set
-        '''
+        """Test if the sync threadpool maximum worker can be set
+        """
         # Configure thread pool max worker
         os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '5'})
         ctrl = testutils.start_mockhost(script_root=self.dispatcher_funcs_dir)
 
         async with ctrl as host:
             await self._check_if_function_is_ok(host)
-
-            # Ensure the dispatcher sync threadpool count is set to 1
             self.assertEqual(ctrl._worker._sync_tp_max_workers, 5)
 
     @patch('azure_functions_worker.dispatcher.logger')
-    async def test_dispatcher_sync_threadpool_invalid_worker_count(
-        self,
-        mock_logger
-    ):
-        '''Test when sync threadpool maximum worker is set to an invalid value,
+    async def test_dispatcher_sync_threadpool_invalid_worker_count(self,
+                                                                   mock_logger):
+        """Test when sync threadpool maximum worker is set to an invalid value,
         the host should fallback to default value 1
-        '''
+        """
         # Configure thread pool max worker to an invalid value
         os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: 'invalid'})
         ctrl = testutils.start_mockhost(script_root=self.dispatcher_funcs_dir)
@@ -64,10 +58,8 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(
             f'{PYTHON_THREADPOOL_THREAD_COUNT} must be an integer')
 
     @patch('azure_functions_worker.dispatcher.logger')
-    async def test_dispatcher_sync_threadpool_below_min_setting(
-        self,
-        mock_logger
-    ):
+    async def test_dispatcher_sync_threadpool_below_min_setting(self,
+                                                                mock_logger):
         # Configure thread pool max worker to an invalid value
         os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '0'})
         ctrl = testutils.start_mockhost(script_root=self.dispatcher_funcs_dir)

tests/unittests/test_http_functions.py

@@ -6,6 +6,8 @@
 import typing
 import os
 
+import pytest
+
 from azure_functions_worker import testutils
 
 
@@ -323,6 +325,7 @@ def check_log_import_module_troubleshooting_url(self,
                       "Troubleshooting Guide: "
                       "https://aka.ms/functions-modulenotfound", host_out)
 
+    @pytest.mark.flaky(reruns=3)
     def test_print_logging_no_flush(self):
         r = self.webhost.request('GET', 'print_logging?message=Secret42')
         self.assertEqual(r.status_code, 200)
@@ -331,6 +334,7 @@ def test_print_logging_no_flush(self):
     def check_log_print_logging_no_flush(self, host_out: typing.List[str]):
         self.assertIn('Secret42', host_out)
 
+    @pytest.mark.flaky(reruns=3)
     def test_print_logging_with_flush(self):
         r = self.webhost.request('GET',
                                  'print_logging?flush=true&message=Secret42')
@@ -346,6 +350,7 @@ def test_print_to_console_stdout(self):
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.text, 'OK-print-logging')
 
+    @pytest.mark.flaky(reruns=3)
     def check_log_print_to_console_stdout(self, host_out: typing.List[str]):
         # System logs stdout should not exist in host_out
         self.assertNotIn('Secret42', host_out)