Merge branch 'chandan/python3.8' into 'master'

Add test environment for Python 3.8

Closes #1173

See merge request BuildStream/buildstream!1647

.gitlab-ci.yml

@@ -24,12 +24,12 @@ variables:
   # Our own variables
   # Version of the docker images we should use for all the images.
   # This is taken from buildstream/buildstream-docker-images
-  DOCKER_IMAGE_VERSION: master-103717922
+  DOCKER_IMAGE_VERSION: master-105004115
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   PYTEST_ARGS: "--color=yes --integration -n 2"
   TEST_COMMAND: "tox -- ${PYTEST_ARGS}"
-  EXTERNAL_TESTS_COMMAND: "tox -e py35-external,py36-external,py37-external -- ${PYTEST_ARGS}"
+  EXTERNAL_TESTS_COMMAND: "tox -e py35-external,py36-external,py37-external,py38-external -- ${PYTEST_ARGS}"
   COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
@@ -88,6 +88,21 @@ tests-centos-7.6:
   <<: *tests
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-${DOCKER_IMAGE_VERSION}
 
+tests-python-3.8-buster:
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-python:3.8-buster-${DOCKER_IMAGE_VERSION}
+  <<: *tests
+  variables:
+    # Our testsuite has issues with coverage on Python 3.8 so disable coverage
+    # in the meantime. For more details, see
+    # https://gitlab.com/BuildStream/buildstream/issues/1173.
+    TEST_COMMAND: "tox -e py38-nocover -- ${PYTEST_ARGS}"
+
+    # This particular testsuite image has both Python 3.7 and Python 3.8 so we
+    # need to explicitly force the 3.8 environment.
+    # Once Python 3.8 is available in distros, we should switch to such an
+    # image, and remove the following override.
+    EXTERNAL_TESTS_COMMAND: "tox -e py38-external -- ${PYTEST_ARGS}"
+
 overnight-fedora-30-aarch64:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-${DOCKER_IMAGE_VERSION}
   tags:
@@ -397,8 +412,8 @@ overnight-randomized:
   - chown -R buildstream:buildstream .
 
   # Don't run tests multiprocessed here, the randomized order doesn't like that
-  - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration"
-  - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized-external -- --color=yes --integration"
+  - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized,py38-randomized -- --color=yes --integration"
+  - su buildstream -c "tox -e py35-randomized-external,py36-randomized-external,py37-randomized-external,py38-randomized-external -- --color=yes --integration"
 
   # We need to override the exclusion from the template in order to run on schedules
   except: []

src/buildstream/_scheduler/scheduler.py

@@ -171,6 +171,12 @@ def run(self, queues, casd_process_manager):
         # Hold on to the queues to process
         self.queues = queues
 
+        # NOTE: Enforce use of `SafeChildWatcher` as we generally don't want
+        # background threads.
+        # In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and
+        # not `SafeChildWatcher`.
+        asyncio.set_child_watcher(asyncio.SafeChildWatcher())
+
         # Ensure that we have a fresh new event loop, in case we want
         # to run another test in this thread.
         self.loop = asyncio.new_event_loop()

tox.ini

@@ -2,7 +2,7 @@
 # Tox global configuration
 #
 [tox]
-envlist = py{35,36,37}
+envlist = py{35,36,37,38}
 skip_missing_interpreters = true
 isolated_build = true
 
@@ -14,22 +14,22 @@ isolated_build = true
 [testenv]
 usedevelop =
     # This is required by Cython in order to get coverage for cython files.
-    py{35,36,37}-!nocover: True
+    py{35,36,37,38}-!nocover: True
 
 commands =
     # Running with coverage reporting enabled
-    py{35,36,37}-!external-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
-    py{35,36,37}-!nocover: mkdir -p .coverage-reports
+    py{35,36,37,38}-!external-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
+    py{35,36,37,38}-!nocover: mkdir -p .coverage-reports
     external-!nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} --cov=buildstream --cov-append {posargs}
-    py{35,36,37}-!external-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
-    py{35,36,37}-external-!nocover: - mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+    py{35,36,37,38}-!external-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+    py{35,36,37,38}-external-!nocover: - mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
     # Running with coverage reporting disabled
-    py{35,36,37}-!external-nocover: pytest --basetemp {envtmpdir} {posargs}
+    py{35,36,37,38}-!external-nocover: pytest --basetemp {envtmpdir} {posargs}
     external-nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} {posargs}
 deps =
-    py{35,36,37}: -rrequirements/requirements.txt
-    py{35,36,37}: -rrequirements/dev-requirements.txt
-    py{35,36,37}: -rrequirements/plugin-requirements.txt
+    py{35,36,37,38}: -rrequirements/requirements.txt
+    py{35,36,37,38}: -rrequirements/dev-requirements.txt
+    py{35,36,37,38}: -rrequirements/plugin-requirements.txt
     git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@5b004e5850ab0e987c00c681b2c768f6ae02586b
 
     # Only require coverage and pytest-cov when using it
@@ -58,17 +58,17 @@ passenv =
 # These keys are not inherited by any other sections
 #
 setenv =
-    py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
-    py{35,36,37}: BST_TEST_HOME = {envtmpdir}
-    py{35,36,37}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache
-    py{35,36,37}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config
-    py{35,36,37}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share
+    py{35,36,37,38}: COVERAGE_FILE = {envtmpdir}/.coverage
+    py{35,36,37,38}: BST_TEST_HOME = {envtmpdir}
+    py{35,36,37,38}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache
+    py{35,36,37,38}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config
+    py{35,36,37,38}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share
     # This is required to get coverage for Cython
-    py{35,36,37}-!nocover: BST_CYTHON_TRACE = 1
+    py{35,36,37,38}-!nocover: BST_CYTHON_TRACE = 1
     randomized: PYTEST_ADDOPTS="--random-order-bucket=global"
 
 whitelist_externals =
-    py{35,36,37}:
+    py{35,36,37,38}:
         mv
         mkdir