AsynchronousTask: add coroutine async_start method

Add coroutine async_start coroutine method which calls an _async_start
template method. Eventually, subclasses having _start implementations
that need to write to a build log will be required to implement an
_async_start coroutine method to replace the _start method as
discussed in bug 709746.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico <zmedico@gentoo.org>

lib/_emerge/AsynchronousTask.py

@@ -1,10 +1,11 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import signal
 
 from portage import os
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.SlotObject import SlotObject
 
 class AsynchronousTask(SlotObject):
@@ -22,13 +23,28 @@ class AsynchronousTask(SlotObject):
 
 	_cancelled_returncode = - signal.SIGINT
 
+	@coroutine
+	def async_start(self):
+		self._start_hook()
+		yield self._async_start()
+
+	@coroutine
+	def _async_start(self):
+		self._start()
+		coroutine_return()
+		yield None
+
 	def start(self):
 		"""
 		Start an asynchronous task and then return as soon as possible.
 		"""
 		self._start_hook()
 		self._start()
 
+	def _start(self):
+		self.returncode = os.EX_OK
+		self._async_wait()
+
 	def async_wait(self):
 		"""
 		Wait for returncode asynchronously. Notification is available
@@ -49,10 +65,6 @@ def async_wait(self):
 			self._async_wait()
 		return waiter
 
-	def _start(self):
-		self.returncode = os.EX_OK
-		self._async_wait()
-
 	def isAlive(self):
 		return self.returncode is None
 

lib/_emerge/CompositeTask.py

@@ -1,8 +1,10 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from _emerge.AsynchronousTask import AsynchronousTask
 from portage import os
+from portage.util.futures import asyncio
+
 
 class CompositeTask(AsynchronousTask):
 
@@ -97,7 +99,7 @@ def _default_final_exit(self, task):
 	def _start_task(self, task, exit_handler):
 		"""
 		Register exit handler for the given task, set it
-		as self._current_task, and call task.start().
+		as self._current_task, and call task.async_start().
 
 		Subclasses can use this as a generic way to start
 		a task.
@@ -109,7 +111,16 @@ def _start_task(self, task, exit_handler):
 			pass
 		task.addExitListener(exit_handler)
 		self._current_task = task
-		task.start()
+		result = asyncio.ensure_future(task.async_start(), loop=self.scheduler)
+		result.add_done_callback(self._current_task_start_cb)
+
+	def _current_task_start_cb(self, future):
+		try:
+			future.result()
+		except asyncio.CancelledError:
+			self.cancelled = True
+			self._was_cancelled()
+			self._async_wait()
 
 	def _task_queued(self, task):
 		task.addStartListener(self._task_queued_start_handler)

lib/_emerge/TaskSequence.py

@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
@@ -42,6 +42,11 @@ def _start_next_task(self):
 
 		self._start_task(task, self._task_exit_handler)
 
+	def _current_task_start_cb(self, future):
+		CompositeTask._current_task_start_cb(self, future)
+		if self.cancelled:
+			self._task_queue.clear()
+
 	def _task_exit_handler(self, task):
 		if self._default_exit(task) != os.EX_OK:
 			self.wait()

lib/portage/tests/ebuild/test_doebuild_fd_pipes.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2016 Gentoo Foundation
+# Copyright 2013-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import portage
@@ -8,6 +8,7 @@
 from portage.package.ebuild._ipc.QueryCommand import QueryCommand
 from portage.util._async.ForkProcess import ForkProcess
 from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures import asyncio
 from _emerge.Package import Package
 from _emerge.PipeReader import PipeReader
 
@@ -54,6 +55,7 @@ def testDoebuild(self):
 		self.assertEqual(true_binary is None, False,
 			"true command not found")
 
+		loop = asyncio._wrap_loop()
 		dev_null = open(os.devnull, 'wb')
 		playground = ResolverPlayground(ebuilds=ebuilds)
 		try:
@@ -115,7 +117,7 @@ def testDoebuild(self):
 					max_jobs=2)
 
 				try:
-					task_scheduler.start()
+					loop.run_until_complete(task_scheduler.async_start())
 				finally:
 					# PipeReader closes pr
 					os.close(pw)

lib/portage/tests/ebuild/test_doebuild_spawn.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2015 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import textwrap
@@ -90,14 +90,14 @@ def testDoebuildSpawn(self):
 				ebuild_phase = EbuildPhase(background=False,
 					phase=phase, scheduler=scheduler,
 					settings=settings)
-				ebuild_phase.start()
+				scheduler.run_until_complete(ebuild_phase.async_start())
 				ebuild_phase.wait()
 				self.assertEqual(ebuild_phase.returncode, os.EX_OK)
 
 			ebuild_phase = MiscFunctionsProcess(background=False,
 				commands=['success_hooks'],
 				scheduler=scheduler, settings=settings)
-			ebuild_phase.start()
+			scheduler.run_until_complete(ebuild_phase.async_start())
 			ebuild_phase.wait()
 			self.assertEqual(ebuild_phase.returncode, os.EX_OK)
 

lib/portage/tests/ebuild/test_fetch.py

@@ -1,4 +1,4 @@
-# Copyright 2019 Gentoo Authors
+# Copyright 2019-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -15,6 +15,7 @@
 from portage.tests.util.test_socks5 import AsyncHTTPServer
 from portage.util.configparser import ConfigParserError
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.futures.executor.fork import ForkExecutor
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.util._eventloop.global_event_loop import global_event_loop
@@ -193,11 +194,13 @@ def allocate():
 					def deallocate(settings):
 						pass
 
+				@coroutine
 				def async_fetch(pkg, ebuild_path):
 					fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path,
 						fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop)
-					fetcher.start()
-					return fetcher.async_wait()
+					yield fetcher.async_start()
+					result = yield fetcher.async_wait()
+					coroutine_return(result)
 
 				for cpv in ebuilds:
 					metadata = dict(zip(Package.metadata_keys,

lib/portage/tests/ebuild/test_ipc_daemon.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2016 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import tempfile
@@ -155,7 +155,7 @@ def _run(self, event_loop, task_scheduler, timeout):
 		task_scheduler.addExitListener(self._exit_callback)
 
 		try:
-			task_scheduler.start()
+			event_loop.run_until_complete(task_scheduler.async_start())
 			event_loop.run_until_complete(self._run_done)
 			event_loop.run_until_complete(task_scheduler.async_wait())
 		finally:

lib/portage/tests/ebuild/test_spawn.py

@@ -1,4 +1,4 @@
-# Copyright 1998-2013 Gentoo Foundation
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -34,7 +34,7 @@ def testLogfile(self):
 				},
 				scheduler=global_event_loop(),
 				logfile=logfile)
-			proc.start()
+			global_event_loop().run_until_complete(proc.async_start())
 			os.close(null_fd)
 			self.assertEqual(proc.wait(), os.EX_OK)
 			f = io.open(_unicode_encode(logfile,

lib/portage/tests/lazyimport/test_lazy_import_portage_baseline.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import re
@@ -60,14 +60,14 @@ def testLazyImportPortageBaseline(self):
 			args=self._baseline_import_cmd,
 			env=env, fd_pipes={1:slave_fd},
 			scheduler=scheduler)
-		producer.start()
+		scheduler.run_until_complete(producer.async_start())
 		slave_file.close()
 
 		consumer = PipeReader(
 			input_files={"producer" : master_file},
 			scheduler=scheduler)
 
-		consumer.start()
+		scheduler.run_until_complete(consumer.async_start())
 		consumer.wait()
 		self.assertEqual(producer.wait(), os.EX_OK)
 		self.assertEqual(consumer.wait(), os.EX_OK)

lib/portage/tests/locks/test_asynchronous_lock.py

@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import signal
@@ -29,15 +29,15 @@ def _testAsynchronousLock(self):
 						scheduler=scheduler, _force_async=force_async,
 						_force_thread=True,
 						_force_dummy=force_dummy)
-					async_lock.start()
+					scheduler.run_until_complete(async_lock.async_start())
 					self.assertEqual(async_lock.wait(), os.EX_OK)
 					self.assertEqual(async_lock.returncode, os.EX_OK)
 					scheduler.run_until_complete(async_lock.async_unlock())
 
 				async_lock = AsynchronousLock(path=path,
 					scheduler=scheduler, _force_async=force_async,
 					_force_process=True)
-				async_lock.start()
+				scheduler.run_until_complete(async_lock.async_start())
 				self.assertEqual(async_lock.wait(), os.EX_OK)
 				self.assertEqual(async_lock.returncode, os.EX_OK)
 				scheduler.run_until_complete(async_lock.async_unlock())
@@ -63,7 +63,7 @@ def _testAsynchronousLockWait(self):
 		try:
 			path = os.path.join(tempdir, 'lock_me')
 			lock1 = AsynchronousLock(path=path, scheduler=scheduler)
-			lock1.start()
+			scheduler.run_until_complete(lock1.async_start())
 			self.assertEqual(lock1.wait(), os.EX_OK)
 			self.assertEqual(lock1.returncode, os.EX_OK)
 
@@ -73,7 +73,7 @@ def _testAsynchronousLockWait(self):
 			# one time concurrently.
 			lock2 = AsynchronousLock(path=path, scheduler=scheduler,
 				_force_async=True, _force_process=True)
-			lock2.start()
+			scheduler.run_until_complete(lock2.async_start())
 			# lock2 should be waiting for lock1 to release
 			self.assertEqual(lock2.poll(), None)
 			self.assertEqual(lock2.returncode, None)
@@ -104,12 +104,12 @@ def _testAsynchronousLockWaitCancel(self):
 		try:
 			path = os.path.join(tempdir, 'lock_me')
 			lock1 = AsynchronousLock(path=path, scheduler=scheduler)
-			lock1.start()
+			scheduler.run_until_complete(lock1.async_start())
 			self.assertEqual(lock1.wait(), os.EX_OK)
 			self.assertEqual(lock1.returncode, os.EX_OK)
 			lock2 = AsynchronousLock(path=path, scheduler=scheduler,
 				_force_async=True, _force_process=True)
-			lock2.start()
+			scheduler.run_until_complete(lock2.async_start())
 			# lock2 should be waiting for lock1 to release
 			self.assertEqual(lock2.poll(), None)
 			self.assertEqual(lock2.returncode, None)
@@ -142,12 +142,12 @@ def _testAsynchronousLockWaitKill(self):
 		try:
 			path = os.path.join(tempdir, 'lock_me')
 			lock1 = AsynchronousLock(path=path, scheduler=scheduler)
-			lock1.start()
+			scheduler.run_until_complete(lock1.async_start())
 			self.assertEqual(lock1.wait(), os.EX_OK)
 			self.assertEqual(lock1.returncode, os.EX_OK)
 			lock2 = AsynchronousLock(path=path, scheduler=scheduler,
 				_force_async=True, _force_process=True)
-			lock2.start()
+			scheduler.run_until_complete(lock2.async_start())
 			# lock2 should be waiting for lock1 to release
 			self.assertEqual(lock2.poll(), None)
 			self.assertEqual(lock2.returncode, None)

lib/portage/tests/process/test_PopenProcess.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2013 Gentoo Foundation
+# Copyright 2012-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess
@@ -34,7 +34,7 @@ def _testPipeReader(self, test_string):
 		consumer = producer.pipe_reader
 		consumer.input_files = {"producer" : producer.proc.stdout}
 
-		producer.start()
+		global_event_loop().run_until_complete(producer.async_start())
 		producer.wait()
 
 		self.assertEqual(producer.returncode, os.EX_OK)
@@ -58,7 +58,7 @@ def _testPipeLogger(self, test_string):
 
 			producer.pipe_reader = consumer
 
-			producer.start()
+			global_event_loop().run_until_complete(producer.async_start())
 			producer.wait()
 
 			self.assertEqual(producer.returncode, os.EX_OK)

lib/portage/tests/process/test_PopenProcessBlockingIO.py

@@ -1,4 +1,4 @@
-# Copyright 2012 Gentoo Foundation
+# Copyright 2012-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess
@@ -40,7 +40,7 @@ def _testPipeReader(self, test_string):
 		consumer = producer.pipe_reader
 		consumer.input_files = {"producer" : producer.proc.stdout}
 
-		producer.start()
+		global_event_loop().run_until_complete(producer.async_start())
 		producer.wait()
 
 		self.assertEqual(producer.returncode, os.EX_OK)

lib/portage/tests/process/test_poll.py

@@ -1,4 +1,4 @@
-# Copyright 1998-2019 Gentoo Authors
+# Copyright 1998-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -67,7 +67,7 @@ def _testPipeReader(self, master_fd, slave_fd, test_string):
 			input_files={"producer" : master_file},
 			_use_array=self._use_array,
 			scheduler=scheduler)
-		consumer.start()
+		scheduler.run_until_complete(consumer.async_start())
 
 		producer = scheduler.run_until_complete(asyncio.create_subprocess_exec(
 			"bash", "-c", self._echo_cmd % test_string,