git: add key refresh retry (bug 660732)

pym/portage/sync/modules/git/git.py

@@ -8,6 +8,7 @@
 import portage
 from portage import os
 from portage.util import writemsg_level, shlex_split
+from portage.util.futures import asyncio
 from portage.output import create_color_func, EOutput
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")
@@ -197,10 +198,8 @@ def verify_head(self, revision='-1'):
 					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
 					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
 						openpgp_env.import_key(f)
-					out.ebegin('Refreshing keys from keyserver')
-					openpgp_env.refresh_keys()
-					out.eend(0)
-				except GematoException as e:
+					self._refresh_keys(openpgp_env)
+				except (GematoException, asyncio.TimeoutError) as e:
 					writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n"
 							% (e,),
 							level=logging.ERROR, noiselevel=-1)

pym/portage/sync/modules/rsync/rsync.py

@@ -7,7 +7,6 @@
 import signal
 import socket
 import datetime
-import functools
 import io
 import re
 import random
@@ -23,10 +22,8 @@
 bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
-from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util import writemsg, writemsg_stdout
 from portage.util.futures import asyncio
-from portage.util.futures.executor.fork import ForkExecutor
 from portage.sync.getaddrinfo_validate import getaddrinfo_validate
 from _emerge.UserQuery import UserQuery
 from portage.sync.syncbase import NewBase
@@ -149,39 +146,11 @@ def update(self):
 			# will not be performed and the user will have to fix it and try again,
 			# so we may as well bail out before actual rsync happens.
 			if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
-
 				try:
 					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
 					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
 						openpgp_env.import_key(f)
-					out.ebegin('Refreshing keys from keyserver')
-					retry_decorator = self._key_refresh_retry_decorator()
-					if retry_decorator is None:
-						openpgp_env.refresh_keys()
-					else:
-						def noisy_refresh_keys():
-							"""
-							Since retry does not help for some types of
-							errors, display errors as soon as they occur.
-							"""
-							try:
-								openpgp_env.refresh_keys()
-							except Exception as e:
-								writemsg_level("%s\n" % (e,),
-									level=logging.ERROR, noiselevel=-1)
-								raise # retry
-
-						# The ThreadPoolExecutor that asyncio uses by default
-						# does not support cancellation of tasks, therefore
-						# use ForkExecutor for task cancellation support, in
-						# order to enforce timeouts.
-						loop = global_event_loop()
-						with ForkExecutor(loop=loop) as executor:
-							func_coroutine = functools.partial(loop.run_in_executor,
-								executor, noisy_refresh_keys)
-							decorated_func = retry_decorator(func_coroutine, loop=loop)
-							loop.run_until_complete(decorated_func())
-					out.eend(0)
+					self._refresh_keys(openpgp_env)
 				except (GematoException, asyncio.TimeoutError) as e:
 					writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
 							% (e,),

pym/portage/sync/syncbase.py

@@ -7,13 +7,16 @@
 '''
 
 from __future__ import unicode_literals
+import functools
 import logging
 import os
 
 import portage
 from portage.util import writemsg_level
+from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.util.backoff import RandomExponentialBackoff
 from portage.util.futures.retry import retry
+from portage.util.futures.executor.fork import ForkExecutor
 from . import _SUBMODULE_PATH_MAP
 
 class SyncBase(object):
@@ -189,6 +192,44 @@ def _key_refresh_retry_decorator(self):
 				multiplier=(1 if retry_delay_mult is None else retry_delay_mult),
 				base=(2 if retry_delay_exp_base is None else retry_delay_exp_base)))
 
+	def _refresh_keys(self, openpgp_env):
+		"""
+		Refresh keys stored in openpgp_env. Raises gemato.exceptions.GematoException
+		or asyncio.TimeoutError on failure.
+
+		@param openpgp_env: openpgp environment
+		@type openpgp_env: gemato.openpgp.OpenPGPEnvironment
+		"""
+		out = portage.output.EOutput(quiet=('--quiet' in self.options['emerge_config'].opts))
+		out.ebegin('Refreshing keys from keyserver')
+		retry_decorator = self._key_refresh_retry_decorator()
+		if retry_decorator is None:
+			openpgp_env.refresh_keys()
+		else:
+			def noisy_refresh_keys():
+				"""
+				Since retry does not help for some types of
+				errors, display errors as soon as they occur.
+				"""
+				try:
+					openpgp_env.refresh_keys()
+				except Exception as e:
+					writemsg_level("%s\n" % (e,),
+						level=logging.ERROR, noiselevel=-1)
+					raise # retry
+
+			# The ThreadPoolExecutor that asyncio uses by default
+			# does not support cancellation of tasks, therefore
+			# use ForkExecutor for task cancellation support, in
+			# order to enforce timeouts.
+			loop = global_event_loop()
+			with ForkExecutor(loop=loop) as executor:
+				func_coroutine = functools.partial(loop.run_in_executor,
+					executor, noisy_refresh_keys)
+				decorated_func = retry_decorator(func_coroutine, loop=loop)
+				loop.run_until_complete(decorated_func())
+		out.eend(0)
+
 
 class NewBase(SyncBase):
 	'''Subclasses Syncbase adding a new() and runs it