Merge pull request #1029 from MVrachev/fix-1010

Fix error "[Errno 111] Connection refused" and make logs more usable

tests/simple_server.py

@@ -66,9 +66,16 @@ def log_request(self, code='-', size='-'):
 
 # NOTE: On Windows/Python2 tests that use this simple_server.py in a
 # subprocesses hang after a certain amount of requests (~68), if a PIPE is
-# passed as Popen's stderr argument. As a simple workaround we silence the
-# server on those Windows/Py2 to not fill the buffer.
-if six.PY2 and platform.system() == 'Windows':
+# passed as Popen's stderr argument. This problem doesn't emerge if
+# we silence the HTTP messages.
+# If you decide to receive the HTTP messages, then this bug
+# could reappear.
+use_quiet_http_request_handler = True
+
+if len(sys.argv) > 2:
+  use_quiet_http_request_handler = sys.argv[2]
+
+if use_quiet_http_request_handler:
   handler = QuietHTTPRequestHandler
 else:
   handler = SimpleHTTPRequestHandler

tests/test_arbitrary_package_attack.py

@@ -81,7 +81,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: ' + str(cls.server_process.pid))
     logger.info('Serving on port: ' + str(cls.SERVER_PORT))

tests/test_endless_data_attack.py

@@ -83,7 +83,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: '+str(cls.server_process.pid))
     logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_extraneous_dependencies_attack.py

@@ -87,7 +87,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: '+str(cls.server_process.pid))
     logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_indefinite_freeze_attack.py

@@ -94,7 +94,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: '+str(cls.server_process.pid))
     logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_key_revocation_integration.py

@@ -83,7 +83,7 @@ def setUpClass(cls):
     # key files, etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('\n\tServer process started.')
     logger.info('\tServer process id: '+str(cls.server_process.pid))
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))

tests/test_mix_and_match_attack.py

@@ -88,7 +88,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: '+str(cls.server_process.pid))
     logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_multiple_repositories_integration.py

@@ -128,17 +128,22 @@ def setUp(self):
     while self.SERVER_PORT == self.SERVER_PORT2:
       self.SERVER_PORT2 = random.SystemRandom().randint(30000, 45000)
 
-    command = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT)]
-    command2 = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT2)]
+    # Needed because in some tests simple_server.py cannot be found.
+    # The reason is that the current working directory
+    # has been changed when executing a subprocess.
+    SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
 
-    self.server_process = subprocess.Popen(command, stderr=subprocess.PIPE,
+    command = ['python', SIMPLE_SERVER_PATH, str(self.SERVER_PORT)]
+    command2 = ['python', SIMPLE_SERVER_PATH, str(self.SERVER_PORT2)]
+
+    self.server_process = subprocess.Popen(command,
         cwd=self.repository_directory)
 
     logger.debug('Server process started.')
     logger.debug('Server process id: ' + str(self.server_process.pid))
     logger.debug('Serving on port: ' + str(self.SERVER_PORT))
 
-    self.server_process2 = subprocess.Popen(command2, stderr=subprocess.PIPE,
+    self.server_process2 = subprocess.Popen(command2,
         cwd=self.repository_directory2)
 
 

tests/test_replay_attack.py

@@ -87,7 +87,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('Server process started.')
     logger.info('Server process id: '+str(cls.server_process.pid))
     logger.info('Serving on port: '+str(cls.SERVER_PORT))

tests/test_slow_retrieval_attack.py

@@ -103,7 +103,7 @@ def _start_slow_server(self, mode):
     # as a delegated role 'targets/role1', three target files, five key files,
     # etc.
     command = ['python', 'slow_retrieval_server.py', str(self.SERVER_PORT), mode]
-    server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    server_process = subprocess.Popen(command)
     logger.info('Slow Retrieval Server process started.')
     logger.info('Server process id: '+str(server_process.pid))
     logger.info('Serving on port: '+str(self.SERVER_PORT))

tests/test_updater.py

@@ -90,6 +90,11 @@ def setUpClass(cls):
     # temporary files are always removed, even when exceptions occur.
     cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd())
 
+    # Needed because in some tests simple_server.py cannot be found.
+    # The reason is that the current working directory
+    # has been changed when executing a subprocess.
+    cls.SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
+
     # Launch a SimpleHTTPServer (serves files in the current directory).
     # Test cases will request metadata and target files that have been
     # pre-generated in 'tuf/tests/repository_data', which will be served
@@ -98,8 +103,8 @@ def setUpClass(cls):
     # as a delegated role 'targets/role1', three target files, five key files,
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
-    command = ['python', '-m', 'tests.simple_server', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    command = ['python', cls.SIMPLE_SERVER_PATH, str(cls.SERVER_PORT)]
+    cls.server_process = subprocess.Popen(command)
     logger.info('\n\tServer process started.')
     logger.info('\tServer process id: '+str(cls.server_process.pid))
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))
@@ -1091,8 +1096,8 @@ def test_6_get_one_valid_targetinfo(self):
     # The SimpleHTTPServer started in the setupclass has a tendency to
     # timeout in Windows after a few tests.
     SERVER_PORT = random.randint(30000, 45000)
-    command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
-    server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+    server_process = subprocess.Popen(command)
 
     # NOTE: Following error is raised if a delay is not long enough:
     # <urlopen error [Errno 111] Connection refused>
@@ -1359,8 +1364,8 @@ def test_7_updated_targets(self):
     # The SimpleHTTPServer started in the setupclass has a tendency to
     # timeout in Windows after a few tests.
     SERVER_PORT = random.randint(30000, 45000)
-    command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
-    server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+    server_process = subprocess.Popen(command)
 
     # NOTE: Following error is raised if a delay is not long enough to allow
     # the server process to set up and start listening:
@@ -1491,8 +1496,8 @@ def test_8_remove_obsolete_targets(self):
     # The SimpleHTTPServer started in the setupclass has a tendency to
     # timeout in Windows after a few tests.
     SERVER_PORT = random.randint(30000, 45000)
-    command = ['python', '-m', 'tests.simple_server', str(SERVER_PORT)]
-    server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    command = ['python', self.SIMPLE_SERVER_PATH, str(SERVER_PORT)]
+    server_process = subprocess.Popen(command)
 
     # NOTE: Following error is raised if a delay is not long enough to allow
     # the server process to set up and start listening:
@@ -1824,6 +1829,11 @@ def setUp(self):
     self.temporary_repository_root = self.make_temp_directory(directory=
         self.temporary_directory)
 
+    # Needed because in some tests simple_server.py cannot be found.
+    # The reason is that the current working directory
+    # has been changed when executing a subprocess.
+    self.SIMPLE_SERVER_PATH = os.path.join(os.getcwd(), 'simple_server.py')
+
     # The original repository, keystore, and client directories will be copied
     # for each test case.
     original_repository = os.path.join(original_repository_files, 'repository')
@@ -1875,17 +1885,17 @@ def setUp(self):
     self.SERVER_PORT = 30001
     self.SERVER_PORT2 = 30002
 
-    command = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT)]
-    command2 = ['python', '-m', 'tests.simple_server', str(self.SERVER_PORT2)]
+    command = ['python', self.SIMPLE_SERVER_PATH, str(self.SERVER_PORT)]
+    command2 = ['python', self.SIMPLE_SERVER_PATH, str(self.SERVER_PORT2)]
 
-    self.server_process = subprocess.Popen(command, stderr=subprocess.PIPE,
+    self.server_process = subprocess.Popen(command,
         cwd=self.repository_directory)
 
     logger.debug('Server process started.')
     logger.debug('Server process id: ' + str(self.server_process.pid))
     logger.debug('Serving on port: ' + str(self.SERVER_PORT))
 
-    self.server_process2 = subprocess.Popen(command2, stderr=subprocess.PIPE,
+    self.server_process2 = subprocess.Popen(command2,
         cwd=self.repository_directory2)
 
     logger.debug('Server process 2 started.')

tests/test_updater_root_rotation_integration.py

@@ -90,7 +90,7 @@ def setUpClass(cls):
     # etc.
     cls.SERVER_PORT = random.randint(30000, 45000)
     command = ['python', 'simple_server.py', str(cls.SERVER_PORT)]
-    cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    cls.server_process = subprocess.Popen(command)
     logger.info('\n\tServer process started.')
     logger.info('\tServer process id: '+str(cls.server_process.pid))
     logger.info('\tServing on port: '+str(cls.SERVER_PORT))