Prevent infinite respawn of child processes

Rather than looking for an exit code of 2 to prevent child processes
from respawning, we look for any non-zero status.

Fixes bug 1074132.
Fixes bug 1065197.

Change-Id: I74ac13737d8a9b8710a8268eb1d67fbe5e75d491

glance/common/wsgi.py

@@ -219,7 +219,7 @@ def wait_on_children(self):
                 if os.WIFEXITED(status) or os.WIFSIGNALED(status):
                     self.logger.error(_('Removing dead child %s') % pid)
                     self.children.remove(pid)
-                    if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 2:
+                    if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
                         self.logger.error(_('Not respawning child %d, cannot '
                                             'recover from termination') % pid)
                         if not self.children:

glance/tests/functional/test_bin_glance_control.py

@@ -240,3 +240,30 @@ def test_reload(self):
 
         # ensure the server has not been respawned
         self.connection_unavailable('deliberately stopped')
+
+    @skip_if_disabled
+    def test_infinite_child_death(self):
+        """Ensure child processes don't respawn forever if they can't start"""
+        self.cleanup()
+        self.api_server.default_store = 'shouldnotexist'
+        self.api_server.workers = 2
+
+        exitcode, out, err = self.api_server.start(**self.__dict__.copy())
+
+        # ensure the service pid has been cached
+        pid_cached = lambda: os.path.exists(self.api_server.pid_file)
+        self.wait_for(pid_cached)
+
+        # ensure glance-control has had a chance to waitpid on child
+        time.sleep(1)
+
+        # bouncing server should not be respawned
+        launch_msg = self.wait_for_servers([self.api_server], False)
+        self.assertTrue(launch_msg is None, launch_msg)
+
+        # ensure server process has gone away
+        process_died = lambda: not os.path.exists('/proc/%d' % self.get_pid())
+        self.wait_for(process_died)
+
+        # ensure the server has not been respawned
+        self.connection_unavailable('bouncing')