Add --ignore-running-workers option to pulp-manage-db

closes #2469
https://pulp.plan.io/issues/2469

docs/user-guide/troubleshooting.rst

@@ -286,3 +286,11 @@ Workers not releasing memory
 
 See the :ref:`process recycling documentation<process_recycling>` for more information on how to
 have your Pulp workers return memory back to the system.
+
+pulp-manage-db prompts for running workers when running automated upgrade
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While running an automated upgrade to Pulp 2.11.x Pulp will prompt you that you have running
+workers even though all workers are stopped. When this happens please call `pulp-manage-db` with
+the `--ignore-running-workers` flag. This flag should only be used in automation, and the
+automation is responsible for ensuring all pulp processes are stopped before the upgrade proceeds.

server/pulp/server/db/manage.py

@@ -54,6 +54,11 @@ def parse_args():
     parser.add_option('--dry-run', action='store_true', dest='dry_run', default=False,
                       help=_('Perform a dry run with no changes made. Returns 1 if there are '
                              'migrations to apply.'))
+
+    parser.add_option('--ignore-running-workers', action='store_true',
+                      dest='ignore_running_workers', default=False,
+                      help=_('Runs migrations without checking for running workers. '
+                             'Please ensure there are no running works before using this flag.'))
     options, args = parser.parse_args()
     if args:
         parser.error(_('Unknown arguments: %s') % ', '.join(args))
@@ -196,14 +201,15 @@ def main():
         _start_logging()
         connection.initialize(max_timeout=1)
 
-        # Prompt the user if there are workers that have not timed out
-        if filter(lambda worker: (UTCDateTimeField().to_python(datetime.now()) -
-                                  worker['last_heartbeat']) <
-                  timedelta(seconds=constants.CELERY_TIMEOUT_SECONDS), status.get_workers()):
-            if not _user_input_continue('There are still running workers, continuing could '
-                                        'corrupt your Pulp installation. Are you sure you wish '
-                                        'to continue?'):
-                return os.EX_OK
+        if not options.ignore_running_workers:
+            # Prompt the user if there are workers that have not timed out
+            if filter(lambda worker: (UTCDateTimeField().to_python(datetime.now()) -
+                                      worker['last_heartbeat']) <
+                      timedelta(seconds=constants.CELERY_TIMEOUT_SECONDS), status.get_workers()):
+                if not _user_input_continue('There are still running workers, continuing could '
+                                            'corrupt your Pulp installation. Are you sure you wish '
+                                            'to continue?'):
+                    return os.EX_OK
         return _auto_manage_db(options)
     except UnperformedMigrationException:
         return 1

server/test/unit/server/db/test_manage.py

@@ -592,7 +592,7 @@ def test_dry_run_no_changes(self, mock_file_config, mock_parse_args, initialize,
                                 mocked_apply_migration, mock_entry, getLogger, mock_ensure_indexes):
         logger = MagicMock()
         getLogger.return_value = logger
-        mock_args = Namespace(dry_run=True, test=False)
+        mock_args = Namespace(dry_run=True, test=False, ignore_running_workers=True)
         mock_parse_args.return_value = mock_args
 
         # Test that when dry run is on, it returns 1 if migrations remain