tkt-38155: fix(middlewared/jail): handle start/stop gracefully using iocage

src/freenas/etc/rc.shutdown.local

@@ -1,3 +1,4 @@
+/usr/local/bin/midclt call core.event_send system ADDED '{"id": "shutdown"}' > /dev/null
 /bin/sh /usr/local/sbin/save_rrds.sh
 if [ -f /etc/killpower ]; then
 	/usr/local/sbin/upsdrvctl shutdown

src/middlewared/middlewared/plugins/jail.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 import re
 import time
@@ -21,6 +22,8 @@
 from middlewared.utils import filter_list
 from middlewared.client import ClientException
 
+SHUTDOWN_LOCK = asyncio.Lock()
+
 
 class JailService(CRUDService):
 
@@ -613,19 +616,32 @@ def get_plugin_version(self, pkg):
 
         return version
 
+    @private
+    def start_on_boot(self):
+        ioc.IOCage(rc=True).start()
+
+    @private
+    def stop_on_shutdown(self):
+        ioc.IOCage(rc=True).stop()
+
+    @private
+    async def terminate(self):
+        await SHUTDOWN_LOCK.acquire()
+
 
-async def __event_system_ready(middleware, event_type, args):
+async def __event_system(middleware, event_type, args):
     """
-    Method called when system is ready, supposed to start jails
+    Method called when system is ready or shutdown, supposed to start/stop jails
     flagged that way.
     """
-    if args['id'] != 'ready':
-        return
-
-    jails = await middleware.call('jail.query', [('boot', '=', 'on')])
-    for jail in sorted(jails, key=lambda j: int(j['priority'])):
-        await middleware.call('jail.start', jail['id'])
+    # We need to call a method in Jail service to make sure it runs in the
+    # process pool because of py-libzfs thread safety issue with iocage and middlewared
+    if args['id'] == 'ready':
+        await middleware.call('jail.start_on_boot')
+    elif args['id'] == 'shutdown':
+        async with SHUTDOWN_LOCK:
+            await middleware.call('jail.stop_on_shutdown')
 
 
 def setup(middleware):
-    middleware.event_subscribe('system', __event_system_ready)
+    middleware.event_subscribe('system', __event_system)