pcb/jail: drain pcb zones on vnet jail exit

There is a problem with pcbs sitting in UMA cache referencing credential,
which references prison.  With zero pcb zone traffic the vnet jails with
all their memory are basically leaked for indefinite time.  This is not
a problem for production, since eventually pcb zone traffic will purge
the caches, but it is a nightmare when running test suite(s) that create
a vnet jail for every test.

This patch has two drawbacks:
- UMA_RECLAIM_DRAIN_CPU will also do cache_shrink(), which is undesired
- For SMR zone (and all pcb zones are SMR) the per-cpu free caches are
  skipped, while theoretically we can smr_wait() on them and drain them
  too.  The implementation isn't ready yet.

Despite drawback freebsd#2 the patch allows to run the netinet or netpfil test
suite in an endless cycle without running out of memory.

sys/kern/kern_jail.c

@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/osd.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
+#include <sys/eventhandler.h>
 #include <sys/epoch.h>
 #include <sys/taskqueue.h>
 #include <sys/fcntl.h>
@@ -2544,6 +2545,8 @@ sys_jail_remove(struct thread *td, struct jail_remove_args *uap)
 		return (0);
 	}
 	prison_deref(pr, PD_KILL | PD_LOCKED | PD_LIST_XLOCKED);
+	if (pr->pr_flags & PR_VNET)
+		EVENTHANDLER_INVOKE(vnet_jail_destroy);
 	return (0);
 }
 

sys/netinet/in_pcb.c

@@ -564,6 +564,17 @@ in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 	mtx_destroy(&pcbinfo->ipi_lock);
 }
 
+#ifdef VIMAGE
+static void
+in_pcbstorage_drain(void *arg)
+{
+	struct inpcbstorage *pcbstor = arg;
+
+	uma_zone_reclaim(pcbstor->ips_zone, UMA_RECLAIM_DRAIN_CPU);
+	uma_zone_reclaim(pcbstor->ips_portzone, UMA_RECLAIM_DRAIN_CPU);
+}
+#endif
+
 /*
  * Initializa a pcbstorage - per protocol zones to allocate inpcbs.
  */
@@ -584,6 +595,10 @@ in_pcbstorage_init(void *arg)
 	    sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 	uma_zone_set_smr(pcbstor->ips_portzone,
 	    uma_zone_get_smr(pcbstor->ips_zone));
+#ifdef VIMAGE
+	pcbstor->ips_vnet_destroy = EVENTHANDLER_REGISTER(vnet_jail_destroy,
+	    in_pcbstorage_drain, pcbstor, EVENTHANDLER_PRI_ANY);
+#endif
 }
 
 /*
@@ -594,6 +609,9 @@ in_pcbstorage_destroy(void *arg)
 {
 	struct inpcbstorage *pcbstor = arg;
 
+#ifdef VIMAGE
+	EVENTHANDLER_DEREGISTER(vnet_jail_destroy, pcbstor->ips_vnet_destroy);
+#endif
 	uma_zdestroy(pcbstor->ips_zone);
 	uma_zdestroy(pcbstor->ips_portzone);
 }

sys/netinet/in_pcb.h

@@ -457,6 +457,9 @@ struct inpcbstorage {
 	const char *	ips_portzone_name;
 	const char *	ips_infolock_name;
 	const char *	ips_hashlock_name;
+#ifdef VIMAGE
+	eventhandler_tag ips_vnet_destroy;
+#endif
 };
 
 #define INPCBSTORAGE_DEFINE(prot, lname, zname, iname, hname)		\

sys/sys/eventhandler.h

@@ -317,4 +317,8 @@ struct ifaddr;
 typedef void (*rt_addrmsg_fn)(void *, struct ifaddr *, int);
 EVENTHANDLER_DECLARE(rt_addrmsg, rt_addrmsg_fn);
 
+/* Jail destroy. */
+typedef void (*jail_destroy_fn)(void *);
+EVENTHANDLER_DECLARE(vnet_jail_destroy, jail_destroy_fn);
+
 #endif /* _SYS_EVENTHANDLER_H_ */