subr_ipi.c

/*	$NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $	*/

/*-
 * Copyright (c) 2014 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Mindaugas Rasiukevicius.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to
 * invoke functions with a constant argument and synchronous IPIs
 * with the cross-call support.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.10 2022/04/09 23:51:22 riastradh Exp $");

#include <sys/param.h>
#include <sys/types.h>

#include <sys/atomic.h>
#include <sys/evcnt.h>
#include <sys/cpu.h>
#include <sys/ipi.h>
#include <sys/intr.h>
#include <sys/kcpuset.h>
#include <sys/kmem.h>
#include <sys/lock.h>
#include <sys/mutex.h>

/*
 * An array of the IPI handlers used for asynchronous invocation.
 * The lock protects the slot allocation.
 */

typedef struct {
	ipi_func_t	func;
	void *		arg;
} ipi_intr_t;

static kmutex_t		ipi_mngmt_lock;
static ipi_intr_t	ipi_intrs[IPI_MAXREG]	__cacheline_aligned;

/*
 * Per-CPU mailbox for IPI messages: it is a single cache line storing
 * up to IPI_MSG_MAX messages.  This interface is built on top of the
 * synchronous IPIs.
 */

#define	IPI_MSG_SLOTS	(CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
#define	IPI_MSG_MAX	IPI_MSG_SLOTS

typedef struct {
	ipi_msg_t *	msg[IPI_MSG_SLOTS];
} ipi_mbox_t;


/* Mailboxes for the synchronous IPIs. */
static ipi_mbox_t *	ipi_mboxes	__read_mostly;
static struct evcnt	ipi_mboxfull_ev	__cacheline_aligned;
static void		ipi_msg_cpu_handler(void *);

/* Handler for the synchronous IPIs - it must be zero. */
#define	IPI_SYNCH_ID	0

#ifndef MULTIPROCESSOR
#define	cpu_ipi(ci)	KASSERT(ci == NULL)
#endif

void
ipi_sysinit(void)
{

	mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
	memset(ipi_intrs, 0, sizeof(ipi_intrs));

	/*
	 * Register the handler for synchronous IPIs.  This mechanism
	 * is built on top of the asynchronous interface.  Slot zero is
	 * reserved permanently; it is also handy to use zero as a failure
	 * for other registers (as it is potentially less error-prone).
	 */
	ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;

	evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
	   "ipi", "full");
}

void
ipi_percpu_init(void)
{
	const size_t len = ncpu * sizeof(ipi_mbox_t);

	/* Initialise the per-CPU bit fields. */
	for (u_int i = 0; i < ncpu; i++) {
		struct cpu_info *ci = cpu_lookup(i);
		memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
	}

	/* Allocate per-CPU IPI mailboxes. */
	ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
	KASSERT(ipi_mboxes != NULL);
}

/*
 * ipi_register: register an asynchronous IPI handler.
 *
 * => Returns IPI ID which is greater than zero; on failure - zero.
 */
u_int
ipi_register(ipi_func_t func, void *arg)
{
	mutex_enter(&ipi_mngmt_lock);
	for (u_int i = 0; i < IPI_MAXREG; i++) {
		if (ipi_intrs[i].func == NULL) {
			/* Register the function. */
			ipi_intrs[i].func = func;
			ipi_intrs[i].arg = arg;
			mutex_exit(&ipi_mngmt_lock);

			KASSERT(i != IPI_SYNCH_ID);
			return i;
		}
	}
	mutex_exit(&ipi_mngmt_lock);
	printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
	return 0;
}

/*
 * ipi_unregister: release the IPI handler given the ID.
 */
void
ipi_unregister(u_int ipi_id)
{
	ipi_msg_t ipimsg = { .func = __FPTRCAST(ipi_func_t, nullop) };

	KASSERT(ipi_id != IPI_SYNCH_ID);
	KASSERT(ipi_id < IPI_MAXREG);

	/* Release the slot. */
	mutex_enter(&ipi_mngmt_lock);
	KASSERT(ipi_intrs[ipi_id].func != NULL);
	ipi_intrs[ipi_id].func = NULL;

	/* Ensure that there are no IPIs in flight. */
	kpreempt_disable();
	ipi_broadcast(&ipimsg, false);
	ipi_wait(&ipimsg);
	kpreempt_enable();
	mutex_exit(&ipi_mngmt_lock);
}

/*
 * ipi_mark_pending: internal routine to mark an IPI pending on the
 * specified CPU (which might be curcpu()).
 */
static bool
ipi_mark_pending(u_int ipi_id, struct cpu_info *ci)
{
	const u_int i = ipi_id >> IPI_BITW_SHIFT;
	const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);

	KASSERT(ipi_id < IPI_MAXREG);
	KASSERT(kpreempt_disabled());

	/* Mark as pending and return true if not previously marked. */
	if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
#ifndef __HAVE_ATOMIC_AS_MEMBAR
		membar_release();
#endif
		atomic_or_32(&ci->ci_ipipend[i], bitm);
		return true;
	}
	return false;
}

/*
 * ipi_trigger: asynchronously send an IPI to the specified CPU.
 */
void
ipi_trigger(u_int ipi_id, struct cpu_info *ci)
{

	KASSERT(curcpu() != ci);
	if (ipi_mark_pending(ipi_id, ci)) {
		cpu_ipi(ci);
	}
}

/*
 * ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and
 * ipi_trigger_broadcast().
 */
static void
ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target,
    bool skip_self)
{
	const cpuid_t selfid = cpu_index(curcpu());
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;

	KASSERT(kpreempt_disabled());
	KASSERT(target != NULL);

	for (CPU_INFO_FOREACH(cii, ci)) {
		const cpuid_t cpuid = cpu_index(ci);

		if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
			continue;
		}
		ipi_trigger(ipi_id, ci);
	}
	if (!skip_self && kcpuset_isset(target, selfid)) {
		ipi_mark_pending(ipi_id, curcpu());
		int s = splhigh();
		ipi_cpu_handler();
		splx(s);
	}
}

/*
 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
 * CPUs given the target CPU set.
 */
void
ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
{
	ipi_trigger_multi_internal(ipi_id, target, false);
}

/*
 * ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached,
 * optionally skipping the sending CPU.
 */
void
ipi_trigger_broadcast(u_int ipi_id, bool skip_self)
{
	ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self);
}

/*
 * put_msg: insert message into the mailbox.
 *
 * Caller is responsible for issuing membar_release first.
 */
static inline void
put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
{
	int count = SPINLOCK_BACKOFF_MIN;
again:
	for (u_int i = 0; i < IPI_MSG_MAX; i++) {
		if (atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
			return;
		}
	}

	/* All slots are full: we have to spin-wait. */
	ipi_mboxfull_ev.ev_count++;
	SPINLOCK_BACKOFF(count);
	goto again;
}

/*
 * ipi_cpu_handler: the IPI handler.
 */
void
ipi_cpu_handler(void)
{
	struct cpu_info * const ci = curcpu();

	/*
	 * Handle asynchronous IPIs: inspect per-CPU bit field, extract
	 * IPI ID numbers and execute functions in those slots.
	 */
	for (u_int i = 0; i < IPI_BITWORDS; i++) {
		uint32_t pending, bit;

		if (atomic_load_relaxed(&ci->ci_ipipend[i]) == 0) {
			continue;
		}
		pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
#ifndef __HAVE_ATOMIC_AS_MEMBAR
		membar_acquire();
#endif
		while ((bit = ffs(pending)) != 0) {
			const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
			ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];

			pending &= ~(1U << bit);
			KASSERT(ipi_hdl->func != NULL);
			ipi_hdl->func(ipi_hdl->arg);
		}
	}
}

/*
 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
 * execute the passed functions and acknowledge the messages.
 */
static void
ipi_msg_cpu_handler(void *arg __unused)
{
	const struct cpu_info * const ci = curcpu();
	ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];

	for (u_int i = 0; i < IPI_MSG_MAX; i++) {
		ipi_msg_t *msg;

		/* Get the message. */
		if ((msg = atomic_load_acquire(&mbox->msg[i])) == NULL) {
			continue;
		}
		atomic_store_relaxed(&mbox->msg[i], NULL);

		/* Execute the handler. */
		KASSERT(msg->func);
		msg->func(msg->arg);

		/* Ack the request. */
#ifndef __HAVE_ATOMIC_AS_MEMBAR
		membar_release();
#endif
		atomic_dec_uint(&msg->_pending);
	}
}

/*
 * ipi_unicast: send an IPI to a single CPU.
 *
 * => The CPU must be remote; must not be local.
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
{
	const cpuid_t id = cpu_index(ci);

	KASSERT(msg->func != NULL);
	KASSERT(kpreempt_disabled());
	KASSERT(curcpu() != ci);

	msg->_pending = 1;
#ifndef __HAVE_ATOMIC_AS_MEMBAR
	membar_release();
#endif

	put_msg(&ipi_mboxes[id], msg);
	ipi_trigger(IPI_SYNCH_ID, ci);
}

/*
 * ipi_multicast: send an IPI to each CPU in the specified set.
 *
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
{
	const struct cpu_info * const self = curcpu();
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;
	u_int local;

	KASSERT(msg->func != NULL);
	KASSERT(kpreempt_disabled());

	local = !!kcpuset_isset(target, cpu_index(self));
	msg->_pending = kcpuset_countset(target) - local;
#ifndef __HAVE_ATOMIC_AS_MEMBAR
	membar_release();
#endif

	for (CPU_INFO_FOREACH(cii, ci)) {
		cpuid_t id;

		if (__predict_false(ci == self)) {
			continue;
		}
		id = cpu_index(ci);
		if (!kcpuset_isset(target, id)) {
			continue;
		}
		put_msg(&ipi_mboxes[id], msg);
		ipi_trigger(IPI_SYNCH_ID, ci);
	}
	if (local) {
		msg->func(msg->arg);
	}
}

/*
 * ipi_broadcast: send an IPI to all CPUs.
 *
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_broadcast(ipi_msg_t *msg, bool skip_self)
{
	const struct cpu_info * const self = curcpu();
	CPU_INFO_ITERATOR cii;
	struct cpu_info *ci;

	KASSERT(msg->func != NULL);
	KASSERT(kpreempt_disabled());

	msg->_pending = ncpu - 1;
#ifndef __HAVE_ATOMIC_AS_MEMBAR
	membar_release();
#endif

	/* Broadcast IPIs for remote CPUs. */
	for (CPU_INFO_FOREACH(cii, ci)) {
		cpuid_t id;

		if (__predict_false(ci == self)) {
			continue;
		}
		id = cpu_index(ci);
		put_msg(&ipi_mboxes[id], msg);
		ipi_trigger(IPI_SYNCH_ID, ci);
	}

	if (!skip_self) {
		/* Finally, execute locally. */
		msg->func(msg->arg);
	}
}

/*
 * ipi_wait: spin-wait until the message is processed.
 */
void
ipi_wait(ipi_msg_t *msg)
{
	int count = SPINLOCK_BACKOFF_MIN;

	while (atomic_load_acquire(&msg->_pending)) {
		KASSERT(atomic_load_relaxed(&msg->_pending) < ncpu);
		SPINLOCK_BACKOFF(count);
	}
}