Add jRCU by Joe Korty

arch/x86/include/asm/thread_info.h

@@ -29,6 +29,7 @@ struct thread_info {
 	__u32			flags;		/* low level flags */
 	__u32			status;		/* thread synchronous flags */
 	__u32			cpu;		/* current CPU */
+#define HAVE_THREAD_INFO_CPU   1
 	int			preempt_count;	/* 0 => preemptable,
 						   <0 => BUG */
 	mm_segment_t		addr_limit;

include/linux/hardirq.h

@@ -139,7 +139,13 @@ static inline void account_system_vtime(struct task_struct *tsk)
 extern void account_system_vtime(struct task_struct *tsk);
 #endif
 
-#if defined(CONFIG_NO_HZ)
+#if defined(CONFIG_JRCU)
+extern int rcu_nmi_seen;
+# define rcu_irq_enter() do { } while (0)
+# define rcu_irq_exit() do { } while (0)
+# define rcu_nmi_enter() do { rcu_nmi_seen = 1; } while (0)
+# define rcu_nmi_exit() do { } while (0)
+#elif defined(CONFIG_NO_HZ)
 #if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
 extern void rcu_enter_nohz(void);
 extern void rcu_exit_nohz(void);

include/linux/jrcu.h

@@ -0,0 +1,80 @@
+/*
+ * JRCU - A tiny single-cpu RCU for small SMP systems.
+ *
+ * Author: Joe Korty <joe.korty@ccur.com>
+ * Copyright Concurrent Computer Corporation, 2011
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __LINUX_JRCU_H
+#define __LINUX_JRCU_H
+
+#define __rcu_read_lock()                      preempt_disable()
+#define __rcu_read_unlock()                    jrcu_read_unlock()
+extern void jrcu_read_unlock(void);
+
+#define __rcu_read_lock_bh()                   __rcu_read_lock()
+#define __rcu_read_unlock_bh()                 __rcu_read_unlock()
+
+extern void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
+
+#define call_rcu_bh                            call_rcu_sched
+#define call_rcu                               call_rcu_sched
+
+extern void rcu_barrier(void);
+
+#define rcu_barrier_sched                      rcu_barrier
+#define rcu_barrier_bh                         rcu_barrier
+
+extern void synchronize_sched(void);
+
+#define synchronize_rcu                                synchronize_sched
+#define synchronize_rcu_bh                     synchronize_sched
+#define synchronize_rcu_expedited              synchronize_sched
+#define synchronize_rcu_bh_expedited           synchronize_sched
+#define synchronize_sched_expedited            synchronize_sched
+
+#define rcu_init(cpu)                          do { } while (0)
+#define rcu_init_sched()                       do { } while (0)
+#define exit_rcu()                             do { } while (0)
+
+static inline void __rcu_check_callbacks(int cpu, int user) { }
+#define rcu_check_callbacks                    __rcu_check_callbacks
+
+#define rcu_needs_cpu(cpu)                     (0)
+#define rcu_batches_completed()                        (0)
+#define rcu_batches_completed_bh()             (0)
+#define rcu_preempt_depth()                    (0)
+
+extern void rcu_force_quiescent_state(void);
+
+#define rcu_sched_force_quiescent_state                rcu_force_quiescent_state
+#define rcu_bh_force_quiescent_state           rcu_force_quiescent_state
+
+#define rcu_enter_nohz()                       do { } while (0)
+#define rcu_exit_nohz()                                do { } while (0)
+
+extern void rcu_note_context_switch(int cpu);
+
+#define rcu_sched_qs                           rcu_note_context_switch
+#define rcu_bh_qs                              rcu_note_context_switch
+#define rcu_virt_note_context_switch           rcu_note_context_switch
+
+extern void rcu_note_might_resched(void);
+
+extern void rcu_scheduler_starting(void);
+extern int rcu_scheduler_active __read_mostly;
+
+#endif /* __LINUX_JRCU_H */

include/linux/kernel.h

@@ -114,11 +114,18 @@ struct completion;
 struct pt_regs;
 struct user;
 
+/* cannot bring in linux/rcupdate.h at this point */
+#ifdef CONFIG_JRCU
+extern void rcu_note_might_resched(void);
+#else
+#define rcu_note_might_resched()
+#endif /*JRCU */
+
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { _cond_resched(); rcu_note_might_resched(); } while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() do { rcu_note_might_resched(); } while (0)
 #endif
 
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP

include/linux/preempt.h

@@ -10,18 +10,45 @@
 #include <linux/linkage.h>
 #include <linux/list.h>
 
+/* cannot include rcupdate.h here, so open-code this */
+
+#if defined(CONFIG_JRCU)
+# define __add_preempt_count(val) do { \
+       int newval = (preempt_count() += (val)); \
+       if (newval == (val)) \
+               smp_wmb(); \
+} while (0)
+#else
+# define __add_preempt_count(val) do { preempt_count() += (val); } while (0)
+#endif
+
+#if defined(CONFIG_JRCU_LAZY) || !defined(CONFIG_JRCU)
+# define __sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
+#else
+# define __sub_preempt_count(val) do { \
+       int newval = (preempt_count() -= (val)); \
+       if (newval == 0) { \
+               /* race with preemption OK, preempt will do the mb for us */ \
+               smp_wmb(); \
+       } \
+} while (0)
+#endif
+
 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
   extern void add_preempt_count(int val);
   extern void sub_preempt_count(int val);
 #else
-# define add_preempt_count(val)	do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val)	do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val) __add_preempt_count(val)
+# define sub_preempt_count(val) __sub_preempt_count(val)
 #endif
 
 #define inc_preempt_count() add_preempt_count(1)
 #define dec_preempt_count() sub_preempt_count(1)
 
 #define preempt_count()	(current_thread_info()->preempt_count)
+#ifdef CONFIG_PREEMPT_COUNT_CPU
+extern int preempt_count_cpu(int cpu);
+#endif
 
 #ifdef CONFIG_PREEMPT
 

include/linux/rcupdate.h

@@ -147,6 +147,8 @@ static inline void rcu_exit_nohz(void)
 #include <linux/rcutree.h>
 #elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
 #include <linux/rcutiny.h>
+#elif defined(CONFIG_JRCU)
+#include <linux/jrcu.h>
 #else
 #error "Unknown RCU implementation specified to kernel configuration"
 #endif

include/linux/sched.h

@@ -1963,6 +1963,8 @@ extern void wake_up_idle_cpu(int cpu);
 static inline void wake_up_idle_cpu(int cpu) { }
 #endif
 
+extern void force_cpu_resched(int cpu);
+
 extern unsigned int sysctl_sched_latency;
 extern unsigned int sysctl_sched_min_granularity;
 extern unsigned int sysctl_sched_wakeup_granularity;

init/Kconfig

@@ -399,6 +399,23 @@ config TREE_PREEMPT_RCU
 	  is also required.  It also scales down nicely to
 	  smaller systems.
 
+config JRCU
+       bool "A tiny single-CPU RCU for small SMP systems"
+       depends on PREEMPT
+       depends on SMP
+       select PREEMPT_COUNT_CPU
+       help
+         This option selects a minimal-footprint RCU suitable for small SMP
+         systems -- that is, those with fewer than 16 or perhaps 32, and
+         certainly less than 64 processors.
+
+         This RCU variant may be a good choice for systems with low latency
+         requirements.  It does RCU garbage collection from a single CPU
+         rather than have each CPU do its own.  This frees up all but one
+         CPU from interference by this periodic requirement.
+
+         Most users should say N here.
+
 config TINY_RCU
 	bool "UP-only small-memory-footprint RCU"
 	depends on !SMP
@@ -424,6 +441,52 @@ config PREEMPT_RCU
 	  This option enables preemptible-RCU code that is common between
 	  the TREE_PREEMPT_RCU and TINY_PREEMPT_RCU implementations.
 
+config JRCU_DAEMON
+       bool
+       depends on JRCU
+       default y
+       help
+         Required. The context switch when leaving the daemon is needed
+         to get the CPU to reliably participate in end-of-batch processing.
+
+config JRCU_DAEMON_PRIO
+       int "JRCU Daemon priority"
+       depends on JRCU_DAEMON
+       default 0
+       help
+         The JRCU daemon priority. If 0 then the daemon runs SCHED_OTHER.
+         If >0 then the daemon runs SCHED_RR and its priority will be
+         the value selected.  If <0 then SCHED_RR is again selected, but
+         now its priority will be the biased downwards from the maximum
+         possible Posix priority.
+
+config JRCU_LAZY
+       bool "Should JRCU be lazy recognizing end-of-batch"
+       depends on JRCU
+       default n
+       help
+         If you say Y here, JRCU will on occasion fail to recognize
+         end-of-batch for an rcu period or two.
+
+         If you say N here, JRCU will be more aggressive; in fact it
+         will always recognize end-of-batch at the earliest possible time.
+
+         Being lazy should be fractionally more efficient in that JRCU
+         inserts fewer memory barriers along some high performance kernel
+         code paths.
+
+         If unsure, say N.
+
+config PREEMPT_COUNT_CPU
+       # bool "Let one CPU look at another CPUs preemption count"
+       bool
+       default n
+       help
+         If Y then the preempt_count_cpu() function will be compiled into
+         the kernel.  Its existance impacts kernel performance slightly,
+         so this option should be selected only if other kernel features
+         that use preempt_count_cpu() are also selected.
+
 config RCU_TRACE
 	bool "Enable tracing for RCU"
 	help

kernel/Makefile

@@ -83,6 +83,7 @@ obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
 obj-$(CONFIG_TREE_RCU) += rcutree.o
 obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
 obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
+obj-$(CONFIG_JRCU) += jrcu.o
 obj-$(CONFIG_TINY_RCU) += rcutiny.o
 obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
 obj-$(CONFIG_RELAY) += relay.o