Permalink
Browse files

RCU: Joe's RCU

Forward port jRCU to linux-3.0.

Thanks goes to Francisco Franco for his help
in testing out this version on the Android
Galaxy Nexus phone.

Signed-off-by: Joe Korty <joe.korty@ccur.com>
Signed-off-by: Ezekeel <notezekeel@googlemail.com>
  • Loading branch information...
1 parent 61c8b57 commit 315f7d532dffbf94c9170e4b5616afd02f91ee38 Joe Korty committed with Jan 9, 2012
@@ -29,6 +29,7 @@ struct thread_info {
__u32 flags; /* low level flags */
__u32 status; /* thread synchronous flags */
__u32 cpu; /* current CPU */
+#define HAVE_THREAD_INFO_CPU 1
int preempt_count; /* 0 => preemptable,
<0 => BUG */
mm_segment_t addr_limit;
@@ -139,7 +139,13 @@ static inline void account_system_vtime(struct task_struct *tsk)
extern void account_system_vtime(struct task_struct *tsk);
#endif
-#if defined(CONFIG_NO_HZ)
+#if defined(CONFIG_JRCU)
+extern int rcu_nmi_seen;
+# define rcu_irq_enter() do { } while (0)
+# define rcu_irq_exit() do { } while (0)
+# define rcu_nmi_enter() do { rcu_nmi_seen = 1; } while (0)
+# define rcu_nmi_exit() do { } while (0)
+#elif defined(CONFIG_NO_HZ)
#if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
extern void rcu_enter_nohz(void);
extern void rcu_exit_nohz(void);
View
@@ -0,0 +1,80 @@
+/*
+ * JRCU - An RCU suitable for small SMP systems.
+ *
+ * Author: Joe Korty <joe.korty@ccur.com>
+ * Copyright Concurrent Computer Corporation, 2011
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __LINUX_JRCU_H
+#define __LINUX_JRCU_H
+
+#define __rcu_read_lock() preempt_disable()
+#define __rcu_read_unlock() jrcu_read_unlock()
+extern void jrcu_read_unlock(void);
+
+#define __rcu_read_lock_bh() __rcu_read_lock()
+#define __rcu_read_unlock_bh() __rcu_read_unlock()
+
+extern void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
+
+#define call_rcu_bh call_rcu_sched
+#define call_rcu call_rcu_sched
+
+extern void rcu_barrier(void);
+
+#define rcu_barrier_sched rcu_barrier
+#define rcu_barrier_bh rcu_barrier
+
+extern void synchronize_sched(void);
+
+#define synchronize_rcu synchronize_sched
+#define synchronize_rcu_bh synchronize_sched
+#define synchronize_rcu_expedited synchronize_sched
+#define synchronize_rcu_bh_expedited synchronize_sched
+#define synchronize_sched_expedited synchronize_sched
+
+#define rcu_init(cpu) do { } while (0)
+#define rcu_init_sched() do { } while (0)
+#define exit_rcu() do { } while (0)
+
+static inline void __rcu_check_callbacks(int cpu, int user) { }
+#define rcu_check_callbacks __rcu_check_callbacks
+
+#define rcu_needs_cpu(cpu) (0)
+#define rcu_batches_completed() (0)
+#define rcu_batches_completed_bh() (0)
+#define rcu_preempt_depth() (0)
+
+extern void rcu_force_quiescent_state(void);
+
+#define rcu_sched_force_quiescent_state rcu_force_quiescent_state
+#define rcu_bh_force_quiescent_state rcu_force_quiescent_state
+
+#define rcu_enter_nohz() do { } while (0)
+#define rcu_exit_nohz() do { } while (0)
+
+extern void rcu_note_context_switch(int cpu);
+
+#define rcu_sched_qs rcu_note_context_switch
+#define rcu_bh_qs rcu_note_context_switch
+#define rcu_virt_note_context_switch rcu_note_context_switch
+
+extern void rcu_note_might_resched(void);
+
+extern void rcu_scheduler_starting(void);
+extern int rcu_scheduler_active __read_mostly;
+
+#endif /* __LINUX_JRCU_H */
@@ -127,11 +127,18 @@ struct completion;
struct pt_regs;
struct user;
+/* cannot bring in linux/rcupdate.h at this point */
+#ifdef CONFIG_JRCU
+extern void rcu_note_might_resched(void);
+#else
+#define rcu_note_might_resched()
+#endif /*JRCU */
+
#ifdef CONFIG_PREEMPT_VOLUNTARY
extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { _cond_resched(); rcu_note_might_resched(); } while (0)
#else
-# define might_resched() do { } while (0)
+# define might_resched() do { rcu_note_might_resched(); } while (0)
#endif
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
@@ -10,18 +10,45 @@
#include <linux/linkage.h>
#include <linux/list.h>
+/* cannot include rcupdate.h here, so open-code this */
+
+#if defined(CONFIG_JRCU)
+# define __add_preempt_count(val) do { \
+ int newval = (preempt_count() += (val)); \
+ if (newval == (val)) \
+ smp_wmb(); \
+} while (0)
+#else
+# define __add_preempt_count(val) do { preempt_count() += (val); } while (0)
+#endif
+
+#if defined(CONFIG_JRCU_LAZY) || !defined(CONFIG_JRCU)
+# define __sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
+#else
+# define __sub_preempt_count(val) do { \
+ int newval = (preempt_count() -= (val)); \
+ if (newval == 0) { \
+ /* race with preemption OK, preempt will do the mb for us */ \
+ smp_wmb(); \
+ } \
+} while (0)
+#endif
+
#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
extern void add_preempt_count(int val);
extern void sub_preempt_count(int val);
#else
-# define add_preempt_count(val) do { preempt_count() += (val); } while (0)
-# define sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
+# define add_preempt_count(val) __add_preempt_count(val)
+# define sub_preempt_count(val) __sub_preempt_count(val)
#endif
#define inc_preempt_count() add_preempt_count(1)
#define dec_preempt_count() sub_preempt_count(1)
#define preempt_count() (current_thread_info()->preempt_count)
+#ifdef CONFIG_PREEMPT_COUNT_CPU
+extern int preempt_count_cpu(int cpu);
+#endif
#ifdef CONFIG_PREEMPT
@@ -156,6 +156,8 @@ static inline void rcu_exit_nohz(void)
#include <linux/rcutree.h>
#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
#include <linux/rcutiny.h>
+#elif defined(CONFIG_JRCU)
+#include <linux/jrcu.h>
#else
#error "Unknown RCU implementation specified to kernel configuration"
#endif
@@ -1963,6 +1963,8 @@ extern void wake_up_idle_cpu(int cpu);
static inline void wake_up_idle_cpu(int cpu) { }
#endif
+extern void force_cpu_resched(int cpu);
+
extern unsigned int sysctl_sched_latency;
extern unsigned int sysctl_sched_min_granularity;
extern unsigned int sysctl_sched_wakeup_granularity;
View
@@ -399,6 +399,24 @@ config TREE_PREEMPT_RCU
is also required. It also scales down nicely to
smaller systems.
+config JRCU
+ bool "An RCU suitable for small SMP systems"
+ depends on PREEMPT
+ depends on SMP
+ select PREEMPT_COUNT_CPU
+ help
+ This option selects a minimal-footprint RCU that is most suitable
+ for small SMP systems -- 'small' in this case meaning all but
+ the largest NUMA systems. It is not clear how big 'small' can
+ be, but it is known to work well on NUMA platforms having 80 CPUs.
+
+ jRCU may also be a good choice for systems with low latency
+ requirements. It does RCU garbage collection from a single
+ CPU rather than have each CPU do its own. This frees up all
+ but one CPU from interference by this periodic requirement.
+
+ Most users should say N here.
+
config TINY_RCU
bool "UP-only small-memory-footprint RCU"
depends on !SMP
@@ -424,6 +442,57 @@ config PREEMPT_RCU
This option enables preemptible-RCU code that is common between
the TREE_PREEMPT_RCU and TINY_PREEMPT_RCU implementations.
+config JRCU_DAEMON
+ bool
+ depends on JRCU
+ default y
+ help
+ Required. The context switch when leaving the daemon is needed
+ to get the CPU to reliably participate in end-of-batch processing.
+
+config JRCU_DAEMON_PRIO
+ int "JRCU Daemon priority"
+ depends on JRCU_DAEMON
+ default 0
+ help
+ The JRCU daemon priority. If 0 then the daemon runs SCHED_OTHER.
+ If >0 then the daemon runs SCHED_RR and its priority will be
+ the value selected. If <0 then SCHED_RR is again selected,
+ but now its priority will be the biased downwards from the
+ maximum possible Posix priority.
+
+ If unsure, select 0. The other values are useful only for those
+ rare setups where 100% of every CPU's utilization will be spent in
+ user SCHED_RR or SCHED_FIFO applications, for long periods of time.
+
+
+config JRCU_LAZY
+ bool "Should JRCU be lazy recognizing end-of-batch"
+ depends on JRCU
+ default n
+ help
+ If you say Y here, JRCU will on occasion fail to recognize
+ end-of-batch for an rcu period or two.
+
+ If you say N here, JRCU will be more aggressive; in fact it
+ will always recognize end-of-batch at the earliest possible time.
+
+ Being lazy should be fractionally more efficient in that JRCU
+ inserts fewer memory barriers along some high performance kernel
+ code paths.
+
+ If unsure, say N.
+
+config PREEMPT_COUNT_CPU
+ # bool "Let one CPU look at another CPUs preemption count"
+ bool
+ default n
+ help
+ If Y then the preempt_count_cpu() function will be compiled into
+ the kernel. Its existance impacts kernel performance slightly,
+ so this option should be selected only if other kernel features
+ that use preempt_count_cpu() are also selected.
+
config RCU_TRACE
bool "Enable tracing for RCU"
help
View
@@ -83,6 +83,7 @@ obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
obj-$(CONFIG_TREE_RCU) += rcutree.o
obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
+obj-$(CONFIG_JRCU) += jrcu.o
obj-$(CONFIG_TINY_RCU) += rcutiny.o
obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
obj-$(CONFIG_RELAY) += relay.o
Oops, something went wrong. Retry.

0 comments on commit 315f7d5

Please sign in to comment.