Skip to content
Newer
Older
100644 328 lines (297 sloc) 12.7 KB
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
1 /*
2 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
3 * Internal non-public definitions.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright IBM Corporation, 2008
20 *
21 * Author: Ingo Molnar <mingo@elte.hu>
22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
23 */
24
25 #include <linux/cache.h>
26 #include <linux/spinlock.h>
27 #include <linux/threads.h>
28 #include <linux/cpumask.h>
29 #include <linux/seqlock.h>
30
31 /*
32 * Define shape of hierarchy based on NR_CPUS and CONFIG_RCU_FANOUT.
33 * In theory, it should be possible to add more levels straightforwardly.
34 * In practice, this has not been tested, so there is probably some
35 * bug somewhere.
36 */
37 #define MAX_RCU_LVLS 3
38 #define RCU_FANOUT (CONFIG_RCU_FANOUT)
39 #define RCU_FANOUT_SQ (RCU_FANOUT * RCU_FANOUT)
40 #define RCU_FANOUT_CUBE (RCU_FANOUT_SQ * RCU_FANOUT)
41
42 #if NR_CPUS <= RCU_FANOUT
43 # define NUM_RCU_LVLS 1
44 # define NUM_RCU_LVL_0 1
45 # define NUM_RCU_LVL_1 (NR_CPUS)
46 # define NUM_RCU_LVL_2 0
47 # define NUM_RCU_LVL_3 0
48 #elif NR_CPUS <= RCU_FANOUT_SQ
49 # define NUM_RCU_LVLS 2
50 # define NUM_RCU_LVL_0 1
fc2219d @paulmck rcu: Clean up code based on review feedback from Josh Triplett
paulmck authored
51 # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT)
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
52 # define NUM_RCU_LVL_2 (NR_CPUS)
53 # define NUM_RCU_LVL_3 0
54 #elif NR_CPUS <= RCU_FANOUT_CUBE
55 # define NUM_RCU_LVLS 3
56 # define NUM_RCU_LVL_0 1
fc2219d @paulmck rcu: Clean up code based on review feedback from Josh Triplett
paulmck authored
57 # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_SQ)
58 # define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT)
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
59 # define NUM_RCU_LVL_3 NR_CPUS
60 #else
61 # error "CONFIG_RCU_FANOUT insufficient for NR_CPUS"
62 #endif /* #if (NR_CPUS) <= RCU_FANOUT */
63
64 #define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3)
65 #define NUM_RCU_NODES (RCU_SUM - NR_CPUS)
66
67 /*
68 * Dynticks per-CPU state.
69 */
70 struct rcu_dynticks {
71 int dynticks_nesting; /* Track nesting level, sort of. */
72 int dynticks; /* Even value for dynticks-idle, else odd. */
73 int dynticks_nmi; /* Even value for either dynticks-idle or */
74 /* not in nmi handler, else odd. So this */
75 /* remains even for nmi from irq handler. */
76 };
77
78 /*
79 * Definition for node within the RCU grace-period-detection hierarchy.
80 */
81 struct rcu_node {
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
82 spinlock_t lock; /* Root rcu_node's lock protects some */
83 /* rcu_state fields as well as following. */
f41d911 @paulmck rcu: Merge preemptable-RCU functionality into hierarchical RCU
paulmck authored
84 long gpnum; /* Current grace period for this node. */
8684896 @paulmck rcu: Changes from reviews: avoid casts, fix/add warnings, improve com…
paulmck authored
85 /* This will either be equal to or one */
86 /* behind the root rcu_node's gpnum. */
d09b62d @paulmck rcu: Fix synchronization for rcu_process_gp_end() uses of ->completed…
paulmck authored
87 long completed; /* Last grace period completed for this node. */
88 /* This will either be equal to or one */
89 /* behind the root rcu_node's gpnum. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
90 unsigned long qsmask; /* CPUs or groups that need to switch in */
91 /* order for current grace period to proceed.*/
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
92 /* In leaf rcu_node, each bit corresponds to */
93 /* an rcu_data structure, otherwise, each */
94 /* bit corresponds to a child rcu_node */
95 /* structure. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
96 unsigned long qsmaskinit;
97 /* Per-GP initialization for qsmask. */
98 unsigned long grpmask; /* Mask to apply to parent qsmask. */
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
99 /* Only one bit will be set in this mask. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
100 int grplo; /* lowest-numbered CPU or group here. */
101 int grphi; /* highest-numbered CPU or group here. */
102 u8 grpnum; /* CPU/group number for next level up. */
103 u8 level; /* root is at level 0. */
104 struct rcu_node *parent;
f41d911 @paulmck rcu: Merge preemptable-RCU functionality into hierarchical RCU
paulmck authored
105 struct list_head blocked_tasks[2];
106 /* Tasks blocked in RCU read-side critsect. */
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
107 /* Grace period number (->gpnum) x blocked */
108 /* by tasks on the (x & 0x1) element of the */
109 /* blocked_tasks[] array. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
110 } ____cacheline_internodealigned_in_smp;
111
a0b6c9a @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 4
paulmck authored
112 /*
113 * Do a full breadth-first scan of the rcu_node structures for the
114 * specified rcu_state structure.
115 */
116 #define rcu_for_each_node_breadth_first(rsp, rnp) \
117 for ((rnp) = &(rsp)->node[0]; \
118 (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
119
120 #define rcu_for_each_leaf_node(rsp, rnp) \
121 for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \
122 (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++)
123
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
124 /* Index values for nxttail array in struct rcu_data. */
125 #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */
126 #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */
127 #define RCU_NEXT_READY_TAIL 2 /* Also RCU_NEXT head. */
128 #define RCU_NEXT_TAIL 3
129 #define RCU_NEXT_SIZE 4
130
131 /* Per-CPU data for read-copy update. */
132 struct rcu_data {
133 /* 1) quiescent-state and grace-period handling : */
134 long completed; /* Track rsp->completed gp number */
135 /* in order to detect GP end. */
136 long gpnum; /* Highest gp number that this CPU */
137 /* is aware of having started. */
138 long passed_quiesc_completed;
139 /* Value of completed at time of qs. */
140 bool passed_quiesc; /* User-mode/idle loop etc. */
141 bool qs_pending; /* Core waits for quiesc state. */
142 bool beenonline; /* CPU online at least once. */
f41d911 @paulmck rcu: Merge preemptable-RCU functionality into hierarchical RCU
paulmck authored
143 bool preemptable; /* Preemptable RCU? */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
144 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */
145 unsigned long grpmask; /* Mask to apply to leaf qsmask. */
146
147 /* 2) batch handling */
148 /*
149 * If nxtlist is not NULL, it is partitioned as follows.
150 * Any of the partitions might be empty, in which case the
151 * pointer to that partition will be equal to the pointer for
152 * the following partition. When the list is empty, all of
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
153 * the nxttail elements point to the ->nxtlist pointer itself,
154 * which in that case is NULL.
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
155 *
156 * [nxtlist, *nxttail[RCU_DONE_TAIL]):
157 * Entries that batch # <= ->completed
158 * The grace period for these entries has completed, and
159 * the other grace-period-completed entries may be moved
160 * here temporarily in rcu_process_callbacks().
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
161 * [*nxttail[RCU_DONE_TAIL], *nxttail[RCU_WAIT_TAIL]):
162 * Entries that batch # <= ->completed - 1: waiting for current GP
163 * [*nxttail[RCU_WAIT_TAIL], *nxttail[RCU_NEXT_READY_TAIL]):
164 * Entries known to have arrived before current GP ended
165 * [*nxttail[RCU_NEXT_READY_TAIL], *nxttail[RCU_NEXT_TAIL]):
166 * Entries that might have arrived after current GP ended
167 * Note that the value of *nxttail[RCU_NEXT_TAIL] will
168 * always be NULL, as this is the end of the list.
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
169 */
170 struct rcu_head *nxtlist;
171 struct rcu_head **nxttail[RCU_NEXT_SIZE];
a71fca5 @paulmck rcu: Fix whitespace inconsistencies
paulmck authored
172 long qlen; /* # of queued callbacks */
37c72e5 @paulmck rcu: Prevent RCU IPI storms in presence of high call_rcu() load
paulmck authored
173 long qlen_last_fqs_check;
174 /* qlen at last check for QS forcing */
175 unsigned long n_force_qs_snap;
176 /* did other CPU force QS recently? */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
177 long blimit; /* Upper limit on a processed batch */
178
179 #ifdef CONFIG_NO_HZ
180 /* 3) dynticks interface. */
181 struct rcu_dynticks *dynticks; /* Shared per-CPU dynticks state. */
182 int dynticks_snap; /* Per-GP tracking for dynticks. */
183 int dynticks_nmi_snap; /* Per-GP tracking for dynticks_nmi. */
184 #endif /* #ifdef CONFIG_NO_HZ */
185
186 /* 4) reasons this CPU needed to be kicked by force_quiescent_state */
187 #ifdef CONFIG_NO_HZ
188 unsigned long dynticks_fqs; /* Kicked due to dynticks idle. */
189 #endif /* #ifdef CONFIG_NO_HZ */
190 unsigned long offline_fqs; /* Kicked due to being offline. */
191 unsigned long resched_ipi; /* Sent a resched IPI. */
192
193 /* 5) __rcu_pending() statistics. */
194 long n_rcu_pending; /* rcu_pending() calls since boot. */
195 long n_rp_qs_pending;
196 long n_rp_cb_ready;
197 long n_rp_cpu_needs_gp;
198 long n_rp_gp_completed;
199 long n_rp_gp_started;
200 long n_rp_need_fqs;
201 long n_rp_need_nothing;
202
203 int cpu;
204 };
205
206 /* Values for signaled field in struct rcu_state. */
83f5b01 @paulmck rcu: Fix long-grace-period race between forcing and initialization
paulmck authored
207 #define RCU_GP_IDLE 0 /* No grace period in progress. */
208 #define RCU_GP_INIT 1 /* Grace period being initialized. */
209 #define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */
281d150 @paulmck rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handli…
paulmck authored
210 #define RCU_SAVE_COMPLETED 3 /* Need to save rsp->completed. */
211 #define RCU_FORCE_QS 4 /* Need to force quiescent state. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
212 #ifdef CONFIG_NO_HZ
213 #define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK
214 #else /* #ifdef CONFIG_NO_HZ */
281d150 @paulmck rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handli…
paulmck authored
215 #define RCU_SIGNAL_INIT RCU_SAVE_COMPLETED
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
216 #endif /* #else #ifdef CONFIG_NO_HZ */
217
218 #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */
219 #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
220 #define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ) /* for rsp->jiffies_stall */
221 #define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ) /* for rsp->jiffies_stall */
222 #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */
223 /* to take at least one */
224 /* scheduling clock irq */
225 /* before ratting on them. */
226
227 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
228
229 /*
230 * RCU global state, including node hierarchy. This hierarchy is
231 * represented in "heap" form in a dense array. The root (first level)
232 * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second
233 * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]),
234 * and the third level in ->node[m+1] and following (->node[m+1] referenced
235 * by ->level[2]). The number of levels is determined by the number of
236 * CPUs and by CONFIG_RCU_FANOUT. Small systems will have a "hierarchy"
237 * consisting of a single rcu_node.
238 */
239 struct rcu_state {
240 struct rcu_node node[NUM_RCU_NODES]; /* Hierarchy. */
241 struct rcu_node *level[NUM_RCU_LVLS]; /* Hierarchy levels. */
242 u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */
243 u8 levelspread[NUM_RCU_LVLS]; /* kids/node in each level. */
244 struct rcu_data *rda[NR_CPUS]; /* array of rdp pointers. */
245
246 /* The following fields are guarded by the root rcu_node's lock. */
247
248 u8 signaled ____cacheline_internodealigned_in_smp;
249 /* Force QS state. */
250 long gpnum; /* Current gp number. */
251 long completed; /* # of last completed gp. */
1eba8f8 @paulmck rcu: Clean up code based on review feedback from Josh Triplett, part 2
paulmck authored
252
253 /* End of fields guarded by root rcu_node's lock. */
254
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
255 spinlock_t onofflock; /* exclude on/offline and */
e74f4c4 @paulmck rcu: Make hot-unplugged CPU relinquish its own RCU callbacks
paulmck authored
256 /* starting new GP. Also */
257 /* protects the following */
258 /* orphan_cbs fields. */
259 struct rcu_head *orphan_cbs_list; /* list of rcu_head structs */
260 /* orphaned by all CPUs in */
261 /* a given leaf rcu_node */
262 /* going offline. */
263 struct rcu_head **orphan_cbs_tail; /* And tail pointer. */
264 long orphan_qlen; /* Number of orphaned cbs. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
265 spinlock_t fqslock; /* Only one task forcing */
266 /* quiescent states. */
267 unsigned long jiffies_force_qs; /* Time at which to invoke */
268 /* force_quiescent_state(). */
269 unsigned long n_force_qs; /* Number of calls to */
270 /* force_quiescent_state(). */
271 unsigned long n_force_qs_lh; /* ~Number of calls leaving */
272 /* due to lock unavailable. */
273 unsigned long n_force_qs_ngp; /* Number of calls leaving */
274 /* due to no GP active. */
275 #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
276 unsigned long gp_start; /* Time at which GP started, */
277 /* but in jiffies. */
278 unsigned long jiffies_stall; /* Time at which to check */
279 /* for CPU stalls. */
280 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
281 long dynticks_completed; /* Value of completed @ snap. */
281d150 @paulmck rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handli…
paulmck authored
282 /* Protected by fqslock. */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
283 };
284
285 #ifdef RCU_TREE_NONCORE
6258c4f kmemtrace, rcu: fix rcu_tree_trace.c data structure dependencies
Ingo Molnar authored
286
287 /*
288 * RCU implementation internal declarations:
289 */
d6714c2 @paulmck rcu: Renamings to increase RCU clarity
paulmck authored
290 extern struct rcu_state rcu_sched_state;
291 DECLARE_PER_CPU(struct rcu_data, rcu_sched_data);
6258c4f kmemtrace, rcu: fix rcu_tree_trace.c data structure dependencies
Ingo Molnar authored
292
293 extern struct rcu_state rcu_bh_state;
294 DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);
295
f41d911 @paulmck rcu: Merge preemptable-RCU functionality into hierarchical RCU
paulmck authored
296 #ifdef CONFIG_TREE_PREEMPT_RCU
297 extern struct rcu_state rcu_preempt_state;
298 DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data);
299 #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
300
9b2619a @paulmck rcu: Clean up code to address Ingo's checkpatch feedback
paulmck authored
301 #else /* #ifdef RCU_TREE_NONCORE */
9f77da9 @paulmck rcu: Move private definitions from include/linux/rcutree.h to kernel/…
paulmck authored
302
9b2619a @paulmck rcu: Clean up code to address Ingo's checkpatch feedback
paulmck authored
303 /* Forward declarations for rcutree_plugin.h */
dbe0135 @paulmck rcu: Remove inline from forward-referenced functions
paulmck authored
304 static void rcu_bootup_announce(void);
9b2619a @paulmck rcu: Clean up code to address Ingo's checkpatch feedback
paulmck authored
305 long rcu_batches_completed(void);
306 static void rcu_preempt_note_context_switch(int cpu);
307 static int rcu_preempted_readers(struct rcu_node *rnp);
308 #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
309 static void rcu_print_task_stall(struct rcu_node *rnp);
310 #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
311 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
312 #ifdef CONFIG_HOTPLUG_CPU
237c80c @paulmck rcu: Fix TREE_PREEMPT_RCU CPU_HOTPLUG bad-luck hang
paulmck authored
313 static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
314 struct rcu_node *rnp,
315 struct rcu_data *rdp);
9b2619a @paulmck rcu: Clean up code to address Ingo's checkpatch feedback
paulmck authored
316 static void rcu_preempt_offline_cpu(int cpu);
317 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
318 static void rcu_preempt_check_callbacks(int cpu);
319 static void rcu_preempt_process_callbacks(void);
320 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
321 static int rcu_preempt_pending(int cpu);
322 static int rcu_preempt_needs_cpu(int cpu);
323 static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
e74f4c4 @paulmck rcu: Make hot-unplugged CPU relinquish its own RCU callbacks
paulmck authored
324 static void rcu_preempt_send_cbs_to_orphanage(void);
9b2619a @paulmck rcu: Clean up code to address Ingo's checkpatch feedback
paulmck authored
325 static void __init __rcu_init_preempt(void);
326
327 #endif /* #else #ifdef RCU_TREE_NONCORE */
Something went wrong with that request. Please try again.