Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 340 lines (287 sloc) 10.612 kb
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
1 /*
2 * async.c: Asynchronous function calls for boot performance
3 *
4 * (C) Copyright 2009 Intel Corporation
5 * Author: Arjan van de Ven <arjan@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 */
12
13
14 /*
15
16 Goals and Theory of Operation
17
18 The primary goal of this feature is to reduce the kernel boot time,
19 by doing various independent hardware delays and discovery operations
20 decoupled and not strictly serialized.
21
22 More specifically, the asynchronous function call concept allows
23 certain operations (primarily during system boot) to happen
24 asynchronously, out of order, while these operations still
25 have their externally visible parts happen sequentially and in-order.
26 (not unlike how out-of-order CPUs retire their instructions in order)
27
28 Key to the asynchronous function call implementation is the concept of
29 a "sequence cookie" (which, although it has an abstracted type, can be
30 thought of as a monotonically incrementing number).
31
32 The async core will assign each scheduled event such a sequence cookie and
33 pass this to the called functions.
34
35 The asynchronously called function should before doing a globally visible
36 operation, such as registering device numbers, call the
37 async_synchronize_cookie() function and pass in its own cookie. The
38 async_synchronize_cookie() function will make sure that all asynchronous
39 operations that were scheduled prior to the operation corresponding with the
40 cookie have completed.
41
42 Subsystem/driver initialization code that scheduled asynchronous probe
43 functions, but which shares global resources with other drivers/subsystems
44 that do not use the asynchronous call feature, need to do a full
45 synchronization with the async_synchronize_full() function, before returning
46 from their init function. This is to maintain strict ordering between the
47 asynchronous and synchronous parts of the kernel.
48
49 */
50
51 #include <linux/async.h>
84c1502 async: Fixed an include coding style issue
Paul McQuade authored
52 #include <linux/atomic.h>
53 #include <linux/ktime.h>
9984de1 kernel: Map most files to use export.h instead of module.h
Paul Gortmaker authored
54 #include <linux/export.h>
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
55 #include <linux/wait.h>
56 #include <linux/sched.h>
5a0e3ad include cleanup: Update gfp.h and slab.h includes to prepare for brea…
Tejun Heo authored
57 #include <linux/slab.h>
083b804 async: use workqueue for worker pool
Tejun Heo authored
58 #include <linux/workqueue.h>
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
59
60 static async_cookie_t next_cookie = 1;
61
62 #define MAX_WORK 32768
63
64 static LIST_HEAD(async_pending);
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
65 static ASYNC_DOMAIN(async_running);
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
66 static LIST_HEAD(async_domains);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
67 static DEFINE_SPINLOCK(async_lock);
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
68 static DEFINE_MUTEX(async_register_mutex);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
69
70 struct async_entry {
083b804 async: use workqueue for worker pool
Tejun Heo authored
71 struct list_head list;
72 struct work_struct work;
73 async_cookie_t cookie;
74 async_func_ptr *func;
75 void *data;
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
76 struct async_domain *running;
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
77 };
78
79 static DECLARE_WAIT_QUEUE_HEAD(async_done);
80
81 static atomic_t entry_count;
82
83
84 /*
85 * MUST be called with the lock held!
86 */
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
87 static async_cookie_t __lowest_in_progress(struct async_domain *running)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
88 {
89 struct async_entry *entry;
d5a877e async: make sure independent async domains can't accidentally entangle
James Bottomley authored
90
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
91 if (!list_empty(&running->domain)) {
92 entry = list_first_entry(&running->domain, typeof(*entry), list);
3af968e @torvalds async: Fix lack of boot-time console due to insufficient synchronization
authored
93 return entry->cookie;
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
94 }
95
3af968e @torvalds async: Fix lack of boot-time console due to insufficient synchronization
authored
96 list_for_each_entry(entry, &async_pending, list)
97 if (entry->running == running)
98 return entry->cookie;
d5a877e async: make sure independent async domains can't accidentally entangle
James Bottomley authored
99
3af968e @torvalds async: Fix lack of boot-time console due to insufficient synchronization
authored
100 return next_cookie; /* "infinity" value */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
101 }
37a76bd @fenrus75 async: fix __lowest_in_progress()
fenrus75 authored
102
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
103 static async_cookie_t lowest_in_progress(struct async_domain *running)
37a76bd @fenrus75 async: fix __lowest_in_progress()
fenrus75 authored
104 {
105 unsigned long flags;
106 async_cookie_t ret;
107
108 spin_lock_irqsave(&async_lock, flags);
109 ret = __lowest_in_progress(running);
110 spin_unlock_irqrestore(&async_lock, flags);
111 return ret;
112 }
083b804 async: use workqueue for worker pool
Tejun Heo authored
113
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
114 /*
115 * pick the first pending entry and run it
116 */
083b804 async: use workqueue for worker pool
Tejun Heo authored
117 static void async_run_entry_fn(struct work_struct *work)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
118 {
083b804 async: use workqueue for worker pool
Tejun Heo authored
119 struct async_entry *entry =
120 container_of(work, struct async_entry, work);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
121 unsigned long flags;
124ff4e @vitalivanov async: uninitialized warning corrections
vitalivanov authored
122 ktime_t uninitialized_var(calltime), delta, rettime;
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
123 struct async_domain *running = entry->running;
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
124
083b804 async: use workqueue for worker pool
Tejun Heo authored
125 /* 1) move self to the running queue */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
126 spin_lock_irqsave(&async_lock, flags);
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
127 list_move_tail(&entry->list, &running->domain);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
128 spin_unlock_irqrestore(&async_lock, flags);
129
083b804 async: use workqueue for worker pool
Tejun Heo authored
130 /* 2) run (and print duration) */
ad160d2 @fenrus75 async: don't do the initcall stuff post boot
fenrus75 authored
131 if (initcall_debug && system_state == SYSTEM_BOOTING) {
84c1502 async: Fixed an include coding style issue
Paul McQuade authored
132 printk(KERN_DEBUG "calling %lli_%pF @ %i\n",
133 (long long)entry->cookie,
58763a2 kernel/async.c: fix printk warnings
Andrew Morton authored
134 entry->func, task_pid_nr(current));
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
135 calltime = ktime_get();
136 }
137 entry->func(entry->data, entry->cookie);
ad160d2 @fenrus75 async: don't do the initcall stuff post boot
fenrus75 authored
138 if (initcall_debug && system_state == SYSTEM_BOOTING) {
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
139 rettime = ktime_get();
140 delta = ktime_sub(rettime, calltime);
84c1502 async: Fixed an include coding style issue
Paul McQuade authored
141 printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n",
58763a2 kernel/async.c: fix printk warnings
Andrew Morton authored
142 (long long)entry->cookie,
143 entry->func,
144 (long long)ktime_to_ns(delta) >> 10);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
145 }
146
083b804 async: use workqueue for worker pool
Tejun Heo authored
147 /* 3) remove self from the running queue */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
148 spin_lock_irqsave(&async_lock, flags);
149 list_del(&entry->list);
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
150 if (running->registered && --running->count == 0)
151 list_del_init(&running->node);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
152
083b804 async: use workqueue for worker pool
Tejun Heo authored
153 /* 4) free the entry */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
154 kfree(entry);
155 atomic_dec(&entry_count);
156
157 spin_unlock_irqrestore(&async_lock, flags);
158
083b804 async: use workqueue for worker pool
Tejun Heo authored
159 /* 5) wake up any waiters */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
160 wake_up(&async_done);
161 }
162
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
163 static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *running)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
164 {
165 struct async_entry *entry;
166 unsigned long flags;
167 async_cookie_t newcookie;
168
169 /* allow irq-off callers */
170 entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
171
172 /*
173 * If we're out of memory or if there's too much work
174 * pending already, we execute synchronously.
175 */
083b804 async: use workqueue for worker pool
Tejun Heo authored
176 if (!entry || atomic_read(&entry_count) > MAX_WORK) {
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
177 kfree(entry);
178 spin_lock_irqsave(&async_lock, flags);
179 newcookie = next_cookie++;
180 spin_unlock_irqrestore(&async_lock, flags);
181
182 /* low on memory.. run synchronously */
183 ptr(data, newcookie);
184 return newcookie;
185 }
083b804 async: use workqueue for worker pool
Tejun Heo authored
186 INIT_WORK(&entry->work, async_run_entry_fn);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
187 entry->func = ptr;
188 entry->data = data;
189 entry->running = running;
190
191 spin_lock_irqsave(&async_lock, flags);
192 newcookie = entry->cookie = next_cookie++;
193 list_add_tail(&entry->list, &async_pending);
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
194 if (running->registered && running->count++ == 0)
195 list_add_tail(&running->node, &async_domains);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
196 atomic_inc(&entry_count);
197 spin_unlock_irqrestore(&async_lock, flags);
083b804 async: use workqueue for worker pool
Tejun Heo authored
198
199 /* schedule for execution */
200 queue_work(system_unbound_wq, &entry->work);
201
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
202 return newcookie;
203 }
204
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
205 /**
206 * async_schedule - schedule a function for asynchronous execution
207 * @ptr: function to execute asynchronously
208 * @data: data pointer to pass to the function
209 *
210 * Returns an async_cookie_t that may be used for checkpointing later.
211 * Note: This function may be called from atomic or non-atomic contexts.
212 */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
213 async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
214 {
7a89bbc @cohuck async: Fix running list handling.
cohuck authored
215 return __async_schedule(ptr, data, &async_running);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
216 }
217 EXPORT_SYMBOL_GPL(async_schedule);
218
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
219 /**
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
220 * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
221 * @ptr: function to execute asynchronously
222 * @data: data pointer to pass to the function
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
223 * @running: running list for the domain
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
224 *
225 * Returns an async_cookie_t that may be used for checkpointing later.
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
226 * @running may be used in the async_synchronize_*_domain() functions
227 * to wait within a certain synchronization domain rather than globally.
228 * A synchronization domain is specified via the running queue @running to use.
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
229 * Note: This function may be called from atomic or non-atomic contexts.
230 */
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
231 async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data,
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
232 struct async_domain *running)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
233 {
234 return __async_schedule(ptr, data, running);
235 }
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
236 EXPORT_SYMBOL_GPL(async_schedule_domain);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
237
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
238 /**
239 * async_synchronize_full - synchronize all asynchronous function calls
240 *
241 * This function waits until all asynchronous function calls have been done.
242 */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
243 void async_synchronize_full(void)
244 {
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
245 mutex_lock(&async_register_mutex);
33b04b9 @fenrus75 async: make async_synchronize_full() more serializing
fenrus75 authored
246 do {
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
247 struct async_domain *domain = NULL;
248
249 spin_lock_irq(&async_lock);
250 if (!list_empty(&async_domains))
251 domain = list_first_entry(&async_domains, typeof(*domain), node);
252 spin_unlock_irq(&async_lock);
253
254 async_synchronize_cookie_domain(next_cookie, domain);
255 } while (!list_empty(&async_domains));
256 mutex_unlock(&async_register_mutex);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
257 }
258 EXPORT_SYMBOL_GPL(async_synchronize_full);
259
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
260 /**
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
261 * async_unregister_domain - ensure no more anonymous waiters on this domain
262 * @domain: idle domain to flush out of any async_synchronize_full instances
263 *
264 * async_synchronize_{cookie|full}_domain() are not flushed since callers
265 * of these routines should know the lifetime of @domain
266 *
267 * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
268 */
269 void async_unregister_domain(struct async_domain *domain)
270 {
271 mutex_lock(&async_register_mutex);
272 spin_lock_irq(&async_lock);
273 WARN_ON(!domain->registered || !list_empty(&domain->node) ||
274 !list_empty(&domain->domain));
275 domain->registered = 0;
276 spin_unlock_irq(&async_lock);
277 mutex_unlock(&async_register_mutex);
278 }
279 EXPORT_SYMBOL_GPL(async_unregister_domain);
280
281 /**
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
282 * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
283 * @domain: running list to synchronize on
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
284 *
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
285 * This function waits until all asynchronous function calls for the
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
286 * synchronization domain specified by the running list @domain have been done.
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
287 */
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
288 void async_synchronize_full_domain(struct async_domain *domain)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
289 {
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
290 async_synchronize_cookie_domain(next_cookie, domain);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
291 }
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
292 EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
293
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
294 /**
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
295 * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
296 * @cookie: async_cookie_t to use as checkpoint
297 * @running: running list to synchronize on
298 *
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
299 * This function waits until all asynchronous function calls for the
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
300 * synchronization domain specified by running list @running submitted
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
301 * prior to @cookie have been done.
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
302 */
2955b47 @djbw [SCSI] async: introduce 'async_domain' type
djbw authored
303 void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *running)
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
304 {
124ff4e @vitalivanov async: uninitialized warning corrections
vitalivanov authored
305 ktime_t uninitialized_var(starttime), delta, endtime;
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
306
a468348 @djbw [SCSI] async: make async_synchronize_full() flush all work regardless…
djbw authored
307 if (!running)
308 return;
309
ad160d2 @fenrus75 async: don't do the initcall stuff post boot
fenrus75 authored
310 if (initcall_debug && system_state == SYSTEM_BOOTING) {
84c1502 async: Fixed an include coding style issue
Paul McQuade authored
311 printk(KERN_DEBUG "async_waiting @ %i\n", task_pid_nr(current));
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
312 starttime = ktime_get();
313 }
314
37a76bd @fenrus75 async: fix __lowest_in_progress()
fenrus75 authored
315 wait_event(async_done, lowest_in_progress(running) >= cookie);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
316
ad160d2 @fenrus75 async: don't do the initcall stuff post boot
fenrus75 authored
317 if (initcall_debug && system_state == SYSTEM_BOOTING) {
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
318 endtime = ktime_get();
319 delta = ktime_sub(endtime, starttime);
320
84c1502 async: Fixed an include coding style issue
Paul McQuade authored
321 printk(KERN_DEBUG "async_continuing @ %i after %lli usec\n",
58763a2 kernel/async.c: fix printk warnings
Andrew Morton authored
322 task_pid_nr(current),
323 (long long)ktime_to_ns(delta) >> 10);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
324 }
325 }
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
326 EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
327
f30d5b3 @cohuck async: Add some documentation.
cohuck authored
328 /**
329 * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
330 * @cookie: async_cookie_t to use as checkpoint
331 *
332 * This function waits until all asynchronous function calls prior to @cookie
333 * have been done.
334 */
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
335 void async_synchronize_cookie(async_cookie_t cookie)
336 {
766ccb9 @cohuck async: Rename _special -> _domain for clarity.
cohuck authored
337 async_synchronize_cookie_domain(cookie, &async_running);
22a9d64 @fenrus75 async: Asynchronous function calls to speed up kernel boot
fenrus75 authored
338 }
339 EXPORT_SYMBOL_GPL(async_synchronize_cookie);
Something went wrong with that request. Please try again.