-
Notifications
You must be signed in to change notification settings - Fork 12.5k
/
sched.rs
1539 lines (1356 loc) · 56.1 KB
/
sched.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::mem;
use std::rt::local::Local;
use std::rt::mutex::NativeMutex;
use std::rt::rtio::{RemoteCallback, PausableIdleCallback, Callback, EventLoop};
use std::rt::task::BlockedTask;
use std::rt::task::Task;
use std::sync::deque;
use std::raw;
use std::rand::{XorShiftRng, Rng, Rand};
use TaskState;
use context::Context;
use coroutine::Coroutine;
use sleeper_list::SleeperList;
use stack::StackPool;
use task::{TypeSched, GreenTask, HomeSched, AnySched};
use msgq = message_queue;
/// A scheduler is responsible for coordinating the execution of Tasks
/// on a single thread. The scheduler runs inside a slightly modified
/// Rust Task. When not running this task is stored in the scheduler
/// struct. The scheduler struct acts like a baton, all scheduling
/// actions are transfers of the baton.
///
/// FIXME: This creates too many callbacks to run_sched_once, resulting
/// in too much allocation and too many events.
pub struct Scheduler {
/// ID number of the pool that this scheduler is a member of. When
/// reawakening green tasks, this is used to ensure that tasks aren't
/// reawoken on the wrong pool of schedulers.
pub pool_id: uint,
/// The pool of stacks that this scheduler has cached
pub stack_pool: StackPool,
/// Bookkeeping for the number of tasks which are currently running around
/// inside this pool of schedulers
pub task_state: TaskState,
/// There are N work queues, one per scheduler.
work_queue: deque::Worker<Box<GreenTask>>,
/// Work queues for the other schedulers. These are created by
/// cloning the core work queues.
work_queues: Vec<deque::Stealer<Box<GreenTask>>>,
/// The queue of incoming messages from other schedulers.
/// These are enqueued by SchedHandles after which a remote callback
/// is triggered to handle the message.
message_queue: msgq::Consumer<SchedMessage>,
/// Producer used to clone sched handles from
message_producer: msgq::Producer<SchedMessage>,
/// A shared list of sleeping schedulers. We'll use this to wake
/// up schedulers when pushing work onto the work queue.
sleeper_list: SleeperList,
/// Indicates that we have previously pushed a handle onto the
/// SleeperList but have not yet received the Wake message.
/// Being `true` does not necessarily mean that the scheduler is
/// not active since there are multiple event sources that may
/// wake the scheduler. It just prevents the scheduler from pushing
/// multiple handles onto the sleeper list.
sleepy: bool,
/// A flag to indicate we've received the shutdown message and should
/// no longer try to go to sleep, but exit instead.
no_sleep: bool,
/// The scheduler runs on a special task. When it is not running
/// it is stored here instead of the work queue.
sched_task: Option<Box<GreenTask>>,
/// An action performed after a context switch on behalf of the
/// code running before the context switch
cleanup_job: Option<CleanupJob>,
/// If the scheduler shouldn't run some tasks, a friend to send
/// them to.
friend_handle: Option<SchedHandle>,
/// Should this scheduler run any task, or only pinned tasks?
run_anything: bool,
/// A fast XorShift rng for scheduler use
rng: XorShiftRng,
/// A toggleable idle callback
idle_callback: Option<Box<PausableIdleCallback + Send>>,
/// A countdown that starts at a random value and is decremented
/// every time a yield check is performed. When it hits 0 a task
/// will yield.
yield_check_count: uint,
/// A flag to tell the scheduler loop it needs to do some stealing
/// in order to introduce randomness as part of a yield
steal_for_yield: bool,
// n.b. currently destructors of an object are run in top-to-bottom in order
// of field declaration. Due to its nature, the pausable idle callback
// must have some sort of handle to the event loop, so it needs to get
// destroyed before the event loop itself. For this reason, we destroy
// the event loop last to ensure that any unsafe references to it are
// destroyed before it's actually destroyed.
/// The event loop used to drive the scheduler and perform I/O
pub event_loop: Box<EventLoop + Send>,
}
/// An indication of how hard to work on a given operation, the difference
/// mainly being whether memory is synchronized or not
#[deriving(PartialEq)]
enum EffortLevel {
DontTryTooHard,
GiveItYourBest
}
static MAX_YIELD_CHECKS: uint = 20000;
fn reset_yield_check(rng: &mut XorShiftRng) -> uint {
let r: uint = Rand::rand(rng);
r % MAX_YIELD_CHECKS + 1
}
impl Scheduler {
// * Initialization Functions
pub fn new(pool_id: uint,
event_loop: Box<EventLoop + Send>,
work_queue: deque::Worker<Box<GreenTask>>,
work_queues: Vec<deque::Stealer<Box<GreenTask>>>,
sleeper_list: SleeperList,
state: TaskState)
-> Scheduler {
Scheduler::new_special(pool_id, event_loop, work_queue, work_queues,
sleeper_list, true, None, state)
}
pub fn new_special(pool_id: uint,
event_loop: Box<EventLoop + Send>,
work_queue: deque::Worker<Box<GreenTask>>,
work_queues: Vec<deque::Stealer<Box<GreenTask>>>,
sleeper_list: SleeperList,
run_anything: bool,
friend: Option<SchedHandle>,
state: TaskState)
-> Scheduler {
let (consumer, producer) = msgq::queue();
let mut sched = Scheduler {
pool_id: pool_id,
sleeper_list: sleeper_list,
message_queue: consumer,
message_producer: producer,
sleepy: false,
no_sleep: false,
event_loop: event_loop,
work_queue: work_queue,
work_queues: work_queues,
stack_pool: StackPool::new(),
sched_task: None,
cleanup_job: None,
run_anything: run_anything,
friend_handle: friend,
rng: new_sched_rng(),
idle_callback: None,
yield_check_count: 0,
steal_for_yield: false,
task_state: state,
};
sched.yield_check_count = reset_yield_check(&mut sched.rng);
return sched;
}
// FIXME: This may eventually need to be refactored so that
// the scheduler itself doesn't have to call event_loop.run.
// That will be important for embedding the runtime into external
// event loops.
// Take a main task to run, and a scheduler to run it in. Create a
// scheduler task and bootstrap into it.
pub fn bootstrap(mut ~self) {
// Build an Idle callback.
let cb = box SchedRunner as Box<Callback + Send>;
self.idle_callback = Some(self.event_loop.pausable_idle_callback(cb));
// Create a task for the scheduler with an empty context.
let sched_task = GreenTask::new_typed(Some(Coroutine::empty()),
TypeSched);
// Before starting our first task, make sure the idle callback
// is active. As we do not start in the sleep state this is
// important.
self.idle_callback.get_mut_ref().resume();
// Now, as far as all the scheduler state is concerned, we are inside
// the "scheduler" context. The scheduler immediately hands over control
// to the event loop, and this will only exit once the event loop no
// longer has any references (handles or I/O objects).
rtdebug!("starting scheduler {}", self.sched_id());
let mut sched_task = self.run(sched_task);
// Close the idle callback.
let mut sched = sched_task.sched.take_unwrap();
sched.idle_callback.take();
// Make one go through the loop to run the close callback.
let mut stask = sched.run(sched_task);
// Now that we are done with the scheduler, clean up the
// scheduler task. Do so by removing it from TLS and manually
// cleaning up the memory it uses. As we didn't actually call
// task.run() on the scheduler task we never get through all
// the cleanup code it runs.
rtdebug!("stopping scheduler {}", stask.sched.get_ref().sched_id());
// Should not have any messages
let message = stask.sched.get_mut_ref().message_queue.pop();
rtassert!(match message { msgq::Empty => true, _ => false });
stask.task.get_mut_ref().destroyed = true;
}
// This does not return a scheduler, as the scheduler is placed
// inside the task.
pub fn run(mut ~self, stask: Box<GreenTask>) -> Box<GreenTask> {
// This is unsafe because we need to place the scheduler, with
// the event_loop inside, inside our task. But we still need a
// mutable reference to the event_loop to give it the "run"
// command.
unsafe {
let event_loop: *mut Box<EventLoop + Send> = &mut self.event_loop;
// Our scheduler must be in the task before the event loop
// is started.
stask.put_with_sched(self);
(*event_loop).run();
}
// This is a serious code smell, but this function could be done away
// with if necessary. The ownership of `stask` was transferred into
// local storage just before the event loop ran, so it is possible to
// transmute `stask` as a uint across the running of the event loop to
// re-acquire ownership here.
//
// This would involve removing the Task from TLS, removing the runtime,
// forgetting the runtime, and then putting the task into `stask`. For
// now, because we have `GreenTask::convert`, I chose to take this
// method for cleanliness. This function is *not* a fundamental reason
// why this function should exist.
GreenTask::convert(Local::take())
}
// * Execution Functions - Core Loop Logic
// This function is run from the idle callback on the uv loop, indicating
// that there are no I/O events pending. When this function returns, we will
// fall back to epoll() in the uv event loop, waiting for more things to
// happen. We may come right back off epoll() if the idle callback is still
// active, in which case we're truly just polling to see if I/O events are
// complete.
//
// The model for this function is to execute as much work as possible while
// still fairly considering I/O tasks. Falling back to epoll() frequently is
// often quite expensive, so we attempt to avoid it as much as possible. If
// we have any active I/O on the event loop, then we're forced to fall back
// to epoll() in order to provide fairness, but as long as we're doing work
// and there's no active I/O, we can continue to do work.
//
// If we try really hard to do some work, but no work is available to be
// done, then we fall back to epoll() to block this thread waiting for more
// work (instead of busy waiting).
fn run_sched_once(mut ~self, stask: Box<GreenTask>) {
// Make sure that we're not lying in that the `stask` argument is indeed
// the scheduler task for this scheduler.
assert!(self.sched_task.is_none());
// Assume that we need to continue idling unless we reach the
// end of this function without performing an action.
self.idle_callback.get_mut_ref().resume();
// First we check for scheduler messages, these are higher
// priority than regular tasks.
let (mut sched, mut stask, mut did_work) =
self.interpret_message_queue(stask, DontTryTooHard);
// After processing a message, we consider doing some more work on the
// event loop. The "keep going" condition changes after the first
// iteration because we don't want to spin here infinitely.
//
// Once we start doing work we can keep doing work so long as the
// iteration does something. Note that we don't want to starve the
// message queue here, so each iteration when we're done working we
// check the message queue regardless of whether we did work or not.
let mut keep_going = !did_work || !sched.event_loop.has_active_io();
while keep_going {
let (a, b, c) = match sched.do_work(stask) {
(sched, task, false) => {
sched.interpret_message_queue(task, GiveItYourBest)
}
(sched, task, true) => {
let (sched, task, _) =
sched.interpret_message_queue(task, GiveItYourBest);
(sched, task, true)
}
};
sched = a;
stask = b;
did_work = c;
// We only keep going if we managed to do something productive and
// also don't have any active I/O. If we didn't do anything, we
// should consider going to sleep, and if we have active I/O we need
// to poll for completion.
keep_going = did_work && !sched.event_loop.has_active_io();
}
// If we ever did some work, then we shouldn't put our scheduler
// entirely to sleep just yet. Leave the idle callback active and fall
// back to epoll() to see what's going on.
if did_work {
return stask.put_with_sched(sched);
}
// If we got here then there was no work to do.
// Generate a SchedHandle and push it to the sleeper list so
// somebody can wake us up later.
if !sched.sleepy && !sched.no_sleep {
rtdebug!("scheduler has no work to do, going to sleep");
sched.sleepy = true;
let handle = sched.make_handle();
sched.sleeper_list.push(handle);
// Since we are sleeping, deactivate the idle callback.
sched.idle_callback.get_mut_ref().pause();
} else {
rtdebug!("not sleeping, already doing so or no_sleep set");
// We may not be sleeping, but we still need to deactivate
// the idle callback.
sched.idle_callback.get_mut_ref().pause();
}
// Finished a cycle without using the Scheduler. Place it back
// in TLS.
stask.put_with_sched(sched);
}
// This function returns None if the scheduler is "used", or it
// returns the still-available scheduler. At this point all
// message-handling will count as a turn of work, and as a result
// return None.
fn interpret_message_queue(mut ~self, stask: Box<GreenTask>,
effort: EffortLevel)
-> (Box<Scheduler>, Box<GreenTask>, bool)
{
let msg = if effort == DontTryTooHard {
self.message_queue.casual_pop()
} else {
// When popping our message queue, we could see an "inconsistent"
// state which means that we *should* be able to pop data, but we
// are unable to at this time. Our options are:
//
// 1. Spin waiting for data
// 2. Ignore this and pretend we didn't find a message
//
// If we choose route 1, then if the pusher in question is currently
// pre-empted, we're going to take up our entire time slice just
// spinning on this queue. If we choose route 2, then the pusher in
// question is still guaranteed to make a send() on its async
// handle, so we will guaranteed wake up and see its message at some
// point.
//
// I have chosen to take route #2.
match self.message_queue.pop() {
msgq::Data(t) => Some(t),
msgq::Empty | msgq::Inconsistent => None
}
};
match msg {
Some(PinnedTask(task)) => {
let mut task = task;
task.give_home(HomeSched(self.make_handle()));
let (sched, task) = self.resume_task_immediately(stask, task);
(sched, task, true)
}
Some(TaskFromFriend(task)) => {
rtdebug!("got a task from a friend. lovely!");
let (sched, task) =
self.process_task(stask, task,
Scheduler::resume_task_immediately_cl);
(sched, task, true)
}
Some(RunOnce(task)) => {
// bypass the process_task logic to force running this task once
// on this home scheduler. This is often used for I/O (homing).
let (sched, task) = self.resume_task_immediately(stask, task);
(sched, task, true)
}
Some(Wake) => {
self.sleepy = false;
(self, stask, true)
}
Some(Shutdown) => {
rtdebug!("shutting down");
if self.sleepy {
// There may be an outstanding handle on the
// sleeper list. Pop them all to make sure that's
// not the case.
loop {
match self.sleeper_list.pop() {
Some(handle) => {
let mut handle = handle;
handle.send(Wake);
}
None => break
}
}
}
// No more sleeping. After there are no outstanding
// event loop references we will shut down.
self.no_sleep = true;
self.sleepy = false;
(self, stask, true)
}
Some(NewNeighbor(neighbor)) => {
self.work_queues.push(neighbor);
(self, stask, false)
}
None => (self, stask, false)
}
}
fn do_work(mut ~self, stask: Box<GreenTask>)
-> (Box<Scheduler>, Box<GreenTask>, bool) {
rtdebug!("scheduler calling do work");
match self.find_work() {
Some(task) => {
rtdebug!("found some work! running the task");
let (sched, task) =
self.process_task(stask, task,
Scheduler::resume_task_immediately_cl);
(sched, task, true)
}
None => {
rtdebug!("no work was found, returning the scheduler struct");
(self, stask, false)
}
}
}
// Workstealing: In this iteration of the runtime each scheduler
// thread has a distinct work queue. When no work is available
// locally, make a few attempts to steal work from the queues of
// other scheduler threads. If a few steals fail we end up in the
// old "no work" path which is fine.
// First step in the process is to find a task. This function does
// that by first checking the local queue, and if there is no work
// there, trying to steal from the remote work queues.
fn find_work(&mut self) -> Option<Box<GreenTask>> {
rtdebug!("scheduler looking for work");
if !self.steal_for_yield {
match self.work_queue.pop() {
Some(task) => {
rtdebug!("found a task locally");
return Some(task)
}
None => {
rtdebug!("scheduler trying to steal");
return self.try_steals();
}
}
} else {
// During execution of the last task, it performed a 'yield',
// so we're doing some work stealing in order to introduce some
// scheduling randomness. Otherwise we would just end up popping
// that same task again. This is pretty lame and is to work around
// the problem that work stealing is not designed for 'non-strict'
// (non-fork-join) task parallelism.
self.steal_for_yield = false;
match self.try_steals() {
Some(task) => {
rtdebug!("stole a task after yielding");
return Some(task);
}
None => {
rtdebug!("did not steal a task after yielding");
// Back to business
return self.find_work();
}
}
}
}
// Try stealing from all queues the scheduler knows about. This
// naive implementation can steal from our own queue or from other
// special schedulers.
fn try_steals(&mut self) -> Option<Box<GreenTask>> {
let work_queues = &mut self.work_queues;
let len = work_queues.len();
let start_index = self.rng.gen_range(0, len);
for index in range(0, len).map(|i| (i + start_index) % len) {
match work_queues.get_mut(index).steal() {
deque::Data(task) => {
rtdebug!("found task by stealing");
return Some(task)
}
_ => ()
}
};
rtdebug!("giving up on stealing");
return None;
}
// * Task Routing Functions - Make sure tasks send up in the right
// place.
fn process_task(mut ~self,
cur: Box<GreenTask>,
mut next: Box<GreenTask>,
schedule_fn: SchedulingFn)
-> (Box<Scheduler>, Box<GreenTask>) {
rtdebug!("processing a task");
match next.take_unwrap_home() {
HomeSched(home_handle) => {
if home_handle.sched_id != self.sched_id() {
rtdebug!("sending task home");
next.give_home(HomeSched(home_handle));
Scheduler::send_task_home(next);
(self, cur)
} else {
rtdebug!("running task here");
next.give_home(HomeSched(home_handle));
schedule_fn(self, cur, next)
}
}
AnySched if self.run_anything => {
rtdebug!("running anysched task here");
next.give_home(AnySched);
schedule_fn(self, cur, next)
}
AnySched => {
rtdebug!("sending task to friend");
next.give_home(AnySched);
self.send_to_friend(next);
(self, cur)
}
}
}
fn send_task_home(task: Box<GreenTask>) {
let mut task = task;
match task.take_unwrap_home() {
HomeSched(mut home_handle) => home_handle.send(PinnedTask(task)),
AnySched => rtabort!("error: cannot send anysched task home"),
}
}
/// Take a non-homed task we aren't allowed to run here and send
/// it to the designated friend scheduler to execute.
fn send_to_friend(&mut self, task: Box<GreenTask>) {
rtdebug!("sending a task to friend");
match self.friend_handle {
Some(ref mut handle) => {
handle.send(TaskFromFriend(task));
}
None => {
rtabort!("tried to send task to a friend but scheduler has no friends");
}
}
}
/// Schedule a task to be executed later.
///
/// Pushes the task onto the work stealing queue and tells the
/// event loop to run it later. Always use this instead of pushing
/// to the work queue directly.
pub fn enqueue_task(&mut self, task: Box<GreenTask>) {
// We push the task onto our local queue clone.
assert!(!task.is_sched());
self.work_queue.push(task);
match self.idle_callback {
Some(ref mut idle) => idle.resume(),
None => {} // allow enqueuing before the scheduler starts
}
// We've made work available. Notify a
// sleeping scheduler.
match self.sleeper_list.casual_pop() {
Some(handle) => {
let mut handle = handle;
handle.send(Wake)
}
None => { (/* pass */) }
};
}
// * Core Context Switching Functions
// The primary function for changing contexts. In the current
// design the scheduler is just a slightly modified GreenTask, so
// all context swaps are from GreenTask to GreenTask. The only difference
// between the various cases is where the inputs come from, and
// what is done with the resulting task. That is specified by the
// cleanup function f, which takes the scheduler and the
// old task as inputs.
pub fn change_task_context(mut ~self,
mut current_task: Box<GreenTask>,
mut next_task: Box<GreenTask>,
f: |&mut Scheduler, Box<GreenTask>|)
-> Box<GreenTask> {
let f_opaque = ClosureConverter::from_fn(f);
let current_task_dupe = &mut *current_task as *mut GreenTask;
// The current task is placed inside an enum with the cleanup
// function. This enum is then placed inside the scheduler.
self.cleanup_job = Some(CleanupJob::new(current_task, f_opaque));
// The scheduler is then placed inside the next task.
next_task.sched = Some(self);
// However we still need an internal mutable pointer to the
// original task. The strategy here was "arrange memory, then
// get pointers", so we crawl back up the chain using
// transmute to eliminate borrowck errors.
unsafe {
let sched: &mut Scheduler =
mem::transmute(&**next_task.sched.get_mut_ref());
let current_task: &mut GreenTask = match sched.cleanup_job {
Some(CleanupJob { task: ref mut task, .. }) => &mut **task,
None => rtabort!("no cleanup job")
};
let (current_task_context, next_task_context) =
Scheduler::get_contexts(current_task, &mut *next_task);
// Done with everything - put the next task in TLS. This
// works because due to transmute the borrow checker
// believes that we have no internal pointers to
// next_task.
mem::forget(next_task);
// The raw context swap operation. The next action taken
// will be running the cleanup job from the context of the
// next task.
Context::swap(current_task_context, next_task_context);
}
// When the context swaps back to this task we immediately
// run the cleanup job, as expected by the previously called
// swap_contexts function.
let mut current_task: Box<GreenTask> = unsafe {
mem::transmute(current_task_dupe)
};
current_task.sched.get_mut_ref().run_cleanup_job();
// See the comments in switch_running_tasks_and_then for why a lock
// is acquired here. This is the resumption points and the "bounce"
// that it is referring to.
unsafe {
let _guard = current_task.nasty_deschedule_lock.lock();
}
return current_task;
}
// Returns a mutable reference to both contexts involved in this
// swap. This is unsafe - we are getting mutable internal
// references to keep even when we don't own the tasks. It looks
// kinda safe because we are doing transmutes before passing in
// the arguments.
pub fn get_contexts<'a>(current_task: &mut GreenTask,
next_task: &mut GreenTask)
-> (&'a mut Context, &'a mut Context)
{
let current_task_context =
&mut current_task.coroutine.get_mut_ref().saved_context;
let next_task_context =
&mut next_task.coroutine.get_mut_ref().saved_context;
unsafe {
(mem::transmute(current_task_context),
mem::transmute(next_task_context))
}
}
// * Context Swapping Helpers - Here be ugliness!
pub fn resume_task_immediately(~self,
cur: Box<GreenTask>,
next: Box<GreenTask>)
-> (Box<Scheduler>, Box<GreenTask>) {
assert!(cur.is_sched());
let mut cur = self.change_task_context(cur, next, |sched, stask| {
assert!(sched.sched_task.is_none());
sched.sched_task = Some(stask);
});
(cur.sched.take_unwrap(), cur)
}
fn resume_task_immediately_cl(sched: Box<Scheduler>,
cur: Box<GreenTask>,
next: Box<GreenTask>)
-> (Box<Scheduler>, Box<GreenTask>) {
sched.resume_task_immediately(cur, next)
}
/// Block a running task, context switch to the scheduler, then pass the
/// blocked task to a closure.
///
/// # Safety note
///
/// The closure here is a *stack* closure that lives in the
/// running task. It gets transmuted to the scheduler's lifetime
/// and called while the task is blocked.
///
/// This passes a Scheduler pointer to the fn after the context switch
/// in order to prevent that fn from performing further scheduling operations.
/// Doing further scheduling could easily result in infinite recursion.
///
/// Note that if the closure provided relinquishes ownership of the
/// BlockedTask, then it is possible for the task to resume execution before
/// the closure has finished executing. This would naturally introduce a
/// race if the closure and task shared portions of the environment.
///
/// This situation is currently prevented, or in other words it is
/// guaranteed that this function will not return before the given closure
/// has returned.
pub fn deschedule_running_task_and_then(mut ~self,
cur: Box<GreenTask>,
f: |&mut Scheduler, BlockedTask|) {
// Trickier - we need to get the scheduler task out of self
// and use it as the destination.
let stask = self.sched_task.take_unwrap();
// Otherwise this is the same as below.
self.switch_running_tasks_and_then(cur, stask, f)
}
pub fn switch_running_tasks_and_then(~self,
cur: Box<GreenTask>,
next: Box<GreenTask>,
f: |&mut Scheduler, BlockedTask|) {
// And here comes one of the sad moments in which a lock is used in a
// core portion of the rust runtime. As always, this is highly
// undesirable, so there's a good reason behind it.
//
// There is an excellent outline of the problem in issue #8132, and it's
// summarized in that `f` is executed on a sched task, but its
// environment is on the previous task. If `f` relinquishes ownership of
// the BlockedTask, then it may introduce a race where `f` is using the
// environment as well as the code after the 'deschedule' block.
//
// The solution we have chosen to adopt for now is to acquire a
// task-local lock around this block. The resumption of the task in
// context switching will bounce on the lock, thereby waiting for this
// block to finish, eliminating the race mentioned above.
// fail!("should never return!");
//
// To actually maintain a handle to the lock, we use an unsafe pointer
// to it, but we're guaranteed that the task won't exit until we've
// unlocked the lock so there's no worry of this memory going away.
let cur = self.change_task_context(cur, next, |sched, mut task| {
let lock: *mut NativeMutex = &mut task.nasty_deschedule_lock;
unsafe {
let _guard = (*lock).lock();
f(sched, BlockedTask::block(task.swap()));
}
});
cur.put();
}
fn switch_task(sched: Box<Scheduler>,
cur: Box<GreenTask>,
next: Box<GreenTask>)
-> (Box<Scheduler>, Box<GreenTask>) {
let mut cur = sched.change_task_context(cur, next, |sched, last_task| {
if last_task.is_sched() {
assert!(sched.sched_task.is_none());
sched.sched_task = Some(last_task);
} else {
sched.enqueue_task(last_task);
}
});
(cur.sched.take_unwrap(), cur)
}
// * Task Context Helpers
/// Called by a running task to end execution, after which it will
/// be recycled by the scheduler for reuse in a new task.
pub fn terminate_current_task(mut ~self, cur: Box<GreenTask>) -> ! {
// Similar to deschedule running task and then, but cannot go through
// the task-blocking path. The task is already dying.
let stask = self.sched_task.take_unwrap();
let _cur = self.change_task_context(cur, stask, |sched, mut dead_task| {
let coroutine = dead_task.coroutine.take_unwrap();
coroutine.recycle(&mut sched.stack_pool);
sched.task_state.decrement();
});
fail!("should never return!");
}
pub fn run_task(~self, cur: Box<GreenTask>, next: Box<GreenTask>) {
let (sched, task) =
self.process_task(cur, next, Scheduler::switch_task);
task.put_with_sched(sched);
}
pub fn run_task_later(mut cur: Box<GreenTask>, next: Box<GreenTask>) {
let mut sched = cur.sched.take_unwrap();
sched.enqueue_task(next);
cur.put_with_sched(sched);
}
/// Yield control to the scheduler, executing another task. This is guaranteed
/// to introduce some amount of randomness to the scheduler. Currently the
/// randomness is a result of performing a round of work stealing (which
/// may end up stealing from the current scheduler).
pub fn yield_now(mut ~self, cur: Box<GreenTask>) {
// Async handles trigger the scheduler by calling yield_now on the local
// task, which eventually gets us to here. See comments in SchedRunner
// for more info on this.
if cur.is_sched() {
assert!(self.sched_task.is_none());
self.run_sched_once(cur);
} else {
self.yield_check_count = reset_yield_check(&mut self.rng);
// Tell the scheduler to start stealing on the next iteration
self.steal_for_yield = true;
let stask = self.sched_task.take_unwrap();
let cur = self.change_task_context(cur, stask, |sched, task| {
sched.enqueue_task(task);
});
cur.put()
}
}
pub fn maybe_yield(mut ~self, cur: Box<GreenTask>) {
// It's possible for sched tasks to possibly call this function, and it
// just means that they're likely sending on channels (which
// occasionally call this function). Sched tasks follow different paths
// when executing yield_now(), which may possibly trip the assertion
// below. For this reason, we just have sched tasks bail out soon.
//
// Sched tasks have no need to yield anyway because as soon as they
// return they'll yield to other threads by falling back to the event
// loop. Additionally, we completely control sched tasks, so we can make
// sure that they never execute more than enough code.
if cur.is_sched() {
return cur.put_with_sched(self)
}
// The number of times to do the yield check before yielding, chosen
// arbitrarily.
rtassert!(self.yield_check_count > 0);
self.yield_check_count -= 1;
if self.yield_check_count == 0 {
self.yield_now(cur);
} else {
cur.put_with_sched(self);
}
}
// * Utility Functions
pub fn sched_id(&self) -> uint { self as *const Scheduler as uint }
pub fn run_cleanup_job(&mut self) {
let cleanup_job = self.cleanup_job.take_unwrap();
cleanup_job.run(self)
}
pub fn make_handle(&mut self) -> SchedHandle {
let remote = self.event_loop.remote_callback(box SchedRunner);
return SchedHandle {
remote: remote,
queue: self.message_producer.clone(),
sched_id: self.sched_id()
}
}
}
// Supporting types
type SchedulingFn = fn(Box<Scheduler>, Box<GreenTask>, Box<GreenTask>)
-> (Box<Scheduler>, Box<GreenTask>);
pub enum SchedMessage {
Wake,
Shutdown,
NewNeighbor(deque::Stealer<Box<GreenTask>>),
PinnedTask(Box<GreenTask>),
TaskFromFriend(Box<GreenTask>),
RunOnce(Box<GreenTask>),
}
pub struct SchedHandle {
remote: Box<RemoteCallback + Send>,
queue: msgq::Producer<SchedMessage>,
pub sched_id: uint
}
impl SchedHandle {
pub fn send(&mut self, msg: SchedMessage) {
self.queue.push(msg);
self.remote.fire();
}
}
struct SchedRunner;
impl Callback for SchedRunner {
fn call(&mut self) {
// In theory, this function needs to invoke the `run_sched_once`
// function on the scheduler. Sadly, we have no context here, except for
// knowledge of the local `Task`. In order to avoid a call to
// `GreenTask::convert`, we just call `yield_now` and the scheduler will
// detect when a sched task performs a yield vs a green task performing
// a yield (and act accordingly).
//
// This function could be converted to `GreenTask::convert` if
// absolutely necessary, but for cleanliness it is much better to not
// use the conversion function.
let task: Box<Task> = Local::take();
task.yield_now();
}
}
struct CleanupJob {
task: Box<GreenTask>,
f: UnsafeTaskReceiver
}
impl CleanupJob {
pub fn new(task: Box<GreenTask>, f: UnsafeTaskReceiver) -> CleanupJob {
CleanupJob {
task: task,
f: f
}
}
pub fn run(self, sched: &mut Scheduler) {
let CleanupJob { task: task, f: f } = self;
f.to_fn()(sched, task)
}
}
// FIXME: Some hacks to put a || closure in Scheduler without borrowck
// complaining
type UnsafeTaskReceiver = raw::Closure;
trait ClosureConverter {
fn from_fn(|&mut Scheduler, Box<GreenTask>|) -> Self;
fn to_fn(self) -> |&mut Scheduler, Box<GreenTask>|:'static ;
}
impl ClosureConverter for UnsafeTaskReceiver {
fn from_fn(f: |&mut Scheduler, Box<GreenTask>|) -> UnsafeTaskReceiver {
unsafe { mem::transmute(f) }
}
fn to_fn(self) -> |&mut Scheduler, Box<GreenTask>|:'static {
unsafe { mem::transmute(self) }
}
}
// On unix, we read randomness straight from /dev/urandom, but the
// default constructor of an XorShiftRng does this via io::fs, which
// relies on the scheduler existing, so we have to manually load
// randomness. Windows has its own C API for this, so we don't need to
// worry there.
#[cfg(windows)]
fn new_sched_rng() -> XorShiftRng {
use std::rand::OsRng;
match OsRng::new() {
Ok(mut r) => r.gen(),
Err(e) => {
rtabort!("sched: failed to create seeded RNG: {}", e)
}
}
}
#[cfg(unix)]
fn new_sched_rng() -> XorShiftRng {
use libc;
use std::mem;
use std::rand::SeedableRng;
let fd = "/dev/urandom".with_c_str(|name| {
unsafe { libc::open(name, libc::O_RDONLY, 0) }
});
if fd == -1 {
rtabort!("could not open /dev/urandom for reading.")
}