Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 370 lines (305 sloc) 9.748 kb
640886c Polymorphic MapReduce!
Eric Holk authored
1 /**
2 A parallel word-frequency counting program.
3
4 This is meant primarily to demonstrate Rust's MapReduce framework.
5
6 It takes a list of files on the command line and outputs a list of
7 words along with how many times each word is used.
8
9 */
10
05543fd Eric Holk Make tests pass
eholk authored
11 // xfail-pretty
12
640886c Polymorphic MapReduce!
Eric Holk authored
13 use std;
14
e5d095d Tim Chevalier Change option::t to option
catamorphism authored
15 import option = option;
8337fa1 Brian Anderson Camel case the option type
brson authored
16 import option::Some;
17 import option::None;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
18 import str;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
19 import std::map;
20 import std::map::hashmap;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
21 import vec;
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
22 import io;
1125831 Brian Anderson Convert more core types to camel case
brson authored
23 import io::WriterUtil;
640886c Polymorphic MapReduce!
Eric Holk authored
24
25 import std::time;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
26 import u64;
27
28 import task;
29 import comm;
9c6890f Brian Anderson Convert more core types to camel case
brson authored
30 import comm::Chan;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
31 import comm::chan;
9c6890f Brian Anderson Convert more core types to camel case
brson authored
32 import comm::Port;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
33 import comm::port;
34 import comm::recv;
35 import comm::send;
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
36
38891b9 Paul Stansifer Update invocation syntax for `macro_rules!`
paulstansifer authored
37 macro_rules! move_out (
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
38 { $x:expr } => { unsafe { let y <- *ptr::addr_of($x); y } }
38891b9 Paul Stansifer Update invocation syntax for `macro_rules!`
paulstansifer authored
39 )
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
40
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
41 trait word_reader {
8337fa1 Brian Anderson Camel case the option type
brson authored
42 fn read_word() -> Option<~str>;
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
43 }
44
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
45 trait hash_key {
97452c0 Niko Matsakis Remove modes from map API and replace with regions.
nikomatsakis authored
46 pure fn hash() -> uint;
47 pure fn eq(&&k: self) -> bool;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
48 }
49
50 fn mk_hash<K: const hash_key, V: copy>() -> map::hashmap<K, V> {
97452c0 Niko Matsakis Remove modes from map API and replace with regions.
nikomatsakis authored
51 pure fn hashfn<K: const hash_key>(k: &K) -> uint { k.hash() }
52 pure fn hasheq<K: const hash_key>(k1: &K, k2: &K) -> bool { k1.eq(*k2) }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
53
97452c0 Niko Matsakis Remove modes from map API and replace with regions.
nikomatsakis authored
54 map::hashmap(hashfn, hasheq)
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
55 }
56
6a0720b Brian Anderson Convert impls to new syntax
brson authored
57 impl ~str: hash_key {
97452c0 Niko Matsakis Remove modes from map API and replace with regions.
nikomatsakis authored
58 pure fn hash() -> uint { str::hash(&self) }
59 pure fn eq(&&x: ~str) -> bool { self == x }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
60 }
61
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
62 // These used to be in task, but they disappeard.
9c6890f Brian Anderson Convert more core types to camel case
brson authored
63 type joinable_task = Port<()>;
d1fc2b5 Brian Anderson Convert to new closure syntax
brson authored
64 fn spawn_joinable(+f: fn~()) -> joinable_task {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
65 let p = port();
66 let c = chan(p);
d1fc2b5 Brian Anderson Convert to new closure syntax
brson authored
67 do task::spawn() |move f| {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
68 f();
69 c.send(());
70 }
71 p
72 }
73
74 fn join(t: joinable_task) {
75 t.recv()
76 }
640886c Polymorphic MapReduce!
Eric Holk authored
77
1125831 Brian Anderson Convert more core types to camel case
brson authored
78 impl io::Reader: word_reader {
8337fa1 Brian Anderson Camel case the option type
brson authored
79 fn read_word() -> Option<~str> { read_word(self) }
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
80 }
81
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
82 fn file_word_reader(filename: ~str) -> word_reader {
c284b8b Graydon Hoare Start using core::path2::Path in a lot of places.
graydon authored
83 match io::file_reader(&Path(filename)) {
025d866 Brian Anderson Switch alts to use arrows
brson authored
84 result::ok(f) => { f as word_reader }
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
85 result::err(e) => { fail fmt!("%?", e) }
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
86 }
87 }
640886c Polymorphic MapReduce!
Eric Holk authored
88
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
89 fn map(f: fn~() -> word_reader, emit: map_reduce::putter<~str, int>) {
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
90 let f = f();
321fd80 Tim Chevalier Add an infinite loop construct
catamorphism authored
91 loop {
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
92 match f.read_word() {
8337fa1 Brian Anderson Camel case the option type
brson authored
93 Some(w) => { emit(w, 1); }
94 None => { break; }
640886c Polymorphic MapReduce!
Eric Holk authored
95 }
96 }
97 }
98
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
99 fn reduce(&&word: ~str, get: map_reduce::getter<int>) {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
100 let mut count = 0;
640886c Polymorphic MapReduce!
Eric Holk authored
101
8337fa1 Brian Anderson Camel case the option type
brson authored
102 loop { match get() { Some(_) => { count += 1; } None => { break; } } }
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
103
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
104 io::println(fmt!("%s\t%?", word, count));
640886c Polymorphic MapReduce!
Eric Holk authored
105 }
106
3ab4b01 Brian Anderson Remove the class keyword
brson authored
107 struct box<T> {
8337fa1 Brian Anderson Camel case the option type
brson authored
108 let mut contents: Option<T>;
109 new(+x: T) { self.contents = Some(x); }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
110
111 fn swap(f: fn(+T) -> T) {
8337fa1 Brian Anderson Camel case the option type
brson authored
112 let mut tmp = None;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
113 self.contents <-> tmp;
8337fa1 Brian Anderson Camel case the option type
brson authored
114 self.contents = Some(f(option::unwrap(tmp)));
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
115 }
116
117 fn unwrap() -> T {
8337fa1 Brian Anderson Camel case the option type
brson authored
118 let mut tmp = None;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
119 self.contents <-> tmp;
120 option::unwrap(tmp)
121 }
122 }
123
640886c Polymorphic MapReduce!
Eric Holk authored
124 mod map_reduce {
125 export putter;
126 export getter;
127 export mapper;
128 export reducer;
129 export map_reduce;
130
60ae159 Marijn Haverbeke Switch to new param kind bound syntax
marijnh authored
131 type putter<K: send, V: send> = fn(K, V);
640886c Polymorphic MapReduce!
Eric Holk authored
132
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
133 type mapper<K1: send, K2: send, V: send> = fn~(K1, putter<K2, V>);
640886c Polymorphic MapReduce!
Eric Holk authored
134
8337fa1 Brian Anderson Camel case the option type
brson authored
135 type getter<V: send> = fn() -> Option<V>;
640886c Polymorphic MapReduce!
Eric Holk authored
136
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
137 type reducer<K: copy send, V: copy send> = fn~(K, getter<V>);
640886c Polymorphic MapReduce!
Eric Holk authored
138
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
139 enum ctrl_proto<K: copy send, V: copy send> {
9c6890f Brian Anderson Convert more core types to camel case
brson authored
140 find_reducer(K, Chan<Chan<reduce_proto<V>>>),
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
141 mapper_done
640886c Polymorphic MapReduce!
Eric Holk authored
142 }
143
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
144
77e83d8 Paul Stansifer Change calls of `proto!` to use parens.
paulstansifer authored
145 proto! ctrl_proto (
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
146 open: send<K: copy send, V: copy send> {
147 find_reducer(K) -> reducer_response<K, V>,
6748f78 Eric Holk Polymorphic protocols work well enough to do MapReduce.
eholk authored
148 mapper_done -> !
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
149 }
150
151 reducer_response: recv<K: copy send, V: copy send> {
9c6890f Brian Anderson Convert more core types to camel case
brson authored
152 reducer(Chan<reduce_proto<V>>) -> open<K, V>
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
153 }
77e83d8 Paul Stansifer Change calls of `proto!` to use parens.
paulstansifer authored
154 )
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
155
c5437c0 Brian Anderson Fix failing tests
brson authored
156 enum reduce_proto<V: copy send> { emit_val(V), done, addref, release }
640886c Polymorphic MapReduce!
Eric Holk authored
157
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
158 fn start_mappers<K1: copy send, K2: const copy send hash_key,
159 V: copy send>(
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
160 map: mapper<K1, K2, V>,
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
161 &ctrls: ~[ctrl_proto::server::open<K2, V>],
162 inputs: ~[K1])
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
163 -> ~[joinable_task]
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
164 {
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
165 let mut tasks = ~[];
d1fc2b5 Brian Anderson Convert to new closure syntax
brson authored
166 for inputs.each |i| {
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
167 let (ctrl, ctrl_server) = ctrl_proto::init();
168 let ctrl = box(ctrl);
bf88ff5 Eric Holk Remove slow vec+=, and make word-count difficulty harder.
eholk authored
169 vec::push(tasks, spawn_joinable(|| map_task(map, ctrl, i) ));
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
170 vec::push(ctrls, ctrl_server);
640886c Polymorphic MapReduce!
Eric Holk authored
171 }
b355936 Brian Anderson Convert ret to return
brson authored
172 return tasks;
640886c Polymorphic MapReduce!
Eric Holk authored
173 }
174
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
175 fn map_task<K1: copy send, K2: const copy send hash_key, V: copy send>(
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
176 map: mapper<K1, K2, V>,
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
177 ctrl: box<ctrl_proto::client::open<K2, V>>,
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
178 input: K1)
179 {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
180 // log(error, "map_task " + input);
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
181 let intermediates = mk_hash();
640886c Polymorphic MapReduce!
Eric Holk authored
182
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
183 do map(input) |key, val| {
8337fa1 Brian Anderson Camel case the option type
brson authored
184 let mut c = None;
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
185 match intermediates.find(key) {
8337fa1 Brian Anderson Camel case the option type
brson authored
186 Some(_c) => { c = Some(_c); }
187 None => {
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
188 do ctrl.swap |ctrl| {
189 let ctrl = ctrl_proto::client::find_reducer(ctrl, key);
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
190 match pipes::recv(ctrl) {
025d866 Brian Anderson Switch alts to use arrows
brson authored
191 ctrl_proto::reducer(c_, ctrl) => {
8337fa1 Brian Anderson Camel case the option type
brson authored
192 c = Some(c_);
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
193 move_out!(ctrl)
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
194 }
195 }
196 }
197 intermediates.insert(key, c.get());
c5437c0 Brian Anderson Fix failing tests
brson authored
198 send(c.get(), addref);
640886c Polymorphic MapReduce!
Eric Holk authored
199 }
200 }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
201 send(c.get(), emit_val(val));
640886c Polymorphic MapReduce!
Eric Holk authored
202 }
203
9c6890f Brian Anderson Convert more core types to camel case
brson authored
204 fn finish<K: copy send, V: copy send>(_k: K, v: Chan<reduce_proto<V>>)
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
205 {
640886c Polymorphic MapReduce!
Eric Holk authored
206 send(v, release);
207 }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
208 for intermediates.each_value |v| { send(v, release) }
209 ctrl_proto::client::mapper_done(ctrl.unwrap());
640886c Polymorphic MapReduce!
Eric Holk authored
210 }
211
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
212 fn reduce_task<K: copy send, V: copy send>(
213 reduce: reducer<K, V>,
214 key: K,
9c6890f Brian Anderson Convert more core types to camel case
brson authored
215 out: Chan<Chan<reduce_proto<V>>>)
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
216 {
640886c Polymorphic MapReduce!
Eric Holk authored
217 let p = port();
218
219 send(out, chan(p));
220
4dcf84e Brian Anderson Remove bind. Issue #2189
brson authored
221 let mut ref_count = 0;
222 let mut is_done = false;
640886c Polymorphic MapReduce!
Eric Holk authored
223
9c6890f Brian Anderson Convert more core types to camel case
brson authored
224 fn get<V: copy send>(p: Port<reduce_proto<V>>,
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
225 &ref_count: int, &is_done: bool)
8337fa1 Brian Anderson Camel case the option type
brson authored
226 -> Option<V> {
640886c Polymorphic MapReduce!
Eric Holk authored
227 while !is_done || ref_count > 0 {
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
228 match recv(p) {
025d866 Brian Anderson Switch alts to use arrows
brson authored
229 emit_val(v) => {
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
230 // error!("received %d", v);
8337fa1 Brian Anderson Camel case the option type
brson authored
231 return Some(v);
640886c Polymorphic MapReduce!
Eric Holk authored
232 }
025d866 Brian Anderson Switch alts to use arrows
brson authored
233 done => {
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
234 // error!("all done");
640886c Polymorphic MapReduce!
Eric Holk authored
235 is_done = true;
236 }
025d866 Brian Anderson Switch alts to use arrows
brson authored
237 addref => { ref_count += 1; }
238 release => { ref_count -= 1; }
640886c Polymorphic MapReduce!
Eric Holk authored
239 }
240 }
8337fa1 Brian Anderson Camel case the option type
brson authored
241 return None;
640886c Polymorphic MapReduce!
Eric Holk authored
242 }
243
d1fc2b5 Brian Anderson Convert to new closure syntax
brson authored
244 reduce(key, || get(p, ref_count, is_done) );
640886c Polymorphic MapReduce!
Eric Holk authored
245 }
246
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
247 fn map_reduce<K1: copy send, K2: const copy send hash_key, V: copy send>(
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
248 map: mapper<K1, K2, V>,
249 reduce: reducer<K2, V>,
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
250 inputs: ~[K1])
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
251 {
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
252 let mut ctrl = ~[];
640886c Polymorphic MapReduce!
Eric Holk authored
253
254 // This task becomes the master control task. It task::_spawns
255 // to do the rest.
256
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
257 let reducers = mk_hash();
258 let mut tasks = start_mappers(map, ctrl, inputs);
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
259 let mut num_mappers = vec::len(inputs) as int;
640886c Polymorphic MapReduce!
Eric Holk authored
260
261 while num_mappers > 0 {
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
262 let (_ready, message, ctrls) = pipes::select(ctrl);
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
263 match option::unwrap(message) {
025d866 Brian Anderson Switch alts to use arrows
brson authored
264 ctrl_proto::mapper_done => {
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
265 // error!("received mapper terminated.");
640886c Polymorphic MapReduce!
Eric Holk authored
266 num_mappers -= 1;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
267 ctrl = ctrls;
640886c Polymorphic MapReduce!
Eric Holk authored
268 }
025d866 Brian Anderson Switch alts to use arrows
brson authored
269 ctrl_proto::find_reducer(k, cc) => {
640886c Polymorphic MapReduce!
Eric Holk authored
270 let c;
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
271 // log(error, "finding reducer for " + k);
ecaf9e3 Brian Anderson Convert alt to match. Stop parsing alt
brson authored
272 match reducers.find(k) {
8337fa1 Brian Anderson Camel case the option type
brson authored
273 Some(_c) => {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
274 // log(error,
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
275 // "reusing existing reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
276 c = _c;
277 }
8337fa1 Brian Anderson Camel case the option type
brson authored
278 None => {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
279 // log(error, "creating new reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
280 let p = port();
a1ef79c Niko Matsakis update to use new spawn syntax
nikomatsakis authored
281 let ch = chan(p);
640886c Polymorphic MapReduce!
Eric Holk authored
282 let r = reduce, kk = k;
bf88ff5 Eric Holk Remove slow vec+=, and make word-count difficulty harder.
eholk authored
283 vec::push(tasks,
284 spawn_joinable(|| reduce_task(r, kk, ch) ));
640886c Polymorphic MapReduce!
Eric Holk authored
285 c = recv(p);
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
286 reducers.insert(k, c);
640886c Polymorphic MapReduce!
Eric Holk authored
287 }
288 }
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
289 ctrl = vec::append_one(
290 ctrls,
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
291 ctrl_proto::server::reducer(move_out!(cc), c));
640886c Polymorphic MapReduce!
Eric Holk authored
292 }
293 }
294 }
295
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
296 for reducers.each_value |v| { send(v, done) }
640886c Polymorphic MapReduce!
Eric Holk authored
297
d1fc2b5 Brian Anderson Convert to new closure syntax
brson authored
298 for tasks.each |t| { join(t); }
640886c Polymorphic MapReduce!
Eric Holk authored
299 }
300 }
301
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
302 fn main(argv: ~[~str]) {
303 if vec::len(argv) < 2u && !os::getenv(~"RUST_BENCH").is_some() {
640886c Polymorphic MapReduce!
Eric Holk authored
304 let out = io::stdout();
305
29f32b4 Paul Stansifer `m1!{...}` -> `m1!(...)`
paulstansifer authored
306 out.write_line(fmt!("Usage: %s <filename> ...", argv[0]));
640886c Polymorphic MapReduce!
Eric Holk authored
307
b355936 Brian Anderson Convert ret to return
brson authored
308 return;
640886c Polymorphic MapReduce!
Eric Holk authored
309 }
310
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
311 let readers: ~[fn~() -> word_reader] = if argv.len() >= 2 {
65beca4 Eric Holk Use iteration protocol for ebml, use vec::view in more places (issue #28...
eholk authored
312 vec::view(argv, 1u, argv.len()).map(
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
313 |f| fn~() -> word_reader { file_word_reader(f) } )
314 }
315 else {
316 let num_readers = 50;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
317 let words_per_reader = 600;
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
318 vec::from_fn(
319 num_readers,
320 |_i| fn~() -> word_reader {
321 random_word_reader(words_per_reader) as word_reader
322 })
323 };
324
640886c Polymorphic MapReduce!
Eric Holk authored
325 let start = time::precise_time_ns();
326
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
327 map_reduce::map_reduce(map, reduce, readers);
640886c Polymorphic MapReduce!
Eric Holk authored
328 let stop = time::precise_time_ns();
329
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
330 let elapsed = (stop - start) / 1000000u64;
640886c Polymorphic MapReduce!
Eric Holk authored
331
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
332 log(error, ~"MapReduce completed in "
333 + u64::str(elapsed) + ~"ms");
640886c Polymorphic MapReduce!
Eric Holk authored
334 }
335
8337fa1 Brian Anderson Camel case the option type
brson authored
336 fn read_word(r: io::Reader) -> Option<~str> {
92743dc Michael J. Sullivan Move the world over to using the new style string literals and types. Cl...
msullivan authored
337 let mut w = ~"";
640886c Polymorphic MapReduce!
Eric Holk authored
338
339 while !r.eof() {
340 let c = r.read_char();
341
342 if is_word_char(c) {
ab6bb03 Brian Anderson Rename std::istr to std::str. Issue #855
brson authored
343 w += str::from_char(c);
8337fa1 Brian Anderson Camel case the option type
brson authored
344 } else { if w != ~"" { return Some(w); } }
640886c Polymorphic MapReduce!
Eric Holk authored
345 }
8337fa1 Brian Anderson Camel case the option type
brson authored
346 return None;
640886c Polymorphic MapReduce!
Eric Holk authored
347 }
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
348
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
349 fn is_word_char(c: char) -> bool {
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
350 char::is_alphabetic(c) || char::is_digit(c) || c == '_'
351 }
352
3ab4b01 Brian Anderson Remove the class keyword
brson authored
353 struct random_word_reader: word_reader {
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
354 let mut remaining: uint;
1125831 Brian Anderson Convert more core types to camel case
brson authored
355 let rng: rand::Rng;
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
356 new(count: uint) {
357 self.remaining = count;
358 self.rng = rand::rng();
359 }
360
8337fa1 Brian Anderson Camel case the option type
brson authored
361 fn read_word() -> Option<~str> {
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
362 if self.remaining > 0 {
363 self.remaining -= 1;
1a276db Eric Holk Switch map-reduce control protocol to use pipes. This exposed a bug in t...
eholk authored
364 let len = self.rng.gen_uint_range(1, 4);
8337fa1 Brian Anderson Camel case the option type
brson authored
365 Some(self.rng.gen_str(len))
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
366 }
8337fa1 Brian Anderson Camel case the option type
brson authored
367 else { None }
a00478b Eric Holk word-count-generic now generates random words in benchmark mode.
eholk authored
368 }
369 }
Something went wrong with that request. Please try again.