Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 370 lines (305 sloc) 9.741 kb
640886c Polymorphic MapReduce!
Eric Holk authored
1 /**
2 A parallel word-frequency counting program.
3
4 This is meant primarily to demonstrate Rust's MapReduce framework.
5
6 It takes a list of files on the command line and outputs a list of
7 words along with how many times each word is used.
8
9 */
10
05543fd @eholk Make tests pass
eholk authored
11 // xfail-pretty
12
640886c Polymorphic MapReduce!
Eric Holk authored
13 use std;
14
e5d095d @catamorphism Change option::t to option
catamorphism authored
15 import option = option;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
16 import option::some;
17 import option::none;
18 import str;
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
19 import std::map;
20 import std::map::hashmap;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
21 import vec;
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
22 import io;
1125831 @brson Convert more core types to camel case
brson authored
23 import io::WriterUtil;
640886c Polymorphic MapReduce!
Eric Holk authored
24
25 import std::time;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
26 import u64;
27
28 import task;
29 import comm;
9c6890f @brson Convert more core types to camel case
brson authored
30 import comm::Chan;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
31 import comm::chan;
9c6890f @brson Convert more core types to camel case
brson authored
32 import comm::Port;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
33 import comm::port;
34 import comm::recv;
35 import comm::send;
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
36
c5437c0 @brson Fix failing tests
brson authored
37 macro_rules! move_out {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
38 { $x:expr } => { unsafe { let y <- *ptr::addr_of($x); y } }
39 }
40
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
41 trait word_reader {
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
42 fn read_word() -> option<~str>;
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
43 }
44
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
45 trait hash_key {
97452c0 @nikomatsakis Remove modes from map API and replace with regions.
nikomatsakis authored
46 pure fn hash() -> uint;
47 pure fn eq(&&k: self) -> bool;
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
48 }
49
50 fn mk_hash<K: const hash_key, V: copy>() -> map::hashmap<K, V> {
97452c0 @nikomatsakis Remove modes from map API and replace with regions.
nikomatsakis authored
51 pure fn hashfn<K: const hash_key>(k: &K) -> uint { k.hash() }
52 pure fn hasheq<K: const hash_key>(k1: &K, k2: &K) -> bool { k1.eq(*k2) }
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
53
97452c0 @nikomatsakis Remove modes from map API and replace with regions.
nikomatsakis authored
54 map::hashmap(hashfn, hasheq)
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
55 }
56
6a0720b @brson Convert impls to new syntax
brson authored
57 impl ~str: hash_key {
97452c0 @nikomatsakis Remove modes from map API and replace with regions.
nikomatsakis authored
58 pure fn hash() -> uint { str::hash(&self) }
59 pure fn eq(&&x: ~str) -> bool { self == x }
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
60 }
61
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
62 // These used to be in task, but they disappeard.
9c6890f @brson Convert more core types to camel case
brson authored
63 type joinable_task = Port<()>;
d1fc2b5 @brson Convert to new closure syntax
brson authored
64 fn spawn_joinable(+f: fn~()) -> joinable_task {
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
65 let p = port();
66 let c = chan(p);
d1fc2b5 @brson Convert to new closure syntax
brson authored
67 do task::spawn() |move f| {
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
68 f();
69 c.send(());
70 }
71 p
72 }
73
74 fn join(t: joinable_task) {
75 t.recv()
76 }
640886c Polymorphic MapReduce!
Eric Holk authored
77
1125831 @brson Convert more core types to camel case
brson authored
78 impl io::Reader: word_reader {
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
79 fn read_word() -> option<~str> { read_word(self) }
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
80 }
81
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
82 fn file_word_reader(filename: ~str) -> word_reader {
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
83 match io::file_reader(filename) {
025d866 @brson Switch alts to use arrows
brson authored
84 result::ok(f) => { f as word_reader }
85 result::err(e) => { fail fmt!{"%?", e} }
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
86 }
87 }
640886c Polymorphic MapReduce!
Eric Holk authored
88
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
89 fn map(f: fn~() -> word_reader, emit: map_reduce::putter<~str, int>) {
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
90 let f = f();
321fd80 @catamorphism Add an infinite loop construct
catamorphism authored
91 loop {
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
92 match f.read_word() {
025d866 @brson Switch alts to use arrows
brson authored
93 some(w) => { emit(w, 1); }
94 none => { break; }
640886c Polymorphic MapReduce!
Eric Holk authored
95 }
96 }
97 }
98
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
99 fn reduce(&&word: ~str, get: map_reduce::getter<int>) {
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
100 let mut count = 0;
640886c Polymorphic MapReduce!
Eric Holk authored
101
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
102 loop { match get() { some(_) => { count += 1; } none => { break; } } }
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
103
a9cc506 @paulstansifer Change syntax extension syntax: `#m[...]` -> `m!{...}`.
paulstansifer authored
104 io::println(fmt!{"%s\t%?", word, count});
640886c Polymorphic MapReduce!
Eric Holk authored
105 }
106
3ab4b01 @brson Remove the class keyword
brson authored
107 struct box<T> {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
108 let mut contents: option<T>;
109 new(+x: T) { self.contents = some(x); }
110
111 fn swap(f: fn(+T) -> T) {
112 let mut tmp = none;
113 self.contents <-> tmp;
114 self.contents = some(f(option::unwrap(tmp)));
115 }
116
117 fn unwrap() -> T {
118 let mut tmp = none;
119 self.contents <-> tmp;
120 option::unwrap(tmp)
121 }
122 }
123
640886c Polymorphic MapReduce!
Eric Holk authored
124 mod map_reduce {
125 export putter;
126 export getter;
127 export mapper;
128 export reducer;
129 export map_reduce;
130
60ae159 @marijnh Switch to new param kind bound syntax
marijnh authored
131 type putter<K: send, V: send> = fn(K, V);
640886c Polymorphic MapReduce!
Eric Holk authored
132
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
133 type mapper<K1: send, K2: send, V: send> = fn~(K1, putter<K2, V>);
640886c Polymorphic MapReduce!
Eric Holk authored
134
60ae159 @marijnh Switch to new param kind bound syntax
marijnh authored
135 type getter<V: send> = fn() -> option<V>;
640886c Polymorphic MapReduce!
Eric Holk authored
136
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
137 type reducer<K: copy send, V: copy send> = fn~(K, getter<V>);
640886c Polymorphic MapReduce!
Eric Holk authored
138
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
139 enum ctrl_proto<K: copy send, V: copy send> {
9c6890f @brson Convert more core types to camel case
brson authored
140 find_reducer(K, Chan<Chan<reduce_proto<V>>>),
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
141 mapper_done
640886c Polymorphic MapReduce!
Eric Holk authored
142 }
143
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
144
145 proto! ctrl_proto {
146 open: send<K: copy send, V: copy send> {
147 find_reducer(K) -> reducer_response<K, V>,
6748f78 @eholk Polymorphic protocols work well enough to do MapReduce.
eholk authored
148 mapper_done -> !
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
149 }
150
151 reducer_response: recv<K: copy send, V: copy send> {
9c6890f @brson Convert more core types to camel case
brson authored
152 reducer(Chan<reduce_proto<V>>) -> open<K, V>
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
153 }
154 }
155
c5437c0 @brson Fix failing tests
brson authored
156 enum reduce_proto<V: copy send> { emit_val(V), done, addref, release }
640886c Polymorphic MapReduce!
Eric Holk authored
157
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
158 fn start_mappers<K1: copy send, K2: const copy send hash_key,
159 V: copy send>(
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
160 map: mapper<K1, K2, V>,
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
161 &ctrls: ~[ctrl_proto::server::open<K2, V>],
162 inputs: ~[K1])
98e161f @msullivan Switch the compiler over to using ~[] notation instead of []/~. Close…
msullivan authored
163 -> ~[joinable_task]
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
164 {
98e161f @msullivan Switch the compiler over to using ~[] notation instead of []/~. Close…
msullivan authored
165 let mut tasks = ~[];
d1fc2b5 @brson Convert to new closure syntax
brson authored
166 for inputs.each |i| {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
167 let (ctrl, ctrl_server) = ctrl_proto::init();
168 let ctrl = box(ctrl);
bf88ff5 @eholk Remove slow vec+=, and make word-count difficulty harder.
eholk authored
169 vec::push(tasks, spawn_joinable(|| map_task(map, ctrl, i) ));
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
170 vec::push(ctrls, ctrl_server);
640886c Polymorphic MapReduce!
Eric Holk authored
171 }
b355936 @brson Convert ret to return
brson authored
172 return tasks;
640886c Polymorphic MapReduce!
Eric Holk authored
173 }
174
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
175 fn map_task<K1: copy send, K2: const copy send hash_key, V: copy send>(
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
176 map: mapper<K1, K2, V>,
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
177 ctrl: box<ctrl_proto::client::open<K2, V>>,
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
178 input: K1)
179 {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
180 // log(error, "map_task " + input);
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
181 let intermediates = mk_hash();
640886c Polymorphic MapReduce!
Eric Holk authored
182
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
183 do map(input) |key, val| {
184 let mut c = none;
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
185 match intermediates.find(key) {
025d866 @brson Switch alts to use arrows
brson authored
186 some(_c) => { c = some(_c); }
187 none => {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
188 do ctrl.swap |ctrl| {
189 let ctrl = ctrl_proto::client::find_reducer(ctrl, key);
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
190 match pipes::recv(ctrl) {
025d866 @brson Switch alts to use arrows
brson authored
191 ctrl_proto::reducer(c_, ctrl) => {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
192 c = some(c_);
c5437c0 @brson Fix failing tests
brson authored
193 move_out!{ctrl}
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
194 }
195 }
196 }
197 intermediates.insert(key, c.get());
c5437c0 @brson Fix failing tests
brson authored
198 send(c.get(), addref);
640886c Polymorphic MapReduce!
Eric Holk authored
199 }
200 }
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
201 send(c.get(), emit_val(val));
640886c Polymorphic MapReduce!
Eric Holk authored
202 }
203
9c6890f @brson Convert more core types to camel case
brson authored
204 fn finish<K: copy send, V: copy send>(_k: K, v: Chan<reduce_proto<V>>)
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
205 {
640886c Polymorphic MapReduce!
Eric Holk authored
206 send(v, release);
207 }
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
208 for intermediates.each_value |v| { send(v, release) }
209 ctrl_proto::client::mapper_done(ctrl.unwrap());
640886c Polymorphic MapReduce!
Eric Holk authored
210 }
211
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
212 fn reduce_task<K: copy send, V: copy send>(
213 reduce: reducer<K, V>,
214 key: K,
9c6890f @brson Convert more core types to camel case
brson authored
215 out: Chan<Chan<reduce_proto<V>>>)
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
216 {
640886c Polymorphic MapReduce!
Eric Holk authored
217 let p = port();
218
219 send(out, chan(p));
220
4dcf84e @brson Remove bind. Issue #2189
brson authored
221 let mut ref_count = 0;
222 let mut is_done = false;
640886c Polymorphic MapReduce!
Eric Holk authored
223
9c6890f @brson Convert more core types to camel case
brson authored
224 fn get<V: copy send>(p: Port<reduce_proto<V>>,
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
225 &ref_count: int, &is_done: bool)
ca1df2b @marijnh Pretty-print for new arg-mode syntax
marijnh authored
226 -> option<V> {
640886c Polymorphic MapReduce!
Eric Holk authored
227 while !is_done || ref_count > 0 {
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
228 match recv(p) {
025d866 @brson Switch alts to use arrows
brson authored
229 emit_val(v) => {
a9cc506 @paulstansifer Change syntax extension syntax: `#m[...]` -> `m!{...}`.
paulstansifer authored
230 // error!{"received %d", v};
b355936 @brson Convert ret to return
brson authored
231 return some(v);
640886c Polymorphic MapReduce!
Eric Holk authored
232 }
025d866 @brson Switch alts to use arrows
brson authored
233 done => {
a9cc506 @paulstansifer Change syntax extension syntax: `#m[...]` -> `m!{...}`.
paulstansifer authored
234 // error!{"all done"};
640886c Polymorphic MapReduce!
Eric Holk authored
235 is_done = true;
236 }
025d866 @brson Switch alts to use arrows
brson authored
237 addref => { ref_count += 1; }
238 release => { ref_count -= 1; }
640886c Polymorphic MapReduce!
Eric Holk authored
239 }
240 }
b355936 @brson Convert ret to return
brson authored
241 return none;
640886c Polymorphic MapReduce!
Eric Holk authored
242 }
243
d1fc2b5 @brson Convert to new closure syntax
brson authored
244 reduce(key, || get(p, ref_count, is_done) );
640886c Polymorphic MapReduce!
Eric Holk authored
245 }
246
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
247 fn map_reduce<K1: copy send, K2: const copy send hash_key, V: copy send>(
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
248 map: mapper<K1, K2, V>,
249 reduce: reducer<K2, V>,
98e161f @msullivan Switch the compiler over to using ~[] notation instead of []/~. Close…
msullivan authored
250 inputs: ~[K1])
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
251 {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
252 let mut ctrl = ~[];
640886c Polymorphic MapReduce!
Eric Holk authored
253
254 // This task becomes the master control task. It task::_spawns
255 // to do the rest.
256
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
257 let reducers = mk_hash();
258 let mut tasks = start_mappers(map, ctrl, inputs);
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
259 let mut num_mappers = vec::len(inputs) as int;
640886c Polymorphic MapReduce!
Eric Holk authored
260
261 while num_mappers > 0 {
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
262 let (_ready, message, ctrls) = pipes::select(ctrl);
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
263 match option::unwrap(message) {
025d866 @brson Switch alts to use arrows
brson authored
264 ctrl_proto::mapper_done => {
a9cc506 @paulstansifer Change syntax extension syntax: `#m[...]` -> `m!{...}`.
paulstansifer authored
265 // error!{"received mapper terminated."};
640886c Polymorphic MapReduce!
Eric Holk authored
266 num_mappers -= 1;
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
267 ctrl = ctrls;
640886c Polymorphic MapReduce!
Eric Holk authored
268 }
025d866 @brson Switch alts to use arrows
brson authored
269 ctrl_proto::find_reducer(k, cc) => {
640886c Polymorphic MapReduce!
Eric Holk authored
270 let c;
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
271 // log(error, "finding reducer for " + k);
ecaf9e3 @brson Convert alt to match. Stop parsing alt
brson authored
272 match reducers.find(k) {
025d866 @brson Switch alts to use arrows
brson authored
273 some(_c) => {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
274 // log(error,
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
275 // "reusing existing reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
276 c = _c;
277 }
025d866 @brson Switch alts to use arrows
brson authored
278 none => {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
279 // log(error, "creating new reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
280 let p = port();
a1ef79c @nikomatsakis update to use new spawn syntax
nikomatsakis authored
281 let ch = chan(p);
640886c Polymorphic MapReduce!
Eric Holk authored
282 let r = reduce, kk = k;
bf88ff5 @eholk Remove slow vec+=, and make word-count difficulty harder.
eholk authored
283 vec::push(tasks,
284 spawn_joinable(|| reduce_task(r, kk, ch) ));
640886c Polymorphic MapReduce!
Eric Holk authored
285 c = recv(p);
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
286 reducers.insert(k, c);
640886c Polymorphic MapReduce!
Eric Holk authored
287 }
288 }
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
289 ctrl = vec::append_one(
290 ctrls,
c5437c0 @brson Fix failing tests
brson authored
291 ctrl_proto::server::reducer(move_out!{cc}, c));
640886c Polymorphic MapReduce!
Eric Holk authored
292 }
293 }
294 }
295
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
296 for reducers.each_value |v| { send(v, done) }
640886c Polymorphic MapReduce!
Eric Holk authored
297
d1fc2b5 @brson Convert to new closure syntax
brson authored
298 for tasks.each |t| { join(t); }
640886c Polymorphic MapReduce!
Eric Holk authored
299 }
300 }
301
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
302 fn main(argv: ~[~str]) {
303 if vec::len(argv) < 2u && !os::getenv(~"RUST_BENCH").is_some() {
640886c Polymorphic MapReduce!
Eric Holk authored
304 let out = io::stdout();
305
a9cc506 @paulstansifer Change syntax extension syntax: `#m[...]` -> `m!{...}`.
paulstansifer authored
306 out.write_line(fmt!{"Usage: %s <filename> ...", argv[0]});
640886c Polymorphic MapReduce!
Eric Holk authored
307
b355936 @brson Convert ret to return
brson authored
308 return;
640886c Polymorphic MapReduce!
Eric Holk authored
309 }
310
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
311 let readers: ~[fn~() -> word_reader] = if argv.len() >= 2 {
65beca4 @eholk Use iteration protocol for ebml, use vec::view in more places (issue …
eholk authored
312 vec::view(argv, 1u, argv.len()).map(
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
313 |f| fn~() -> word_reader { file_word_reader(f) } )
314 }
315 else {
316 let num_readers = 50;
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
317 let words_per_reader = 600;
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
318 vec::from_fn(
319 num_readers,
320 |_i| fn~() -> word_reader {
321 random_word_reader(words_per_reader) as word_reader
322 })
323 };
324
640886c Polymorphic MapReduce!
Eric Holk authored
325 let start = time::precise_time_ns();
326
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
327 map_reduce::map_reduce(map, reduce, readers);
640886c Polymorphic MapReduce!
Eric Holk authored
328 let stop = time::precise_time_ns();
329
0c3a128 @eholk Update word-count-generic to latest syntax and un-xfail it. Closes #1…
eholk authored
330 let elapsed = (stop - start) / 1000000u64;
640886c Polymorphic MapReduce!
Eric Holk authored
331
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
332 log(error, ~"MapReduce completed in "
333 + u64::str(elapsed) + ~"ms");
640886c Polymorphic MapReduce!
Eric Holk authored
334 }
335
1125831 @brson Convert more core types to camel case
brson authored
336 fn read_word(r: io::Reader) -> option<~str> {
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
337 let mut w = ~"";
640886c Polymorphic MapReduce!
Eric Holk authored
338
339 while !r.eof() {
340 let c = r.read_char();
341
342 if is_word_char(c) {
ab6bb03 @brson Rename std::istr to std::str. Issue #855
brson authored
343 w += str::from_char(c);
b355936 @brson Convert ret to return
brson authored
344 } else { if w != ~"" { return some(w); } }
640886c Polymorphic MapReduce!
Eric Holk authored
345 }
b355936 @brson Convert ret to return
brson authored
346 return none;
640886c Polymorphic MapReduce!
Eric Holk authored
347 }
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
348
df83a79 @eholk In generic word count, use str instead of [u8], and use built in is_a…
eholk authored
349 fn is_word_char(c: char) -> bool {
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
350 char::is_alphabetic(c) || char::is_digit(c) || c == '_'
351 }
352
3ab4b01 @brson Remove the class keyword
brson authored
353 struct random_word_reader: word_reader {
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
354 let mut remaining: uint;
1125831 @brson Convert more core types to camel case
brson authored
355 let rng: rand::Rng;
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
356 new(count: uint) {
357 self.remaining = count;
358 self.rng = rand::rng();
359 }
360
92743dc @msullivan Move the world over to using the new style string literals and types.…
msullivan authored
361 fn read_word() -> option<~str> {
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
362 if self.remaining > 0 {
363 self.remaining -= 1;
1a276db @eholk Switch map-reduce control protocol to use pipes. This exposed a bug i…
eholk authored
364 let len = self.rng.gen_uint_range(1, 4);
365 some(self.rng.gen_str(len))
a00478b @eholk word-count-generic now generates random words in benchmark mode.
eholk authored
366 }
367 else { none }
368 }
369 }
Something went wrong with that request. Please try again.