Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 269 lines (220 sloc) 6.991 kb
640886c Polymorphic MapReduce!
Eric Holk authored
1 /**
2 A parallel word-frequency counting program.
3
4 This is meant primarily to demonstrate Rust's MapReduce framework.
5
6 It takes a list of files on the command line and outputs a list of
7 words along with how many times each word is used.
8
9 */
10
11 use std;
12
e5d095d Tim Chevalier Change option::t to option
catamorphism authored
13 import option = option;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
14 import option::some;
15 import option::none;
16 import str;
640886c Polymorphic MapReduce!
Eric Holk authored
17 import std::treemap;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
18 import vec;
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
19 import io;
20 import io::{reader_util, writer_util};
640886c Polymorphic MapReduce!
Eric Holk authored
21
22 import std::time;
fa9ad98 Graydon Hoare Copy first batch of material from libstd to libcore.
graydon authored
23 import u64;
24
25 import task;
26 import comm;
27 import comm::chan;
28 import comm::port;
29 import comm::recv;
30 import comm::send;
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
31 import comm::methods;
32
33 // These used to be in task, but they disappeard.
34 type joinable_task = port<()>;
35 fn spawn_joinable(f: fn~()) -> joinable_task {
36 let p = port();
37 let c = chan(p);
38 task::spawn() {||
39 f();
40 c.send(());
41 }
42 p
43 }
44
45 fn join(t: joinable_task) {
46 t.recv()
47 }
640886c Polymorphic MapReduce!
Eric Holk authored
48
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
49 fn map(&&filename: str, emit: map_reduce::putter<str, int>) {
50 let f = alt io::file_reader(filename) {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
51 result::ok(f) { f }
52 result::err(e) { fail #fmt("%?", e) }
53 };
640886c Polymorphic MapReduce!
Eric Holk authored
54
321fd80 Tim Chevalier Add an infinite loop construct
catamorphism authored
55 loop {
640886c Polymorphic MapReduce!
Eric Holk authored
56 alt read_word(f) {
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
57 some(w) { emit(w, 1); }
04a2887 Tim Chevalier Remove '.' after nullary tags in patterns
catamorphism authored
58 none { break; }
640886c Polymorphic MapReduce!
Eric Holk authored
59 }
60 }
61 }
62
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
63 fn reduce(&&word: str, get: map_reduce::getter<int>) {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
64 let mut count = 0;
640886c Polymorphic MapReduce!
Eric Holk authored
65
321fd80 Tim Chevalier Add an infinite loop construct
catamorphism authored
66 loop { alt get() { some(_) { count += 1; } none { break; } } }
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
67
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
68 io::println(#fmt("%s\t%?", word, count));
640886c Polymorphic MapReduce!
Eric Holk authored
69 }
70
71 mod map_reduce {
72 export putter;
73 export getter;
74 export mapper;
75 export reducer;
76 export map_reduce;
77
60ae159 Marijn Haverbeke Switch to new param kind bound syntax
marijnh authored
78 type putter<K: send, V: send> = fn(K, V);
640886c Polymorphic MapReduce!
Eric Holk authored
79
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
80 type mapper<K1: send, K2: send, V: send> = fn~(K1, putter<K2, V>);
640886c Polymorphic MapReduce!
Eric Holk authored
81
60ae159 Marijn Haverbeke Switch to new param kind bound syntax
marijnh authored
82 type getter<V: send> = fn() -> option<V>;
640886c Polymorphic MapReduce!
Eric Holk authored
83
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
84 type reducer<K: copy send, V: copy send> = fn~(K, getter<V>);
640886c Polymorphic MapReduce!
Eric Holk authored
85
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
86 enum ctrl_proto<K: copy send, V: copy send> {
87 find_reducer(K, chan<chan<reduce_proto<V>>>),
88 mapper_done
640886c Polymorphic MapReduce!
Eric Holk authored
89 }
90
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
91 enum reduce_proto<V: copy send> { emit_val(V), done, ref, release }
640886c Polymorphic MapReduce!
Eric Holk authored
92
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
93 fn start_mappers<K1: copy send, K2: copy send, V: copy send>(
94 map: mapper<K1, K2, V>,
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
95 ctrl: chan<ctrl_proto<K2, V>>, inputs: ~[K1])
96 -> ~[joinable_task]
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
97 {
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
98 let mut tasks = ~[];
c902eaf Marijn Haverbeke Convert old-style for loops to new-style
marijnh authored
99 for inputs.each {|i|
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
100 tasks += ~[spawn_joinable {|| map_task(map, ctrl, i)}];
640886c Polymorphic MapReduce!
Eric Holk authored
101 }
102 ret tasks;
103 }
104
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
105 fn map_task<K1: copy send, K2: copy send, V: copy send>(
106 map: mapper<K1, K2, V>,
107 ctrl: chan<ctrl_proto<K2, V>>,
108 input: K1)
109 {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
110 // log(error, "map_task " + input);
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
111 let intermediates = treemap::treemap();
640886c Polymorphic MapReduce!
Eric Holk authored
112
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
113 fn emit<K2: copy send, V: copy send>(
114 im: treemap::treemap<K2, chan<reduce_proto<V>>>,
115 ctrl: chan<ctrl_proto<K2, V>>, key: K2, val: V)
116 {
640886c Polymorphic MapReduce!
Eric Holk authored
117 let c;
118 alt treemap::find(im, key) {
7298b8f Marijn Haverbeke Insert omitted semicolons for statements
marijnh authored
119 some(_c) { c = _c; }
04a2887 Tim Chevalier Remove '.' after nullary tags in patterns
catamorphism authored
120 none {
640886c Polymorphic MapReduce!
Eric Holk authored
121 let p = port();
122 send(ctrl, find_reducer(key, chan(p)));
123 c = recv(p);
124 treemap::insert(im, key, c);
125 send(c, ref);
126 }
127 }
128 send(c, emit_val(val));
129 }
130
4dcf84e Brian Anderson Remove bind. Issue #2189
brson authored
131 map(input, {|a,b|emit(intermediates, ctrl, a, b)});
640886c Polymorphic MapReduce!
Eric Holk authored
132
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
133 fn finish<K: copy send, V: copy send>(_k: K, v: chan<reduce_proto<V>>)
134 {
640886c Polymorphic MapReduce!
Eric Holk authored
135 send(v, release);
136 }
137 treemap::traverse(intermediates, finish);
138 send(ctrl, mapper_done);
139 }
140
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
141 fn reduce_task<K: copy send, V: copy send>(
142 reduce: reducer<K, V>,
143 key: K,
144 out: chan<chan<reduce_proto<V>>>)
145 {
640886c Polymorphic MapReduce!
Eric Holk authored
146 let p = port();
147
148 send(out, chan(p));
149
4dcf84e Brian Anderson Remove bind. Issue #2189
brson authored
150 let mut ref_count = 0;
151 let mut is_done = false;
640886c Polymorphic MapReduce!
Eric Holk authored
152
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
153 fn get<V: copy send>(p: port<reduce_proto<V>>,
154 &ref_count: int, &is_done: bool)
ca1df2b Marijn Haverbeke Pretty-print for new arg-mode syntax
marijnh authored
155 -> option<V> {
640886c Polymorphic MapReduce!
Eric Holk authored
156 while !is_done || ref_count > 0 {
157 alt recv(p) {
158 emit_val(v) {
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
159 // #error("received %d", v);
640886c Polymorphic MapReduce!
Eric Holk authored
160 ret some(v);
161 }
04a2887 Tim Chevalier Remove '.' after nullary tags in patterns
catamorphism authored
162 done {
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
163 // #error("all done");
640886c Polymorphic MapReduce!
Eric Holk authored
164 is_done = true;
165 }
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
166 ref { ref_count += 1; }
167 release { ref_count -= 1; }
640886c Polymorphic MapReduce!
Eric Holk authored
168 }
169 }
170 ret none;
171 }
172
4dcf84e Brian Anderson Remove bind. Issue #2189
brson authored
173 reduce(key, {||get(p, ref_count, is_done)});
640886c Polymorphic MapReduce!
Eric Holk authored
174 }
175
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
176 fn map_reduce<K1: copy send, K2: copy send, V: copy send>(
177 map: mapper<K1, K2, V>,
178 reduce: reducer<K2, V>,
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
179 inputs: ~[K1])
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
180 {
640886c Polymorphic MapReduce!
Eric Holk authored
181 let ctrl = port();
182
183 // This task becomes the master control task. It task::_spawns
184 // to do the rest.
185
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
186 let reducers = treemap::treemap();
187 let mut tasks = start_mappers(map, chan(ctrl), inputs);
188 let mut num_mappers = vec::len(inputs) as int;
640886c Polymorphic MapReduce!
Eric Holk authored
189
190 while num_mappers > 0 {
191 alt recv(ctrl) {
04a2887 Tim Chevalier Remove '.' after nullary tags in patterns
catamorphism authored
192 mapper_done {
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
193 // #error("received mapper terminated.");
640886c Polymorphic MapReduce!
Eric Holk authored
194 num_mappers -= 1;
195 }
196 find_reducer(k, cc) {
197 let c;
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
198 // log(error, "finding reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
199 alt treemap::find(reducers, k) {
200 some(_c) {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
201 // log(error,
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
202 // "reusing existing reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
203 c = _c;
204 }
04a2887 Tim Chevalier Remove '.' after nullary tags in patterns
catamorphism authored
205 none {
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
206 // log(error, "creating new reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
207 let p = port();
a1ef79c Niko Matsakis update to use new spawn syntax
nikomatsakis authored
208 let ch = chan(p);
640886c Polymorphic MapReduce!
Eric Holk authored
209 let r = reduce, kk = k;
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
210 tasks += ~[
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
211 spawn_joinable {|| reduce_task(r, kk, ch) }
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
212 ];
640886c Polymorphic MapReduce!
Eric Holk authored
213 c = recv(p);
214 treemap::insert(reducers, k, c);
215 }
216 }
217 send(cc, c);
218 }
219 }
220 }
221
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
222 fn finish<K: copy send, V: copy send>(_k: K, v: chan<reduce_proto<V>>)
223 {
cfdf193 Marijn Haverbeke Update our code to new type parameter kind syntax
marijnh authored
224 send(v, done);
225 }
640886c Polymorphic MapReduce!
Eric Holk authored
226 treemap::traverse(reducers, finish);
227
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
228 for tasks.each {|t| join(t); }
640886c Polymorphic MapReduce!
Eric Holk authored
229 }
230 }
231
98e161f Michael J. Sullivan Switch the compiler over to using ~[] notation instead of []/~. Closes #...
msullivan authored
232 fn main(argv: ~[str]) {
640886c Polymorphic MapReduce!
Eric Holk authored
233 if vec::len(argv) < 2u {
234 let out = io::stdout();
235
5c49e4f Brian Anderson Reformat. Issue #855
brson authored
236 out.write_line(#fmt["Usage: %s <filename> ...", argv[0]]);
640886c Polymorphic MapReduce!
Eric Holk authored
237
238 // TODO: run something just to make sure the code hasn't
239 // broken yet. This is the unit test mode of this program.
240
241 ret;
242 }
243
244 let start = time::precise_time_ns();
245
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
246 map_reduce::map_reduce(map, reduce, vec::slice(argv, 1u, argv.len()));
640886c Polymorphic MapReduce!
Eric Holk authored
247 let stop = time::precise_time_ns();
248
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
249 let elapsed = (stop - start) / 1000000u64;
640886c Polymorphic MapReduce!
Eric Holk authored
250
f0dfbe7 Graydon Hoare Register new snapshots, purge log_err and log_full in favour of log(...)...
graydon authored
251 log(error, "MapReduce completed in "
8b58095 Graydon Hoare Register snapshots and switch logging over to use of log_full or #error ...
graydon authored
252 + u64::str(elapsed) + "ms");
640886c Polymorphic MapReduce!
Eric Holk authored
253 }
254
5c49e4f Brian Anderson Reformat. Issue #855
brson authored
255 fn read_word(r: io::reader) -> option<str> {
0c3a128 Eric Holk Update word-count-generic to latest syntax and un-xfail it. Closes #1740...
eholk authored
256 let mut w = "";
640886c Polymorphic MapReduce!
Eric Holk authored
257
258 while !r.eof() {
259 let c = r.read_char();
260
261 if is_word_char(c) {
ab6bb03 Brian Anderson Rename std::istr to std::str. Issue #855
brson authored
262 w += str::from_char(c);
5c49e4f Brian Anderson Reformat. Issue #855
brson authored
263 } else { if w != "" { ret some(w); } }
640886c Polymorphic MapReduce!
Eric Holk authored
264 }
265 ret none;
266 }
df83a79 Eric Holk In generic word count, use str instead of [u8], and use built in is_alph...
eholk authored
267 fn is_word_char(c: char) -> bool {
268 char::is_alphabetic(c) || char::is_digit(c) || c == '_' }
Something went wrong with that request. Please try again.