Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 333 lines (277 sloc) 8.277 kb
a4956a7 @brson XFAIL task-perf-word-count-generic
brson authored
1 // xfail-test - #1038 - Can't do this safely with bare functions
2
640886c Polymorphic MapReduce!
Eric Holk authored
3 /**
4 A parallel word-frequency counting program.
5
6 This is meant primarily to demonstrate Rust's MapReduce framework.
7
8 It takes a list of files on the command line and outputs a list of
9 words along with how many times each word is used.
10
11 */
12
13 use std;
14
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
15 import option = option::t;
16 import option::some;
17 import option::none;
18 import str;
640886c Polymorphic MapReduce!
Eric Holk authored
19 import std::treemap;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
20 import vec;
640886c Polymorphic MapReduce!
Eric Holk authored
21 import std::io;
22
23 import std::time;
fa9ad98 @graydon Copy first batch of material from libstd to libcore.
graydon authored
24 import u64;
25
26 import task;
27 import task::joinable_task;
28 import comm;
29 import comm::chan;
30 import comm::port;
31 import comm::recv;
32 import comm::send;
640886c Polymorphic MapReduce!
Eric Holk authored
33
33167f7 @marijnh Adjust function signatures to allow for vecs being immediate
marijnh authored
34 fn map(&&filename: [u8], emit: map_reduce::putter<[u8], int>) {
ab6bb03 @brson Rename std::istr to std::str. Issue #855
brson authored
35 let f = io::file_reader(str::unsafe_from_bytes(filename));
640886c Polymorphic MapReduce!
Eric Holk authored
36
37 while true {
38 alt read_word(f) {
ab6bb03 @brson Rename std::istr to std::str. Issue #855
brson authored
39 some(w) { emit(str::bytes(w), 1); }
640886c Polymorphic MapReduce!
Eric Holk authored
40 none. { break; }
41 }
42 }
43 }
44
33167f7 @marijnh Adjust function signatures to allow for vecs being immediate
marijnh authored
45 fn reduce(&&_word: [u8], get: map_reduce::getter<int>) {
640886c Polymorphic MapReduce!
Eric Holk authored
46 let count = 0;
47
7298b8f @marijnh Insert omitted semicolons for statements
marijnh authored
48 while true { alt get() { some(_) { count += 1; } none. { break; } } }
640886c Polymorphic MapReduce!
Eric Holk authored
49 }
50
51 mod map_reduce {
52 export putter;
53 export getter;
54 export mapper;
55 export reducer;
56 export map_reduce;
57
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
58 type putter<send K, send V> = fn(K, V);
640886c Polymorphic MapReduce!
Eric Holk authored
59
60 // FIXME: the first K1 parameter should probably be a -, but that
61 // doesn't parse at the moment.
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
62 type mapper<send K1, send K2, send V> = fn(K1, putter<K2, V>);
640886c Polymorphic MapReduce!
Eric Holk authored
63
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
64 type getter<send V> = fn() -> option<V>;
640886c Polymorphic MapReduce!
Eric Holk authored
65
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
66 type reducer<send K, send V> = fn(K, getter<V>);
640886c Polymorphic MapReduce!
Eric Holk authored
67
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
68 tag ctrl_proto<send K, send V> {
640886c Polymorphic MapReduce!
Eric Holk authored
69 find_reducer(K, chan<chan<reduce_proto<V>>>);
70 mapper_done;
71 }
72
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
73 tag reduce_proto<send V> { emit_val(V); done; ref; release; }
640886c Polymorphic MapReduce!
Eric Holk authored
74
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
75 fn start_mappers<send K1, send K2,
76 send V>(map: mapper<K1, K2, V>,
fc6b7c8 @marijnh Reformat for new mode syntax, step 1
marijnh authored
77 ctrl: chan<ctrl_proto<K2, V>>, inputs: [K1]) ->
5c49e4f @brson Reformat. Issue #855
brson authored
78 [joinable_task] {
640886c Polymorphic MapReduce!
Eric Holk authored
79 let tasks = [];
80 for i in inputs {
81 let m = map, c = ctrl, ii = i;
82 tasks += [task::spawn_joinable(bind map_task(m, c, ii))];
83 }
84 ret tasks;
85 }
86
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
87 fn map_task<send K1, send K2,
88 send V>(-map: mapper<K1, K2, V>,
cfdf193 @marijnh Update our code to new type parameter kind syntax
marijnh authored
89 -ctrl: chan<ctrl_proto<K2, V>>,
ca1df2b @marijnh Pretty-print for new arg-mode syntax
marijnh authored
90 -input: K1) {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
91 // log(error, "map_task " + input);
640886c Polymorphic MapReduce!
Eric Holk authored
92 let intermediates = treemap::init();
93
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
94 fn emit<send K2,
95 send V>(im: treemap::treemap<K2, chan<reduce_proto<V>>>,
fc6b7c8 @marijnh Reformat for new mode syntax, step 1
marijnh authored
96 ctrl: chan<ctrl_proto<K2, V>>, key: K2, val: V) {
640886c Polymorphic MapReduce!
Eric Holk authored
97 let c;
98 alt treemap::find(im, key) {
7298b8f @marijnh Insert omitted semicolons for statements
marijnh authored
99 some(_c) { c = _c; }
640886c Polymorphic MapReduce!
Eric Holk authored
100 none. {
101 let p = port();
102 send(ctrl, find_reducer(key, chan(p)));
103 c = recv(p);
104 treemap::insert(im, key, c);
105 send(c, ref);
106 }
107 }
108 send(c, emit_val(val));
109 }
110
111 map(input, bind emit(intermediates, ctrl, _, _));
112
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
113 fn finish<send K, send V>(_k: K, v: chan<reduce_proto<V>>) {
640886c Polymorphic MapReduce!
Eric Holk authored
114 send(v, release);
115 }
116 treemap::traverse(intermediates, finish);
117 send(ctrl, mapper_done);
118 }
119
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
120 fn reduce_task<send K,
121 send V>(-reduce: reducer<K, V>, -key: K,
ca1df2b @marijnh Pretty-print for new arg-mode syntax
marijnh authored
122 -out: chan<chan<reduce_proto<V>>>) {
640886c Polymorphic MapReduce!
Eric Holk authored
123 let p = port();
124
125 send(out, chan(p));
126
127 let ref_count = 0;
128 let is_done = false;
129
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
130 fn get<send V>(p: port<reduce_proto<V>>,
cfdf193 @marijnh Update our code to new type parameter kind syntax
marijnh authored
131 &ref_count: int, &is_done: bool)
ca1df2b @marijnh Pretty-print for new arg-mode syntax
marijnh authored
132 -> option<V> {
640886c Polymorphic MapReduce!
Eric Holk authored
133 while !is_done || ref_count > 0 {
134 alt recv(p) {
135 emit_val(v) {
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
136 // #error("received %d", v);
640886c Polymorphic MapReduce!
Eric Holk authored
137 ret some(v);
138 }
139 done. {
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
140 // #error("all done");
640886c Polymorphic MapReduce!
Eric Holk authored
141 is_done = true;
142 }
143 ref. { ref_count += 1; }
144 release. { ref_count -= 1; }
145 }
146 }
147 ret none;
148 }
149
150 reduce(key, bind get(p, ref_count, is_done));
151 }
152
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
153 fn map_reduce<send K1, send K2,
154 send V>(map: mapper<K1, K2, V>, reduce: reducer<K2, V>,
fc6b7c8 @marijnh Reformat for new mode syntax, step 1
marijnh authored
155 inputs: [K1]) {
640886c Polymorphic MapReduce!
Eric Holk authored
156 let ctrl = port();
157
158 // This task becomes the master control task. It task::_spawns
159 // to do the rest.
160
161 let reducers = treemap::init();
162
163 let tasks = start_mappers(map, chan(ctrl), inputs);
164
165 let num_mappers = vec::len(inputs) as int;
166
167 while num_mappers > 0 {
168 alt recv(ctrl) {
169 mapper_done. {
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
170 // #error("received mapper terminated.");
640886c Polymorphic MapReduce!
Eric Holk authored
171 num_mappers -= 1;
172 }
173 find_reducer(k, cc) {
174 let c;
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
175 // log(error, "finding reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
176 alt treemap::find(reducers, k) {
177 some(_c) {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
178 // log(error,
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
179 // "reusing existing reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
180 c = _c;
181 }
182 none. {
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
183 // log(error, "creating new reducer for " + k);
640886c Polymorphic MapReduce!
Eric Holk authored
184 let p = port();
185 let r = reduce, kk = k;
186 tasks +=
5c49e4f @brson Reformat. Issue #855
brson authored
187 [task::spawn_joinable(bind reduce_task(r, kk,
188 chan(p)))];
640886c Polymorphic MapReduce!
Eric Holk authored
189 c = recv(p);
190 treemap::insert(reducers, k, c);
191 }
192 }
193 send(cc, c);
194 }
195 }
196 }
197
f6491bb @marijnh Update stdlib, compiler, and tests to new kind system
marijnh authored
198 fn finish<send K, send V>(_k: K, v: chan<reduce_proto<V>>) {
cfdf193 @marijnh Update our code to new type parameter kind syntax
marijnh authored
199 send(v, done);
200 }
640886c Polymorphic MapReduce!
Eric Holk authored
201 treemap::traverse(reducers, finish);
202
203 for t in tasks { task::join(t); }
204 }
205 }
206
5c49e4f @brson Reformat. Issue #855
brson authored
207 fn main(argv: [str]) {
640886c Polymorphic MapReduce!
Eric Holk authored
208 if vec::len(argv) < 2u {
209 let out = io::stdout();
210
5c49e4f @brson Reformat. Issue #855
brson authored
211 out.write_line(#fmt["Usage: %s <filename> ...", argv[0]]);
640886c Polymorphic MapReduce!
Eric Holk authored
212
213 // TODO: run something just to make sure the code hasn't
214 // broken yet. This is the unit test mode of this program.
215
216 ret;
217 }
218
219 let iargs = [];
220 for a in vec::slice(argv, 1u, vec::len(argv)) {
ab6bb03 @brson Rename std::istr to std::str. Issue #855
brson authored
221 iargs += [str::bytes(a)];
640886c Polymorphic MapReduce!
Eric Holk authored
222 }
223
224 let start = time::precise_time_ns();
225
226 map_reduce::map_reduce(map, reduce, iargs);
227 let stop = time::precise_time_ns();
228
229 let elapsed = stop - start;
230 elapsed /= 1000000u64;
231
f0dfbe7 @graydon Register new snapshots, purge log_err and log_full in favour of log(.…
graydon authored
232 log(error, "MapReduce completed in "
8b58095 @graydon Register snapshots and switch logging over to use of log_full or #err…
graydon authored
233 + u64::str(elapsed) + "ms");
640886c Polymorphic MapReduce!
Eric Holk authored
234 }
235
5c49e4f @brson Reformat. Issue #855
brson authored
236 fn read_word(r: io::reader) -> option<str> {
237 let w = "";
640886c Polymorphic MapReduce!
Eric Holk authored
238
239 while !r.eof() {
240 let c = r.read_char();
241
242
243 if is_word_char(c) {
ab6bb03 @brson Rename std::istr to std::str. Issue #855
brson authored
244 w += str::from_char(c);
5c49e4f @brson Reformat. Issue #855
brson authored
245 } else { if w != "" { ret some(w); } }
640886c Polymorphic MapReduce!
Eric Holk authored
246 }
247 ret none;
248 }
249
250 fn is_digit(c: char) -> bool {
251 alt c {
252 '0' { true }
253 '1' { true }
254 '2' { true }
255 '3' { true }
256 '4' { true }
257 '5' { true }
258 '6' { true }
259 '7' { true }
260 '8' { true }
261 '9' { true }
262 _ { false }
263 }
264 }
265
266 fn is_alpha_lower(c: char) -> bool {
267 alt c {
268 'a' { true }
269 'b' { true }
270 'c' { true }
271 'd' { true }
272 'e' { true }
273 'f' { true }
274 'g' { true }
275 'h' { true }
276 'i' { true }
277 'j' { true }
278 'k' { true }
279 'l' { true }
280 'm' { true }
281 'n' { true }
282 'o' { true }
283 'p' { true }
284 'q' { true }
285 'r' { true }
286 's' { true }
287 't' { true }
288 'u' { true }
289 'v' { true }
290 'w' { true }
291 'x' { true }
292 'y' { true }
293 'z' { true }
294 _ { false }
295 }
296 }
297
298 fn is_alpha_upper(c: char) -> bool {
299 alt c {
300 'A' { true }
301 'B' { true }
302 'C' { true }
303 'D' { true }
304 'E' { true }
305 'F' { true }
306 'G' { true }
307 'H' { true }
308 'I' { true }
309 'J' { true }
310 'K' { true }
311 'L' { true }
312 'M' { true }
313 'N' { true }
314 'O' { true }
315 'P' { true }
316 'Q' { true }
317 'R' { true }
318 'S' { true }
319 'T' { true }
320 'U' { true }
321 'V' { true }
322 'W' { true }
323 'X' { true }
324 'Y' { true }
325 'Z' { true }
326 _ { false }
327 }
328 }
329
330 fn is_alpha(c: char) -> bool { is_alpha_upper(c) || is_alpha_lower(c) }
331
332 fn is_word_char(c: char) -> bool { is_alpha(c) || is_digit(c) || c == '_' }
Something went wrong with that request. Please try again.