-
Notifications
You must be signed in to change notification settings - Fork 2
/
mrword.cpp
102 lines (82 loc) · 1.94 KB
/
mrword.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
#include <sector.h>
#include <sys/time.h>
#include <iostream>
using namespace std;
int main(int argc, char** /*argv*/)
{
if (1 != argc)
{
cout << "usage: mrsort" << endl;
return 0;
}
Sector client;
if (Utility::login(client) < 0)
return -1;
vector<string> files;
files.insert(files.end(), "/html");
SphereStream input;
if (input.init(files) < 0)
{
cout << "unable to locate input data files. quit.\n";
Utility::logout(client);
return -1;
}
SphereStream output;
output.setOutputPath("/mrword", "inverted_index");
output.init(256);
SphereProcess* myproc = client.createSphereProcess();
if (myproc->loadOperator("./funcs/mr_word.so") < 0)
{
cout << "cannot find mr_word.so.\n";
Utility::logout(client);
return -1;
}
timeval t;
gettimeofday(&t, 0);
cout << "start time " << t.tv_sec << endl;
int result = myproc->run_mr(input, output, "mr_word", 0);
if (result < 0)
{
Utility::print_error(result);
Utility::logout(client);
return -1;
}
timeval t1, t2;
gettimeofday(&t1, 0);
t2 = t1;
while (true)
{
SphereResult* res = NULL;
if (myproc->read(res) <= 0)
{
if (myproc->checkMapProgress() <= 0)
{
cerr << "all SPEs failed\n";
break;
}
if (myproc->checkMapProgress() == 100)
break;
}
else
{
delete res;
}
gettimeofday(&t2, 0);
if (t2.tv_sec - t1.tv_sec > 60)
{
cout << "MAP PROGRESS: " << myproc->checkProgress() << "%" << endl;
t1 = t2;
}
}
while (myproc->checkReduceProgress() < 100)
{
usleep(10);
}
gettimeofday(&t, 0);
cout << "mapreduce sort accomplished " << t.tv_sec << endl;
cout << "SPE COMPLETED " << endl;
myproc->close();
client.releaseSphereProcess(myproc);
Utility::logout(client);
return 0;
}