/
main.go
92 lines (78 loc) · 2.26 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package main
import (
"log"
"net/http"
"os"
"path/filepath"
)
func main() {
//runtime.GOMAXPROCS(1)
root, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
r := 3
address := "localhost:8080"
/*
m := 9
tempdir := filepath.Join(root, "test/part2/map")
source := filepath.Join(root, "test/split/in/austen.db")
outputPattern := "map_%d_input.db"
go func() {
http.Handle("/data/", http.StripPrefix("/data", http.FileServer(http.Dir(tempdir))))
if err := http.ListenAndServe(address, nil); err != nil {
log.Printf("Error in HTTP server for %s: %v", address, err)
}
}()
_, err = splitDatabase(source, tempdir, outputPattern, m)
if err != nil {
log.Fatal(err)
}
//fmt.Printf("the lst from split : %s\n", lst)
for n := 0; n < m; n++ {
m := MapTask{M: m, R: r, N: n, SourceHost: makeURL("localhost:8080/", "austen.db")}
if err = m.Process(tempdir, Client{}); err != nil {
log.Fatal(err)
}
}
go func() {
http.Handle("/data/", http.StripPrefix("/data", http.FileServer(http.Dir(filepath.Join(root, "test/part2/map")))))
if err := http.ListenAndServe(address, nil); err != nil {
log.Printf("Error in HTTP server for %s: %v", address, err)
}
}()
tempdir = filepath.Join(root, "test/part2/reduce")
for i := 0; i < r; i++ {
var lst []string
for j := 0; j < m; j++ {
file := mapOutputFile(j, i)
lst = append(lst, makeURL(address, file))
}
m := ReduceTask{M: m, R: r, N: i, SourceHosts: lst}
if err = m.Process(tempdir, Client{}); err != nil {
log.Fatal(err)
}
}
*/
// final merger
var urls []string
for i := 0; i < r; i++ {
file := reduceOutputFile(i)
urls = append(urls, makeURL(address, file))
}
log.Print(urls)
after := filepath.Join(root, "test/part2/reduce/reduce_out.db")
go func() {
http.Handle("/data/", http.StripPrefix("/data", http.FileServer(http.Dir(filepath.Join(root, "test/part2/reduce")))))
if err := http.ListenAndServe(address, nil); err != nil {
log.Printf("Error in HTTP server for %s: %v", address, err)
}
}()
_, err = mergeDatabases(urls, after, filepath.Join(root, "test/part2/map"))
if err != nil {
log.Fatal(err)
} else {
log.Print("\ndone")
}
//defer os.RemoveAll(tempdir)
}