-
Notifications
You must be signed in to change notification settings - Fork 0
/
web-crawler.go
102 lines (91 loc) · 1.94 KB
/
web-crawler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
package main
import (
"fmt"
"sync"
)
type Fetcher interface {
// Fetch returns the body of URL and
// a slice of URLs found on that page.
Fetch(url string) (body string, urls []string, err error)
}
var fetched = struct {
m map[string]error
sync.Mutex
}{m: map[string]error{}}
// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
if _, ok := fetched.m[url]; ok || depth <= 0 {
return
}
body, urls, err := fetcher.Fetch(url)
fetched.Lock()
fetched.m[url] = err
fetched.Unlock()
if err != nil {
return
}
fmt.Printf("Found: %s %q\n", url, body)
var wg sync.WaitGroup
for _, u := range urls {
wg.Add(1)
go func(url string) {
defer wg.Done()
Crawl(url, depth-1, fetcher)
}(u)
}
wg.Wait()
}
func main() {
Crawl("http://golang.org/", 4, fetcher)
for k, v := range fetched.m {
if v == nil {
fmt.Printf("existing url: %v\n", k)
}
}
}
// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult
type fakeResult struct {
body string
urls []string
}
func (f fakeFetcher) Fetch(url string) (string, []string, error) {
if res, ok := f[url]; ok {
return res.body, res.urls, nil
}
return "", nil, fmt.Errorf("not found: %s", url)
}
// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
"http://golang.org/": &fakeResult{
"The Go Programming Language",
[]string{
"http://golang.org/pkg/",
"http://golang.org/cmd/",
},
},
"http://golang.org/pkg/": &fakeResult{
"Packages",
[]string{
"http://golang.org/",
"http://golang.org/cmd/",
"http://golang.org/pkg/fmt/",
"http://golang.org/pkg/os/",
},
},
"http://golang.org/pkg/fmt/": &fakeResult{
"Package fmt",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
"http://golang.org/pkg/os/": &fakeResult{
"Package os",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
}