-
Notifications
You must be signed in to change notification settings - Fork 0
/
exercice-web-crawler.go
139 lines (125 loc) · 2.83 KB
/
exercice-web-crawler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
package main
import (
"fmt"
"go-tour/utils"
"os"
"runtime"
"strconv"
"strings"
"sync"
)
type Fetcher interface {
// Fetch returns the body of URL and
// a slide of URLs found on that page
Fetch(url string) (body string, urls []string, err error)
}
type SafeUrl struct {
urls map[string]bool
mux sync.Mutex
}
// Check if an URL has already been visited, add it to the list
// in any case.
func (u *SafeUrl) visit(url string) bool {
u.mux.Lock()
defer u.mux.Unlock()
_, ok := u.urls[url]
if ok {
// URL already visited
return false
} else {
// Add URL to list and visit it
u.urls[url] = true
return true
}
}
// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum depth.
func Crawl(url string, depth int, fetcher Fetcher, urlsVisited *SafeUrl) {
if depth <= 0 {
return
}
body, urls, err := fetcher.Fetch(url)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("found: %s %q\n", url, body)
for _, u := range urls {
if urlsVisited.visit(u) {
go Crawl(u, depth-1, fetcher, urlsVisited)
}
}
return
}
func exerciceWebCrawler() {
var DEBUG = utils.Contains(
[]string{"true", "yes", "y", "1"},
strings.ToLower(os.Getenv("DEBUG")))
var DEPTH int
i, ok := strconv.ParseInt(os.Getenv("DEPTH"), 10, 0)
if ok != nil || i < 1 {
DEPTH = 4
} else {
DEPTH = int(i)
}
fmt.Printf(
"You can set DEPTH (current: %v) or DEBUG (current: %v) as environment variables\n",
DEPTH, DEBUG)
urlsVisited := SafeUrl{
urls: make(map[string]bool),
}
initialProcs := runtime.NumGoroutine()
urlsVisited.visit("https://golang.org/") // Mark base url as visited
Crawl("https://golang.org/", DEPTH, fetcher, &urlsVisited)
for procs := runtime.NumGoroutine(); procs != initialProcs; {
if DEBUG {
fmt.Println("Current number of Goroutines:", procs)
}
procs = runtime.NumGoroutine()
}
}
// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult
type fakeResult struct {
body string
urls []string
}
func (f fakeFetcher) Fetch(url string) (string, []string, error) {
if res, ok := f[url]; ok {
return res.body, res.urls, nil
}
return "", nil, fmt.Errorf("not found: %s", url)
}
// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
"https://golang.org/": &fakeResult{
"The Go Programming Language",
[]string{
"https://golang.org/pkg/",
"https://golang.org/cmd/",
},
},
"https://golang.org/pkg/": &fakeResult{
"Packages",
[]string{
"https://golang.org/",
"https://golang.org/cmd/",
"https://golang.org/pkg/fmt/",
"https://golang.org/pkg/os/",
},
},
"https://golang.org/pkg/fmt/": &fakeResult{
"Package fmt",
[]string{
"https://golang.org/",
"https://golang.org/pkg/",
},
},
"https://golang.org/pkg/os/": &fakeResult{
"Package os",
[]string{
"https://golang.org/",
"https://golang.org/pkg/",
},
},
}