-
Notifications
You must be signed in to change notification settings - Fork 0
/
grablinks.go
48 lines (43 loc) · 827 Bytes
/
grablinks.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
package grablinks
import (
"golang.org/x/net/html"
"io"
"strconv"
"strings"
)
func All(body io.Reader) []string {
var links []string
var isAdded = make(map[string]bool)
page := html.NewTokenizer(body)
for {
tokenType := page.Next()
if tokenType == html.ErrorToken {
return links
}
token := page.Token()
if tokenType == html.StartTagToken && token.DataAtom.String() == "a" {
for _, attr := range token.Attr {
if attr.Key == "href" {
url := attr.Val
if strings.Contains(url, "#") {
url = trimHash(url)
}
if !isAdded[url] {
links = append(links, url)
isAdded[url] = true
}
}
}
}
}
}
func trimHash(l string) string {
var index int
for i, str := range l {
if strconv.QuoteRune(str) == "'#'" {
index = i
break
}
}
return l[:index]
}