-
Notifications
You must be signed in to change notification settings - Fork 3
/
pageinfo.go
144 lines (114 loc) · 2.94 KB
/
pageinfo.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package pageinfo
import (
"bytes"
"crypto/sha1"
"errors"
"fmt"
"log"
"net/http"
"os"
strip "github.com/grokify/html-strip-tags-go"
"golang.org/x/net/html"
)
var (
errPageFetch = errors.New("unable to fetch page")
userAgentString = "Digraph Agent"
)
// PageInfo holds information about a page that has been fetched.
type PageInfo struct {
URL string
Title *string
Body []byte
}
// Fetcher is an interface for structs that fetch pages from the internet.
type Fetcher interface {
FetchPage(string) (*PageInfo, error)
}
// HTMLFetcher is the default fetcher.
type HTMLFetcher struct {
client *http.Client
}
// New returns an HTMLFetcher initialized with the http.Client passed in.
func New(client *http.Client) *HTMLFetcher {
return &HTMLFetcher{client}
}
func isTitleElement(n *html.Node) bool {
return n.Type == html.ElementNode &&
n.Data == "title" &&
n.FirstChild != nil
}
func traverse(n *html.Node) (string, error) {
if isTitleElement(n) {
return n.FirstChild.Data, nil
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type != html.ElementNode {
continue
}
result, err := traverse(c)
if err != nil {
return "", err
}
if result != "" {
return result, nil
}
}
return "", nil
}
// GetHTMLTitle extracts the title from a parsed document.
func GetHTMLTitle(doc *html.Node) (string, error) {
string, err := traverse(doc)
if err != nil {
return "", err
}
// Remove html elements from title
return strip.StripTags(string), nil
}
const browserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
// FetchPage satisfies the Fetcher interface.
func (f *HTMLFetcher) FetchPage(url string) (*PageInfo, error) {
log.Println("Attempting to fetch url:", url)
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", browserUserAgent)
resp, err := f.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
buf := new(bytes.Buffer)
buf.ReadFrom(resp.Body)
body := buf.Bytes()
doc, err := html.Parse(bytes.NewReader(body))
if err != nil {
return nil, err
}
title, err := GetHTMLTitle(doc)
if err != nil {
return nil, err
}
return &PageInfo{URL: url, Title: &title, Body: body}, nil
}
// MissingTitle returns true if no title was found.
func (p *PageInfo) MissingTitle() bool {
return p.Title == nil || *p.Title == ""
}
// Sha1String returns a sha1 of the contents of the page that was fetched.
func (p *PageInfo) Sha1String() string {
return fmt.Sprintf("%x", sha1.Sum(p.Body))
}
// WriteToFile writes the contents of the page that was fetched to the path provided.
func (p *PageInfo) WriteToFile(path string) error {
log.Println(fmt.Sprintf("Writing %s to %s", p.URL, path))
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
if _, err := f.Write(p.Body); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return nil
}