-
Notifications
You must be signed in to change notification settings - Fork 0
/
rss.go
132 lines (111 loc) · 2.65 KB
/
rss.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
package internal
import (
"fmt"
"log"
"net/http"
"sync"
"github.com/gilliek/go-opml/opml"
"github.com/mmcdole/gofeed"
)
type titleFeed struct {
title string // the title will be same as titleURL
feed *gofeed.Feed
}
type titleURL struct {
Title string // title set by opml or user's config
URL string
}
type RSS struct {
titleFeeds []titleFeed
titleURLs []titleURL
c *Controller
}
func (r *RSS) Init(c *Controller) {
r.c = c
if c.conf.OPML != "" {
r.GetTitleURLFromOPML(c.conf.OPML)
} else {
r.GetTitleURLFromConfig()
}
}
func (r *RSS) getURLFromOPML(b opml.Outline) string {
str := ""
if b.XMLURL != "" {
str = b.XMLURL
} else if b.HTMLURL != "" {
str = b.HTMLURL
} else if b.URL != "" {
str = b.URL
}
return str
}
func (r *RSS) GetTitleURLFromOPML(opmlFile string) {
op, err := opml.NewOPMLFromFile(opmlFile)
if err != nil {
log.Fatal("Can't load opml file: ", r.c.conf.OPML, err)
}
for _, b := range op.Body.Outlines {
if b.Outlines != nil {
for _, ib := range b.Outlines {
url := r.getURLFromOPML(ib)
if url != "" {
r.titleURLs = append(r.titleURLs, titleURL{ib.Title, url})
}
}
} else {
url := r.getURLFromOPML(b)
if url != "" {
r.titleURLs = append(r.titleURLs, titleURL{b.Title, url})
}
}
}
}
func (r *RSS) GetTitleURLFromConfig() {
for _, feed := range r.c.conf.Feeds {
r.titleURLs = append(r.titleURLs, titleURL{feed["title"], feed["url"]})
}
}
// FetchURL will send a request to url and use gofeed parse response's body.
func (r *RSS) FetchURL(fp *gofeed.Parser, url string) (*gofeed.Feed, error) {
client := http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp != nil {
defer func() {
// there should be a error handle after defer
resp.Body.Close()
}()
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("failed to get url %v, %v", resp.StatusCode, resp.Status)
}
return fp.Parse(resp.Body)
}
// Update will fetch all content from rss
func (r *RSS) Update() {
var m sync.Mutex
var wg sync.WaitGroup
for _, f := range r.titleURLs {
wg.Add(1)
go func(f titleURL) {
fp := gofeed.NewParser()
feed, err := r.FetchURL(fp, f.URL)
if err != nil {
log.Printf("Error occur when fetch url: %s, err: %v", f.URL, err)
} else {
m.Lock()
r.titleFeeds = append(r.titleFeeds, titleFeed{f.Title, feed})
m.Unlock()
}
wg.Done()
}(f)
}
wg.Wait()
}