/
rss.go
117 lines (88 loc) · 2.18 KB
/
rss.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
package feed
import (
"context"
"fmt"
"log/slog"
"sort"
"time"
"github.com/mmcdole/gofeed"
)
type RSSFeedItem struct {
ChannelName string
ChannelURL string
Title string
Link string
ImageURL string
PublishedAt time.Time
}
type RSSFeedRequest struct {
Url string `yaml:"url"`
Title string `yaml:"title"`
}
type RSSFeedItems []RSSFeedItem
func (f RSSFeedItems) SortByNewest() RSSFeedItems {
sort.Slice(f, func(i, j int) bool {
return f[i].PublishedAt.After(f[j].PublishedAt)
})
return f
}
var feedParser = gofeed.NewParser()
func getItemsFromRSSFeedTask(request RSSFeedRequest) ([]RSSFeedItem, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
feed, err := feedParser.ParseURLWithContext(request.Url, ctx)
if err != nil {
return nil, err
}
items := make(RSSFeedItems, 0, len(feed.Items))
for i := range feed.Items {
item := feed.Items[i]
rssItem := RSSFeedItem{
ChannelURL: feed.Link,
Title: item.Title,
Link: item.Link,
}
if request.Title != "" {
rssItem.ChannelName = request.Title
} else {
rssItem.ChannelName = feed.Title
}
if item.Image != nil {
rssItem.ImageURL = item.Image.URL
} else if feed.Image != nil {
rssItem.ImageURL = feed.Image.URL
}
if item.PublishedParsed != nil {
rssItem.PublishedAt = *item.PublishedParsed
} else {
rssItem.PublishedAt = time.Now()
}
items = append(items, rssItem)
}
return items, nil
}
func GetItemsFromRSSFeeds(requests []RSSFeedRequest) (RSSFeedItems, error) {
job := newJob(getItemsFromRSSFeedTask, requests).withWorkers(10)
feeds, errs, err := workerPoolDo(job)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrNoContent, err)
}
failed := 0
entries := make(RSSFeedItems, 0, len(feeds)*10)
for i := range feeds {
if errs[i] != nil {
failed++
slog.Error("failed to get rss feed", "error", errs[i], "url", requests[i].Url)
continue
}
entries = append(entries, feeds[i]...)
}
if len(entries) == 0 {
return nil, ErrNoContent
}
entries.SortByNewest()
if failed > 0 {
return entries, fmt.Errorf("%w: missing %d RSS feeds", ErrPartialContent, failed)
}
return entries, nil
}