-
Notifications
You must be signed in to change notification settings - Fork 1
/
feed.go
101 lines (84 loc) · 1.87 KB
/
feed.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
package feed
import (
"log/slog"
"regexp"
"time"
"github.com/mmcdole/gofeed"
"github.com/theandrew168/bloggulus/backend/fetch"
)
var (
schemeRegexp = regexp.MustCompile("https?://")
)
type Blog struct {
FeedURL string
SiteURL string
Title string
Posts []Post
}
type Post struct {
URL string
Title string
Content string
PublishedAt time.Time
}
func Parse(feedURL string, feedBody string) (Blog, error) {
fp := gofeed.NewParser()
feed, err := fp.ParseString(feedBody)
if err != nil {
return Blog{}, err
}
var posts []Post
for _, item := range feed.Items {
// skip items without a link or title
if item.Link == "" || item.Title == "" {
continue
}
url := item.Link
// ensure url includes the site's domain
if url[0] == '/' {
url = feed.Link + url
}
// ensure url includes a scheme (assume https if necessary)
hasScheme := schemeRegexp.MatchString(url)
if !hasScheme {
url = "https://" + url
}
// check for a publish date, else default to now
publishedAt := time.Now()
if item.PublishedParsed != nil {
publishedAt = *item.PublishedParsed
}
// ensure publishedAt is in UTC
publishedAt = publishedAt.UTC()
post := Post{
URL: url,
Title: item.Title,
Content: item.Content,
PublishedAt: publishedAt,
}
posts = append(posts, post)
}
blog := Blog{
FeedURL: feedURL,
SiteURL: feed.Link,
Title: feed.Title,
Posts: posts,
}
return blog, nil
}
func Hydrate(blog Blog, pageFetcher fetch.PageFetcher) (Blog, error) {
var hydratedPosts []Post
for _, post := range blog.Posts {
if post.Content == "" {
content, err := pageFetcher.FetchPage(post.URL)
if err != nil {
slog.Warn("failed to fetch page", "url", post.URL)
} else {
post.Content = content
}
}
hydratedPosts = append(hydratedPosts, post)
}
blog.Posts = hydratedPosts
return blog, nil
}