-
-
Notifications
You must be signed in to change notification settings - Fork 11
/
feed.go
93 lines (83 loc) · 2.51 KB
/
feed.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
package fetch
import (
"fmt"
"html"
"net/http"
"os"
"strings"
"time"
"github.com/microcosm-cc/bluemonday"
"github.com/mmcdole/gofeed"
"github.com/slurdge/goeland/internal/goeland"
"github.com/slurdge/goeland/version"
)
const minContentLen = 10
var policy *bluemonday.Policy
//from https://github.com/mmcdole/gofeed/issues/74#
type userAgentTransport struct {
http.RoundTripper
}
func (c *userAgentTransport) roundTrip(r *http.Request) (*http.Response, error) {
r.Header.Set("User-Agent", "multiple:goeland:"+version.Version+" (commit id:"+version.GitCommit+") (by /u/goelandrss)")
return c.RoundTripper.RoundTrip(r)
}
func fetchFeed(source *goeland.Source, feedLocation string, isFile bool) error {
fp := gofeed.NewParser()
var feed *gofeed.Feed
var err error
if isFile {
file, err := os.Open(feedLocation)
if err != nil {
return fmt.Errorf("cannot open: %s", feedLocation)
}
defer file.Close()
feed, err = fp.Parse(file)
if err != nil {
return fmt.Errorf("cannot parse file: %s", feedLocation)
}
} else {
fp.Client = &http.Client{
Transport: &userAgentTransport{http.DefaultTransport},
}
feed, err = fp.ParseURL(feedLocation)
if err != nil {
return fmt.Errorf("cannot open or parse url: %s", feedLocation)
}
}
for _, item := range feed.Items {
entry := goeland.Entry{}
entry.Title = html.UnescapeString(item.Title)
entry.Content = html.UnescapeString(item.Description)
if len(strings.TrimSpace(entry.Content)) < minContentLen {
entry.Content = html.UnescapeString(item.Content)
}
entry.Title = policy.Sanitize(entry.Title)
entry.Content = policy.Sanitize(entry.Content)
entry.UID = item.GUID
if item.PublishedParsed != nil {
entry.Date = *item.PublishedParsed
} else {
entry.Date = time.Now()
}
entry.URL = item.Link
if len(item.Enclosures) > 0 && strings.HasPrefix(item.Enclosures[0].Type, "image") {
entry.ImageURL = item.Enclosures[0].URL
} else if item.Image != nil && item.Image.URL != "" {
entry.ImageURL = item.Image.URL
} else {
for _, extension := range item.Extensions["media"]["content"] {
if strings.ToLower(extension.Name) == "content" {
if strings.HasPrefix(extension.Attrs["type"], "image") && extension.Attrs["url"] != "" {
entry.ImageURL = extension.Attrs["url"]
}
}
}
}
source.Entries = append(source.Entries, entry)
}
source.Title = feed.Title
return nil
}
func init() {
policy = bluemonday.UGCPolicy()
}