-
Notifications
You must be signed in to change notification settings - Fork 26
/
feed.go
144 lines (124 loc) · 3.01 KB
/
feed.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package builder
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"mime"
"net/http"
"strings"
"time"
"github.com/mmcdole/gofeed"
"github.com/ncarlier/feedpushr/v2/autogen/app"
"github.com/ncarlier/feedpushr/v2/pkg/common"
"github.com/ncarlier/feedpushr/v2/pkg/html"
"github.com/ncarlier/feedpushr/v2/pkg/strcase"
)
// GetFeedID converts URL to feed ID (HASH)
func GetFeedID(url string) string {
hasher := md5.New()
hasher.Write([]byte(url))
return hex.EncodeToString(hasher.Sum(nil))
}
// NewFeed creates new Feed DTO
func NewFeed(url string, tags *string) (*app.Feed, error) {
// Set timeout context
ctx, cancel := context.WithCancel(context.TODO())
timeout := time.AfterFunc(common.DefaultTimeout, func() {
cancel()
})
// Create the request
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
req.Header.Set("User-Agent", common.UserAgent)
// Do HTTP call
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
timeout.Stop()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, fmt.Errorf("http error: %s", res.Status)
}
// Get content-type
contentTypeHeader := res.Header.Get("Content-type")
contentType, _, err := mime.ParseMediaType(contentTypeHeader)
if err != nil {
return nil, err
}
if contentType == "text/html" {
urls, err := html.ExtractFeedLinks(res.Body)
if err != nil {
return nil, err
}
if len(urls) == 0 {
return nil, fmt.Errorf("no feed URL found on this page: %s", url)
}
return NewFeed(urls[0], tags)
}
if !common.ValidFeedContentType.MatchString(contentType) {
return nil, fmt.Errorf("unsupported content type: %s", contentType)
}
// Parse feed
fp := gofeed.NewParser()
fp.AtomTranslator = NewCustomAtomTranslator()
fp.RSSTranslator = NewCustomRSSTranslator()
rawFeed, err := fp.Parse(res.Body)
if err != nil {
return nil, err
}
feed := &app.Feed{
ID: GetFeedID(url),
XMLURL: url,
HTMLURL: &rawFeed.Link,
Title: rawFeed.Title,
Mdate: time.Now(),
Cdate: time.Now(),
Tags: GetFeedTags(tags),
}
if hub, ok := rawFeed.Custom["hub"]; ok {
feed.HubURL = &hub
}
return feed, nil
}
// JoinTags join tags in a comma separated string
func JoinTags(tags ...string) string {
result := ""
for _, tag := range tags {
if result != "" {
if tag != "" {
result += "," + tag
}
} else {
result = tag
}
}
return result
}
// GetFeedTags extracts tags from a comma separated list of tags
func GetFeedTags(tags *string) []string {
if tags == nil || strings.Trim(*tags, " ") == "" {
return []string{}
}
result := strings.Split(*tags, ",")
for i, v := range result {
v = strings.TrimPrefix(v, "/")
result[i] = strcase.ToSnake(v)
}
return deduplicate(result)
}
func deduplicate(list []string) []string {
keys := make(map[string]bool)
result := []string{}
for _, entry := range list {
if _, value := keys[entry]; !value {
keys[entry] = true
result = append(result, entry)
}
}
return result
}