forked from getlantern/lantern
/
feed.go
323 lines (277 loc) · 8.56 KB
/
feed.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
package feed
import (
"compress/gzip"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"sync"
"time"
"github.com/getlantern/flashlight/geolookup"
"github.com/getlantern/flashlight/proxied"
"github.com/getlantern/golog"
)
const (
// the Feed endpoint where recent content is published to
// mostly just a compendium of RSS feeds
defaultFeedEndpoint = "https://feeds.getiantem.org/%s/feed.json"
fallbackEndpoint = "https://feeds.getiantem.org/en_US/feed.json"
en = "en_US"
)
var (
feed *Feed
_httpClient *http.Client
httpClientMutex sync.Mutex
log = golog.LoggerFor("feed")
)
// Feed contains the data we get back
// from the public feed
type Feed struct {
Feeds map[string]*Source `json:"feeds"`
Entries FeedItems `json:"entries"`
Items map[string]FeedItems `json:"-"`
Sorted []string `json:"sorted_feeds"`
}
// Source represents a feed authority,
// a place where content is fetched from
// e.g. BBC, NYT, Reddit, etc.
type Source struct {
FeedUrl string `json:"feedUrl"`
Title string `json:"title"`
Url string `json:"link"`
ExcludeFromAll bool `json:"excludeFromAll"`
Entries []int `json:"entries"`
}
type FeedItem struct {
Title string `json:"title"`
Link string `json:"link"`
Image string `json:"image"`
Meta map[string]interface{} `json:"meta,omitempty"`
Content string `json:"contentSnippetText"`
Source string `json:"source"`
Description string `json:"-"`
}
type FeedItems []*FeedItem
type FeedProvider interface {
// AddSource: called for each new feed source
AddSource(string)
}
type FeedRetriever interface {
// AddFeed: used to add a new entry to a given feed
AddFeed(string, string, string, string)
}
// FeedByName checks the previously created feed for an
// entry with the given source name
func FeedByName(name string, retriever FeedRetriever) {
if feed != nil && feed.Items != nil {
if items, exists := feed.Items[name]; exists {
for _, i := range items {
retriever.AddFeed(i.Title, i.Description,
i.Image, i.Link)
}
}
}
}
// NumFeedEntries just returns the total number of entries
// across all feeds
func NumFeedEntries() int {
count := len(feed.Entries)
log.Debugf("Number of feed entries: %d", count)
return count
}
// Returns the latest feed or nil if it doesn't exist
func CurrentFeed() *Feed {
return feed
}
// GetFeed creates an http.Client and fetches the latest
// Lantern public feed for displaying on the home screen.
func GetFeed(locale string, allStr string, shouldProxy bool, provider FeedProvider) {
doGetFeed(defaultFeedEndpoint, locale, shouldProxy, allStr, provider)
}
// handleError logs the given error message and sets the current feed to nil
func handleError(err error) {
log.Error(err)
feed = nil
}
// doRequest creates an HTTP request for the given feedURL and returns an HTTP
// response. If an invalid status code is returned, it could be
// because there is no feed available for the given locale so we
// default to the given fallback url.
func doRequest(httpClient *http.Client, feedURL string) (*http.Response, error) {
var err error
var req *http.Request
var res *http.Response
if req, err = http.NewRequest("GET", feedURL, nil); err != nil {
handleError(fmt.Errorf("Error fetching feed: %v", err))
return nil, err
}
// ask for gzipped feed content
req.Header.Add("Accept-Encoding", "gzip")
if res, err = httpClient.Do(req); err != nil {
handleError(fmt.Errorf("Error fetching feed: %v", err))
return nil, err
}
if res.StatusCode != http.StatusOK {
defer res.Body.Close()
err = fmt.Errorf("Unexpected response status %d fetching feed from %s",
res.StatusCode, feedURL)
// no feed available at the given URL, default to the English feed
if feedURL != fallbackEndpoint {
log.Debugf("%v; attempting to fetch fallback feed from %s",
err, fallbackEndpoint)
return doRequest(httpClient, fallbackEndpoint)
} else {
handleError(err)
return nil, err
}
}
return res, nil
}
func doGetFeed(feedEndpoint string, locale string, shouldProxy bool, allStr string,
provider FeedProvider) {
var res *http.Response
var httpClient *http.Client
var err error
if !shouldProxy {
// Connect directly
httpClient = &http.Client{}
} else {
// Connect through proxy
httpClient, err = getHTTPClient()
if err != nil {
handleError(err)
return
}
}
feed = &Feed{}
feedURL := getFeedURL(feedEndpoint, locale)
log.Debugf("Downloading latest feed from %s", feedURL)
res, err = doRequest(httpClient, feedURL)
if err != nil {
handleError(fmt.Errorf("Error fetching feed: %v", err))
return
}
defer func() {
if err := res.Body.Close(); err != nil {
handleError(fmt.Errorf("Error closing response body: %v", err))
}
}()
gzReader, err := gzip.NewReader(res.Body)
if err != nil {
handleError(fmt.Errorf("Unable to open gzip reader: %s", err))
return
}
defer func() {
if err := gzReader.Close(); err != nil {
handleError(fmt.Errorf("Unable to close gzip reader: %s", err))
}
}()
contents, err := ioutil.ReadAll(gzReader)
if err != nil {
handleError(fmt.Errorf("Error reading feed: %v", err))
return
}
err = json.Unmarshal(contents, feed)
if err != nil {
handleError(fmt.Errorf("Error parsing feed: %v", err))
return
}
processFeed(allStr, provider)
}
// processFeed is used after a feed has been downloaded
// to extract feed sources and items for processing.
func processFeed(allStr string, provider FeedProvider) {
log.Debugf("Num of Feed Entries: %v", len(feed.Entries))
feed.Items = make(map[string]FeedItems)
// Add a (shortened) description to every article
for i, entry := range feed.Entries {
desc := ""
if aDesc := entry.Meta["description"]; aDesc != nil {
desc = strings.TrimSpace(aDesc.(string))
}
if desc == "" {
desc = entry.Content
}
feed.Entries[i].Description = desc
}
// the 'all' tab contains every article that's not associated with an
// excluded feed.
all := make(FeedItems, 0, len(feed.Entries))
for _, entry := range feed.Entries {
if !feed.Feeds[entry.Source].ExcludeFromAll {
all = append(all, entry)
}
}
feed.Items[allStr] = all
// Get a list of feed sources and send those back to the UI
for _, source := range feed.Sorted {
if entry, exists := feed.Feeds[source]; exists {
if entry.Title != "" {
log.Debugf("Adding feed source: %s", entry.Title)
provider.AddSource(entry.Title)
} else {
log.Errorf("Skipping feed source: %s; missing title", source)
}
} else {
log.Errorf("Couldn't add feed: %s; missing from map", source)
}
}
for _, s := range feed.Feeds {
for _, i := range s.Entries {
entry := feed.Entries[i]
// every feed item gets appended to a feed source array
// for quick reference
feed.Items[s.Title] = append(feed.Items[s.Title], entry)
}
}
}
// GetFeedURL returns the URL to use for looking up the feed by looking up
// the users country before defaulting to the specified default locale if the
// country can't be determined.
func GetFeedURL(defaultLocale string) string {
return getFeedURL(defaultFeedEndpoint, defaultLocale)
}
func getFeedURL(feedEndpoint, defaultLocale string) string {
locale := determineLocale(defaultLocale)
url := fmt.Sprintf(feedEndpoint, locale)
log.Debugf("Returning feed URL: %v", url)
return url
}
func determineLocale(defaultLocale string) string {
if defaultLocale == "" {
defaultLocale = en
}
// As of this writing the only countries we know of where we want a unique
// feed for the country that's different from the dominantly installed
// language are Iran and Malaysia. In both countries english is the most
// common language on people's machines. We can therefor optimize a little
// bit here and skip the country lookup if the locale is not en_US.
if !strings.EqualFold(en, defaultLocale) {
return defaultLocale
}
country := geolookup.GetCountry(time.Duration(10) * time.Second)
if country == "" {
// This means the country lookup failed, so just use whatever the default is.
log.Debug("Could not lookup country")
return defaultLocale
} else if strings.EqualFold("ir", country) {
return "fa_IR"
} else if strings.EqualFold("my", country) {
return "ms_MY"
}
return defaultLocale
}
func getHTTPClient() (*http.Client, error) {
var err error
httpClientMutex.Lock()
if _httpClient == nil {
var rt http.RoundTripper
rt, err = proxied.ChainedNonPersistent("")
if err == nil {
_httpClient = &http.Client{Transport: rt}
}
}
httpClientMutex.Unlock()
return _httpClient, err
}