/
source_hackernews.go
122 lines (109 loc) · 2.39 KB
/
source_hackernews.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
package fakenews
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"sync"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
)
const (
hackernewsBaseURL = "https://hacker-news.firebaseio.com/v0"
hackernewsListPath = "topstories.json"
hackernewsItemPath = "item"
hackernewsDefaultConcurrency = 8
)
type hackernewsStoryList []int
type hackernewsStoryItem struct {
Title string `json:"title"`
}
type HackernewsSource struct {
Client Client
Limit int
Concurrency int
items []string
mux sync.Mutex
}
func (s *HackernewsSource) Fetch(ctx context.Context) error {
s.mux.Lock()
defer s.mux.Unlock()
if s.items != nil {
return nil
}
if s.Client == nil {
s.Client = http.DefaultClient
}
if s.Concurrency == 0 {
s.Concurrency = hackernewsDefaultConcurrency
}
list, err := s.fetchList(ctx)
if err != nil {
return err
}
if s.Limit > 0 {
list = list[:s.Limit]
}
s.items = make([]string, len(list))
g, ctx := errgroup.WithContext(ctx)
sem := semaphore.NewWeighted(int64(s.Concurrency))
for idx0, id0 := range list {
err = sem.Acquire(ctx, 1)
if err != nil {
return err
}
idx1, id1 := idx0, id0
g.Go(func() (err error) {
defer sem.Release(1)
err = s.fetchItem(ctx, idx1, id1)
if err != nil {
return err
}
return nil
})
}
return g.Wait()
}
func (s *HackernewsSource) Items() []string {
return s.items
}
func (s *HackernewsSource) fetchItem(ctx context.Context, idx, id int) error {
rq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s/%d.json", hackernewsBaseURL, hackernewsItemPath, id), nil)
if err != nil {
return err
}
rs, err := s.Client.Do(rq.WithContext(ctx))
if err != nil {
return err
}
defer rs.Body.Close()
body, err := ioutil.ReadAll(rs.Body)
if err != nil {
return err
}
var item hackernewsStoryItem
err = json.Unmarshal(body, &item)
if err != nil {
return err
}
s.items[idx] = item.Title
return nil
}
func (s *HackernewsSource) fetchList(ctx context.Context) (list hackernewsStoryList, err error) {
rq, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", hackernewsBaseURL, hackernewsListPath), nil)
if err != nil {
return
}
rs, err := s.Client.Do(rq.WithContext(ctx))
if err != nil {
return
}
defer rs.Body.Close()
body, err := ioutil.ReadAll(rs.Body)
if err != nil {
return
}
err = json.Unmarshal(body, &list)
return
}