11package rss
22
33import (
4- "encoding/json"
4+ "context"
5+ "flag"
56 "fmt"
67 "io"
78 "net/url"
89 "os"
9- "sort"
1010
1111 "github.com/frioux/leatherman/internal/lmhttp"
1212 "github.com/mmcdole/gofeed"
13+ "golang.org/x/sync/errgroup"
1314)
1415
1516/*
16- Run is a minimalist rss client. Outputs links as markdown on STDOUT. Takes url
17- to feed and path to state file. Example usage:
17+ Run is a minimalist rss client. Outputs links as markdown on STDOUT. Takes urls
18+ to feeds and path to state file. Example usage:
1819
1920```bash
20- $ rss https://blog.afoolishmanifesto.com/index.xml afm.json
21+ $ rss -state feed.json https://blog.afoolishmanifesto.com/index.xml
2122[Announcing shellquote](https://blog.afoolishmanifesto.com/posts/announcing-shellquote/)
2223[Detecting who used the EC2 metadata server with BCC](https://blog.afoolishmanifesto.com/posts/detecting-who-used-ec2-metadata-server-bcc/)
2324[Centralized known_hosts for ssh](https://blog.afoolishmanifesto.com/posts/centralized-known-hosts-for-ssh/)
@@ -28,44 +29,102 @@ $ rss https://blog.afoolishmanifesto.com/index.xml afm.json
2829Command: rss
2930*/
3031func Run (args []string , _ io.Reader ) error {
31- if len (args ) != 3 {
32- fmt .Fprintf (os .Stderr , "Usage: %s feedURL statefile\n " , args [0 ])
32+ flags := flag .NewFlagSet ("rss" , flag .ExitOnError )
33+
34+ var statePath string
35+
36+ flags .StringVar (& statePath , "state" , "" , "location to store state" )
37+ if err := flags .Parse (args [1 :]); err != nil {
38+ return fmt .Errorf ("flags.Parse: %w" , err )
39+ }
40+
41+ if len (flags .Args ()) == 0 {
42+ fmt .Fprintf (os .Stderr , "Usage: %s -state rss.json <url> [<url>...]\n " , args [0 ])
3343 os .Exit (1 )
3444 }
3545
36- return run (args [1 ], args [2 ], os .Stdout )
46+ if statePath == "" {
47+ fmt .Fprintln (os .Stderr , "-state is required" )
48+ os .Exit (1 )
49+ }
50+
51+ return run (statePath , flags .Args (), os .Stdout )
3752}
3853
39- func run (urlString , statePath string , w io.Writer ) error {
40- fp := gofeed .NewParser ()
54+ func loadFeed (fp * gofeed.Parser , urlString string ) ([]* gofeed.Item , error ) {
4155 feedURL , err := url .Parse (urlString )
4256 if err != nil {
43- return fmt .Errorf ("Couldn't parse feed url (%s): %w" , feedURL , err )
57+ return nil , fmt .Errorf ("Couldn't parse feed url (%s): %w" , urlString , err )
4458 }
4559
4660 resp , err := lmhttp .Get (urlString )
4761 if err != nil {
48- return fmt .Errorf ("Couldn't get feed: %w" , err )
62+ return nil , fmt .Errorf ("Couldn't get feed: %w" , err )
4963 }
5064
5165 f , err := fp .Parse (resp .Body )
5266 if err != nil {
53- return fmt .Errorf ("Couldn't fetch feed (%s): %w" , feedURL , err )
67+ return nil , fmt .Errorf ("Couldn't fetch feed (%s): %w" , feedURL , err )
5468 }
5569 fixItems (feedURL , f .Items )
5670
57- seen , err := syncRead (statePath , f .Items )
58- if err != nil {
59- return fmt .Errorf ("Couldn't sync read (%s): %w" , feedURL , err )
71+ return f .Items , nil
72+ }
73+
74+ func syncFeed (state indexedStates , items []* gofeed.Item , urlString string , w io.Writer ) error {
75+ if state [urlString ] == nil {
76+ state [urlString ] = make (map [string ]bool , len (items ))
6077 }
6178
62- items := newItems (seen , f .Items )
79+ items = newItems (state [urlString ], items )
80+
81+ for _ , i := range items {
82+ state [urlString ][i.GUID ] = true
83+ }
6384
6485 renderItems (w , items )
6586
66- err = os .Rename (statePath + ".tmp" , statePath )
87+ return nil
88+ }
89+
90+ func run (statePath string , urls []string , w io.Writer ) error {
91+ state , err := readState (statePath )
6792 if err != nil {
68- return fmt .Errorf ("Couldn't rename state file (%s): %w" , feedURL , err )
93+ return fmt .Errorf ("couldn't read state: %w" , err )
94+ }
95+ fp := gofeed .NewParser ()
96+
97+ results := make ([][]* gofeed.Item , len (urls ))
98+ g , _ := errgroup .WithContext (context .Background ())
99+
100+ for i , urlString := range urls {
101+ i , urlString := i , urlString
102+ g .Go (func () error { // O(n) goroutines
103+ items , err := loadFeed (fp , urlString )
104+ if err != nil {
105+ return err
106+ }
107+ results [i ] = items
108+ return nil
109+ })
110+ }
111+
112+ if err := g .Wait (); err != nil {
113+ fmt .Fprintf (os .Stderr , "%s\n " , err )
114+ os .Exit (1 )
115+ }
116+ for i , items := range results {
117+ if err := syncFeed (state , items , urls [i ], w ); err != nil {
118+ fmt .Fprintf (os .Stderr , "%s\n " , err )
119+ os .Exit (1 )
120+ }
121+ }
122+
123+ if err := writeState (statePath , state ); err != nil {
124+ return fmt .Errorf ("Couldn't save state file: %w" , err )
125+ }
126+ if err := os .Rename (statePath + ".tmp" , statePath ); err != nil {
127+ return fmt .Errorf ("Couldn't rename state file: %w" , err )
69128 }
70129
71130 return nil
@@ -108,76 +167,3 @@ func newItems(seen map[string]bool, items []*gofeed.Item) []*gofeed.Item {
108167
109168 return ret
110169}
111-
112- // Store JSON containing seen GUIDs for the current feed.
113- func syncRead (state string , items []* gofeed.Item ) (map [string ]bool , error ) {
114- ret := make (map [string ]bool , len (items ))
115-
116- guids , err := readState (state )
117- if err != nil {
118- return nil , fmt .Errorf ("couldn't read state: %w" , err )
119- }
120-
121- for _ , g := range guids {
122- ret [g ] = true
123- }
124-
125- // Generate news state
126- newState := make (map [string ]bool , len (items )+ len (guids ))
127-
128- for _ , g := range guids {
129- newState [g ] = true
130- }
131- for _ , i := range items {
132- newState [i .GUID ] = true
133- }
134- toStore := make ([]string , 0 , len (newState ))
135-
136- for k := range newState {
137- toStore = append (toStore , k )
138- }
139- sort .Strings (toStore )
140-
141- err = writeState (state , toStore )
142- if err != nil {
143- return nil , fmt .Errorf ("couldn't write state: %w" , err )
144- }
145- return ret , nil
146- }
147-
148- func readState (state string ) ([]string , error ) {
149- file , err := os .Open (state )
150- if err != nil && ! os .IsNotExist (err ) {
151- return nil , fmt .Errorf ("couldn't open state file: %w" , err )
152- }
153-
154- var guids []string
155-
156- if err == nil {
157- decoder := json .NewDecoder (file )
158- err = decoder .Decode (& guids )
159- if err != nil && ! os .IsNotExist (err ) {
160- return nil , fmt .Errorf ("couldn't decode state file: %w" , err )
161- }
162- }
163-
164- return guids , nil
165- }
166-
167- func writeState (state string , guids []string ) error {
168- tmp , err := os .Create (state + ".tmp" )
169- if err != nil {
170- return fmt .Errorf ("couldn't create state file: %w" , err )
171- }
172- encoder := json .NewEncoder (tmp )
173- encoder .SetIndent ("" , "\t " )
174- err = encoder .Encode (guids )
175- if err != nil {
176- return fmt .Errorf ("couldn't encode state file: %w" , err )
177- }
178- err = tmp .Close ()
179- if err != nil {
180- return fmt .Errorf ("couldn't write state file: %w" , err )
181- }
182- return nil
183- }
0 commit comments