-
Notifications
You must be signed in to change notification settings - Fork 0
/
func_scrape_feeds.go
70 lines (56 loc) · 1.59 KB
/
func_scrape_feeds.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
package main
import (
"context"
"log"
"strings"
"sync"
"time"
)
const ERR_MSG_DUPLICATE_URL_KEY = `pq: duplicate key value violates unique constraint "posts_url_key"`
func (cfg *apiConfig) scrapeFeeds(interval time.Duration) {
log.Printf("Starting RSS feed scraper...")
ticker := time.NewTicker(interval)
for ; ; <-ticker.C {
log.Printf("Scraping...")
feeds, err := cfg.getFeedsToFetch()
if err != nil {
log.Printf("Error: cfg.scrapeFeeds: cfg.getFeedsToFetch: %v", err)
return
}
wg := sync.WaitGroup{}
for _, feed := range feeds {
wg.Add(1)
go func(feed Feed) {
defer wg.Done()
defer func() {
if err := recover(); err != nil {
log.Printf("Panic occurred: %s - %v", feed.Url, err)
}
}()
log.Printf("Scraping: %s", feed.Url)
rss, err := fetchRSSFromURL(feed.Url)
if err != nil {
log.Printf("Error: cfg.scrapeFeeds: fetchRSSDataFromURL(%s) %v", feed.Url, err)
}
ctx := context.Background()
err = cfg.DB.MarkFeedFetched(ctx, feed.ID)
if err != nil {
log.Printf("Error: cfg.scrapeFeeds: cfg.DB.MarkFeedFetched: %v", err)
}
log.Printf("Posting from: %s - %s", feed.Url, *rss.Channel.Title)
for _, item := range rss.Channel.Items {
err = cfg.createPost(ctx, feed.ID, item)
if err != nil {
if strings.Contains(err.Error(), ERR_MSG_DUPLICATE_URL_KEY) {
continue
}
log.Printf("Error: cfg.scrapeFeeds: cfg.CreatePost(ctx, item, feed.ID): %v", err)
} else {
log.Printf("Creating post: %s - %s", *rss.Channel.Title, *item.Title)
}
}
}(feed)
}
wg.Wait()
}
}