/
main.go
53 lines (45 loc) · 1.07 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
package main
import (
"flag"
"log"
"os"
"os/signal"
"sync"
"time"
)
func main() {
var wg sync.WaitGroup
elasticURL := flag.String("url", "http://localhost:9200", "elastic search api url")
index := flag.String("index", "pastebin", "pastebin")
dailyIndex := flag.Bool("daily", false, time.Now().Format("use daily indexes, e.g. pastebin-2006-1-2"))
APIServer := flag.String("server", "scrape.pastebin.com", "Pastebin scraping API url")
flag.Parse()
// make elastic storage module
e := &ElasticStorageModule{
index: *index,
dailyIndexes: *dailyIndex,
elasticURL: *elasticURL,
}
// make and start fetch worker
fw := FetchWorker{
OutModules: []StorageModule{e},
PastebinURLReplacement: *APIServer,
}
fw.Run(&wg)
// make and start query worker
qw := QueryWorker{
APIServer: *APIServer,
SleepTime: 30000 * time.Millisecond,
}
qw.Run(&wg)
c := make(chan os.Signal)
signal.Notify(c, os.Interrupt)
go func() {
<-c
log.Print("Shutting down... Please Wait")
fw.Stop(&wg)
qw.Stop(&wg)
}()
wg.Wait()
log.Print("Shutdown complete")
}