-
Notifications
You must be signed in to change notification settings - Fork 0
/
processor.go
157 lines (129 loc) · 3.99 KB
/
processor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package processor
import (
"fmt"
"github.com/go-rod/rod"
"github.com/margostino/anfield/common"
"github.com/margostino/anfield/db"
"github.com/margostino/anfield/domain"
"log"
"strings"
"sync"
"time"
)
//var webScrapper *scrapper.Scrapper
//var waitGroups map[string]*sync.WaitGroup
//var waitGroups = sync.Map{}
var waitGroup *sync.WaitGroup
var stats = sync.Map{}
//var matchDateBuffer map[string]chan string
//var lineupsBuffer map[string]chan *domain.Lineups
//var commentaryBuffer map[string]chan *domain.Commentary
func Initialize() {
//webScrapper = scrapper.New()
//waitGroups = make(map[string]*sync.WaitGroup, 0)
//commentaryBuffer = make(map[string]chan *domain.Commentary)
//matchDateBuffer = make(map[string]chan string)
//lineupsBuffer = make(map[string]chan *domain.Lineups)
}
func NewChannels() *Channels {
return &Channels{
commentary: make(map[string]chan *domain.Commentary),
matchDate: make(map[string]chan time.Time),
lineups: make(map[string]chan *domain.Lineups),
}
}
func NewWaitGroups() sync.Map {
return sync.Map{}
}
func (a App) InitializeChannels(url string) {
//a.waitGroups.Store(url, common.WaitGroup(4))
a.channels.matchDate[url] = make(chan time.Time)
a.channels.lineups[url] = make(chan *domain.Lineups)
a.channels.commentary[url] = make(chan *domain.Commentary)
}
func (a App) Process(urls []string) error {
eventsToProcess := len(urls)
if eventsToProcess == 0 {
log.Println("URLs Not Found!")
} else {
waitGroup = common.WaitGroup(len(urls) * (1 + 1 + 1 + 1))
log.Println("Events to process: ", eventsToProcess)
for _, url := range urls {
if a.shouldProcess(url) {
// TODO: if match already exists then remove it and start from the beginning
a.InitializeChannels(url)
a.produce(url)
go a.consume(url)
} else {
a.logger.info(fmt.Sprintf("URL %s was already processed", url))
waitGroup.Add(-4)
}
}
waitGroup.Wait()
}
return nil // TODO: to be defined
}
func (a App) wait(url string) {
wg, _ := a.waitGroups.Load(url)
wg.(*sync.WaitGroup).Wait()
}
func (a App) getUrlsByConfig() []string {
matches := a.configuration.Events.Matches
baseUrl := a.configuration.Scrapper.Url
var urls []string
for _, url := range matches {
urls = append(urls, baseUrl+url)
}
return urls
}
func (a App) getUrlsByScrapper() []string {
baseUrl := a.configuration.Scrapper.Url
fixturesUrl := baseUrl + a.configuration.Scrapper.FixturesPath // Matches in progress
resultsUrl := baseUrl + a.configuration.Scrapper.ResultsPath // Matches finished
fixtureUrls := a.getMatchUrlsFrom(fixturesUrl)
resultUrls := a.getMatchUrlsFrom(resultsUrl)
return append(fixtureUrls, resultUrls...)
}
func (a App) getMatchUrlsFrom(url string) []string {
selector := a.configuration.Scrapper.MatchRowsSelector
property := a.configuration.Scrapper.UrlProperty
pattern := a.configuration.Scrapper.HrefPattern
var urls []string
var prevSize = 0
var tolerance = 35
var currentSize = -1
var equalsCounter = 0
var elements rod.Elements
webScrapper := a.scrapper.GoPage(url) // TODO: fix and reuse the scrapper in the struct instead of this
for {
elements = webScrapper.DynamicElementsByPattern(selector, pattern)
for _, element := range elements {
//status := element.MustText() ---> inProgress(status) # For Realtime.
url := element.MustProperty(property).String()
if !common.InSlice(url, urls) {
urls = append(urls, url)
}
}
prevSize, currentSize = currentSize, len(urls)
if prevSize == currentSize {
equalsCounter += 1
} else {
equalsCounter = 0
}
if equalsCounter == tolerance {
break
}
}
return urls
}
func inProgress(status string) bool {
prefix := strings.Split(status, "\n")[0]
return common.IsTimeCounter(prefix)
}
// TODO: support partial completion and continue
func (a App) shouldProcess(url string) bool {
var document domain.MatchDocument
filter := db.MatchFilter(url)
a.db.Matches.FindOne(filter, document)
return document.Metadata == nil || (document.Metadata != nil && !document.Metadata.Finished)
}