-
Notifications
You must be signed in to change notification settings - Fork 3
/
script.go
103 lines (95 loc) · 2.4 KB
/
script.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
package quebec
import (
"context"
"math"
"github.com/sirupsen/logrus"
"github.com/whitewater-guide/gorge/core"
)
type optionsQuebec struct{}
type scriptQuebec struct {
name string
codesURL string
referenceListURL string
stationURLFormat string
readingsURLFormat string
core.LoggingScript
}
func (s *scriptQuebec) ListGauges() (result core.Gauges, err error) {
federal, err := s.getReferenceList()
var local []stationInfo
if err != nil {
return nil, err
}
codes, err := s.getCodes()
if err != nil {
return nil, err
}
jobsCh := make(chan string, len(codes))
resultsCh := make(chan stationInfo, len(codes))
numWorkers := int(math.Min(5, float64(len(codes))))
for i := 0; i < numWorkers; i++ {
go s.stationWorker(jobsCh, resultsCh)
}
for _, code := range codes {
jobsCh <- code
}
close(jobsCh)
for range codes {
gauge := <-resultsCh
if gauge.isLocal {
local = append(local, gauge)
}
}
close(resultsCh)
for _, gauge := range local {
var gauges core.Gauge
if fedInfo, ok := federal[gauge.federalCode]; ok {
gauges = fedInfo
gauges.Name = gauge.name + " (" + gauge.code + ")"
gauges.Code = gauge.code
gauges.URL = "https://www.cehq.gouv.qc.ca/suivihydro/graphique.asp?NoStation=" + gauge.code
} else {
s.GetLogger().WithFields(logrus.Fields{
"script": s.name,
"command": "harvest",
"federalCode": gauge.federalCode,
"code": gauge.code,
}).Warn("no federal reference found")
gauges = core.Gauge{
GaugeID: core.GaugeID{
Code: gauge.code,
Script: s.name,
},
Name: gauge.name + " (" + gauge.code + ")",
FlowUnit: "m3/s",
LevelUnit: "m",
URL: "https://www.cehq.gouv.qc.ca/suivihydro/graphique.asp?NoStation=" + gauge.code,
}
}
result = append(result, gauges)
}
return
}
func (s *scriptQuebec) Harvest(ctx context.Context, recv chan<- *core.Measurement, errs chan<- error, codes core.StringSet, since int64) {
defer close(recv)
defer close(errs)
code, err := codes.Only()
if err != nil {
errs <- err
return
}
s.getReadings(recv, errs, code)
}
func (s *scriptQuebec) stationWorker(codes <-chan string, results chan<- stationInfo) {
for code := range codes {
gauge, err := s.parsePage(code)
if err != nil {
s.GetLogger().WithFields(logrus.Fields{
"script": s.name,
"command": "harvest",
"code": code,
}).Error(err)
}
results <- *gauge
}
}