/
harvest.go
89 lines (79 loc) · 1.62 KB
/
harvest.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
package annote
import (
"log"
"time"
)
//
// Harvester
//
var (
harvestControl chan int
// should have a mutex protecting it
harvestStatus int
)
const (
HNow = iota
HExit
StatusWaiting = iota
StatusHarvesting
)
func BackgroundHarvester() {
var lastHarvest time.Time
var harvestInterval time.Duration
// we store our config as a special "curate item"
config, err := Datasource.FindItem("system")
if err == nil {
s := config.FirstField("last-harvest")
if s != "" {
lastHarvest, _ = time.Parse(time.RFC3339, s)
}
s = config.FirstField("harvest-interval")
if s != "" {
harvestInterval, _ = time.ParseDuration(s)
}
}
harvestControl = make(chan int, 100)
for {
harvestStatus = StatusWaiting
var timer <-chan time.Time
if harvestInterval > 0 {
timer = time.After(harvestInterval)
}
select {
case msg := <-harvestControl:
if msg == HExit {
return
}
case <-timer:
}
log.Println("Start Harvest since", lastHarvest)
harvestStatus = StatusHarvesting
t := time.Now()
c := make(chan CurateItem, 10)
go func() {
for item := range c {
err := Datasource.IndexItem(item)
if err != nil {
log.Println(err)
}
SearchEngine.IndexRecord(item)
}
}()
err := HarvestCurateObjects(TargetFedora, lastHarvest, func(item CurateItem) error {
c <- item
return nil
})
if err != nil {
log.Println(err)
} else {
lastHarvest = t
config, err := Datasource.FindItem("system")
if err == nil {
config.RemoveAll("last-harvest")
config.Add("last-harvest", t.Format(time.RFC3339))
Datasource.IndexItem(config)
}
}
log.Println("Finish Harvest")
}
}