-
Notifications
You must be signed in to change notification settings - Fork 10
/
millers.go
86 lines (72 loc) · 2.22 KB
/
millers.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
package millers
import (
"fmt"
"log"
"time"
"github.com/gleanerio/gleaner/internal/config"
"github.com/gleanerio/gleaner/internal/millers/graph"
"github.com/minio/minio-go/v7"
"github.com/spf13/viper"
)
type Sources struct {
Name string
Logo string
URL string
Headless bool
// SitemapFormat string
// Active bool
}
// Millers is our main controller for calling the various milling paths we will
// do on the JSON-LD data graphs
func Millers(mc *minio.Client, v1 *viper.Viper) {
st := time.Now()
log.Printf("Miller start time: %s \n", st) // Log the time at start for the record
// Put the sources in the config file into a struct
//var domains []Sources
//err := v1.UnmarshalKey("sources", &domains)
//domains, err := config.GetSources(v1)
domains, err := config.GetActiveSources(v1)
if err != nil {
log.Println(err)
}
// Make an array "as" of active buckets to process...
as := []string{}
for i := range domains {
m := fmt.Sprintf("summoned/%s", domains[i].Name)
as = append(as, m)
log.Printf("Adding bucket to milling list: %s\n", m)
}
// Make array of prov buckets.. sad I have to do this.. I could just pass
// the domains and let each miller now pick where to get things from. I
// only had to add this due to the prov data not being in summoned
ap := []string{}
for i := range domains {
m := fmt.Sprintf("prov/%s", domains[i].Name)
ap = append(ap, m)
log.Printf("Adding bucket to prov building list: %s\n", m)
}
mcfg := v1.GetStringMapString("millers") // get the millers we want to run from the config file
// Graph is the miller to convert from JSON-LD to nquads with validation of well formed
// TODO none of these (graph, shacl, prov) deal with the returned error
if mcfg["graph"] == "true" {
for d := range as {
graph.GraphNG(mc, as[d], v1)
}
}
// if mcfg["shacl"] == "true" {
// for d := range as {
// shapes.ShapeNG(mc, as[d], v1)
// // shapes.SHACLMillObjects(mc, as[d], v1)
// }
// }
// if mcfg["prov"] == "true" {
// for d := range ap {
// graph.AssembleObjs(mc, ap[d], v1)
// }
// }
// Time report
et := time.Now()
diff := et.Sub(st)
log.Printf("Miller end time: %s \n", et)
log.Printf("Miller run time: %f \n", diff.Minutes())
}