-
Notifications
You must be signed in to change notification settings - Fork 5
/
cargo.go
95 lines (75 loc) · 2.43 KB
/
cargo.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
package ingestors
import (
"errors"
"io"
"time"
"github.com/buger/jsonparser"
log "github.com/sirupsen/logrus"
"github.com/librariesio/depper/data"
)
const cargoSchedule = "*/5 * * * *"
const cargoFeed = "https://crates.io/api/v1/summary"
type Cargo struct {
LatestRun time.Time
}
func NewCargo() *Cargo {
return &Cargo{}
}
func (ingestor *Cargo) Schedule() string {
return cargoSchedule
}
func (ingestor *Cargo) Ingest() []data.PackageVersion {
packages := ingestor.ingestURL(cargoFeed)
ingestor.LatestRun = time.Now()
return packages
}
func (ingestor *Cargo) ingestURL(url string) []data.PackageVersion {
var results []data.PackageVersion
response, err := depperGetUrl(url)
if err != nil {
log.WithFields(log.Fields{"ingestor": "cargo", "error": err}).Error()
return results
}
defer response.Body.Close()
body, _ := io.ReadAll(response.Body)
err = jsonparser.ObjectEach(body, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
var subErr error
if string(key) == "errors" {
// there can be multiple errors returned based on the API response schema
// we can concatenate the messages together to return in a Go Error object but log them out individually
var totalErrorMessage = ""
_, subErr = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
errorMessage, _ := jsonparser.GetString(value, "detail")
totalErrorMessage += "|" + errorMessage
log.WithFields(log.Fields{"ingestor": "cargo", "error": errorMessage}).Error()
})
if subErr == nil {
subErr = errors.New(totalErrorMessage)
}
}
if string(key) == "just_updated" || string(key) == "new_crates" {
_, subErr = jsonparser.ArrayEach(value, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
name, _ := jsonparser.GetString(value, "name")
version, _ := jsonparser.GetString(value, "newest_version")
createdAt, _ := jsonparser.GetString(value, "updated_at")
createdAtTime, _ := time.Parse(time.RFC3339, createdAt)
discoveryLag := time.Since(createdAtTime)
results = append(
results,
data.PackageVersion{
Platform: "cargo",
Name: name,
Version: version,
CreatedAt: createdAtTime,
DiscoveryLag: discoveryLag,
},
)
})
}
return subErr
})
if err != nil {
log.WithFields(log.Fields{"ingestor": "cargo", "error": err}).Error()
}
return results
}