/
upload.go
146 lines (133 loc) · 3.52 KB
/
upload.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"golang.org/x/net/context"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
)
func upload(ctx context.Context, w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
// Parse entire payload
var err error
var set []*point
dec := json.NewDecoder(r.Body)
for err == nil {
var ben point
if err = dec.Decode(&ben); err == nil {
set = append(set, &ben)
}
}
if err != io.EOF {
http.Error(w, "Invalid Data Format", http.StatusExpectationFailed)
return
}
if len(set) == 0 {
w.WriteHeader(http.StatusNonAuthoritativeInfo)
w.Write([]byte("Empty Data Set"))
return
}
now := time.Now()
// TODO (bign8): place batch metadata and get batch key
query := r.URL.Query()
batchKey, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Batch", nil), &batch{
Stamp: now,
Branch: query.Get("branch"),
Commit: query.Get("commit"),
Build: query.Get("build"),
BuildURL: query.Get("build_url"),
Tag: query.Get("tag"),
})
if err != nil {
log.Errorf(ctx, "Batch Put: %s", err)
}
// Determine set of benches needed to be stored
tree := make(trie)
benches := make([]*pair, 0, len(set))
for _, ben := range set {
tree.Add(strings.Split(ben.Suite, "/"))
benches = append(benches, &pair{
Key: datastore.NewKey(ctx, "Bench", ben.Name, 0, path2key(ctx, ben.Suite)),
Val: &bench{Name: ben.Name, Seen: now},
})
}
getAll(ctx, benches) // Don't care about errs (just pre-fetching so we don't lose attributes later)
newBenches := make([]*pair, 0, len(set))
for _, p := range benches {
if p.Val.(*bench).Seen == now {
newBenches = append(newBenches, p)
}
}
// walk the tree and build all the path objects
objs := tree.Walk(&pair{
Key: nil,
Val: &path{
Name: "",
Parent: nil,
},
}, func(val string, parent interface{}) interface{} {
par := parent.(*pair)
var pat string
if par.Key == nil {
pat = val
} else {
pat = key2path(par.Key) + "/" + val
}
return &pair{
Key: path2key(ctx, pat),
Val: &path{
Name: val,
Parent: par.Key,
},
}
})
paths := make([]*pair, len(objs))
for i, obj := range objs {
paths[i] = obj.(*pair)
}
// iterate paths and generate the points
points := make([]*pair, 0, len(set))
for _, pat := range set {
parent := datastore.NewKey(ctx, "Bench", pat.Name, 0, path2key(ctx, pat.Suite))
pat.Batch = batchKey
pat.Stamp = now
points = append(points, &pair{
Key: datastore.NewIncompleteKey(ctx, "Point", parent),
Val: pat,
})
}
paths = append(paths, points...)
// Lets store all this data and return to the user
err = putAll(ctx, append(paths, newBenches...))
if err != nil {
log.Errorf(ctx, "Error Storing Data: %s", err)
http.Error(w, "Problem Storing Data", http.StatusInternalServerError)
} else {
slug := strings.Join(tree.Prefix(3), "/") // <host>/<user>/<repo>
log.Infof(ctx, "Full Suite Prefix: %q", slug)
fmt.Fprintf(w, "Success! Avilable at http://%s/%s\n", r.Host, slug)
}
}
func putAll(ctx context.Context, list []*pair) error {
keys := make([]*datastore.Key, len(list))
vals := make([]interface{}, len(list))
for i, p := range list {
keys[i] = p.Key
vals[i] = p.Val
}
_, err := datastore.PutMulti(ctx, keys, vals)
return err
}
func getAll(ctx context.Context, list []*pair) error {
keys := make([]*datastore.Key, len(list))
vals := make([]interface{}, len(list))
for i, p := range list {
keys[i] = p.Key
vals[i] = p.Val
}
return datastore.GetMulti(ctx, keys, vals)
}