forked from prometheus-community/json_exporter
-
Notifications
You must be signed in to change notification settings - Fork 0
/
google.go
216 lines (183 loc) · 5.33 KB
/
google.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
package providers
import (
"cloud.google.com/go/bigquery"
"context"
"encoding/json"
"fmt"
"github.com/ines-cruz/json_exporter/config"
"github.com/pkg/errors"
pconfig "github.com/prometheus/common/config"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"io"
"io/ioutil"
"math"
"net/http"
"os"
"strconv"
"strings"
"time"
)
func GetGoogle(config config.Config, ctx context.Context, endpoint string) ([]byte, error) {
httpClientConfig := config.HTTPClientConfig
client2, err := pconfig.NewClientFromConfig(httpClientConfig, "fetch_json", true, false)
if err != nil {
fmt.Println("Error generating HTTP client")
return nil, err
}
// GCP
//Create client
//Name of the Google BigQuery DB
env := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
client, err := bigquery.NewClient(ctx, "billing-cern", option.WithCredentialsFile(env))
if err != nil {
fmt.Printf("Client create error: %v\n", err)
}
row := client.Query(`SELECT cost, sku.description, system_labels, project.id, usage_start_time, usage_end_time FROM sbsl_cern_billing_info.gcp_billing_export_v1_012C54_B3DAFC_973FAF WHERE project.id IS NOT NULL AND project.id!="billing-cern" ORDER BY project.id, usage_start_time`)
rows, err := row.Read(ctx)
if err != nil {
fmt.Printf("Error2: %v\n", err)
}
var ex = printResults(rows)
thisMap := make(map[string]([]map[string]interface{}))
thisMap["values"] = ex
file, _ := json.MarshalIndent(thisMap, "", "")
_ = ioutil.WriteFile("examples/output.json", file, 0644)
req, err := http.NewRequest("GET", endpoint, nil)
req = req.WithContext(ctx)
if err != nil {
fmt.Println("Failed to create request")
return nil, err
}
for key, value := range config.Headers {
req.Header.Add(key, value)
}
if req.Header.Get("Accept") == "" {
req.Header.Add("Accept", "application/json")
}
resp, err := client2.Do(req)
if err != nil {
return nil, err
}
defer func() {
if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
fmt.Println("Failed to discard body", "err", err)
}
_ = resp.Body.Close()
}()
if resp.StatusCode/100 != 2 {
return nil, errors.New(resp.Status)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return data, nil
}
// printResults prints results from a query to the _ public dataset.
func printResults(iter *bigquery.RowIterator) []map[string]interface{} {
var sumCost float64
var sumGPU float64
var sumCores float64
var sumMem float64
var sumVM float64
var projectid string
var previous string
var startTime time.Time
var endTime time.Time
var data = []map[string]interface{}{}
d := map[string]interface{}{}
for {
previous = projectid
var row []bigquery.Value
err := iter.Next(&row)
if err == iterator.Done {
break
}
if len(row) != 0 {
startTime = row[4].(time.Time)
if checkDate(startTime) {
projectid = row[3].(string)
if !verifyID(previous, projectid) && len(d) > 0 {
data = append(data, d)
sumVM = 0
sumMem = 0
sumCores = 0
sumGPU = 0
d = map[string]interface{}{}
}
sumCost = getSum(row, sumCost, projectid, previous)
endTime = row[5].(time.Time)
var valGPU = fmt.Sprint(row[1])
if verifyStrings(valGPU, "GPU") {
sumGPU = getDuration(projectid, previous, sumGPU, endTime, startTime)
}
system_labels := row[2].([]bigquery.Value)
if len(system_labels) > 0 {
var valCores = fmt.Sprint(system_labels[0])
if verifyStrings(valCores, "cores") {
sumCores = getDuration(projectid, previous, sumCores, endTime, startTime)
}
var valVM = fmt.Sprint(system_labels[1])
if verifyStrings(valVM, "machine_spec") {
sumVM = getVM(sumVM, previous, projectid)
}
var valMem = fmt.Sprint(system_labels[2])
if verifyStrings(valMem, "memory") {
sumMem = getMem(system_labels, sumMem, previous, projectid)
}
}
d["month"] = startTime.Format("01-2006")
d["amountSpent"] = math.Round(sumCost*100) / 100
d["numberVM"] = sumVM
d["memoryMB"] = sumMem
d["CPUh"] = sumCores
d["GPUh"] = sumGPU
d["projectid"] = projectid
}
}
}
data = append(data, d)
return data
}
func checkDate(startDate time.Time) bool {
return startDate.Month() == time.Now().Month() && startDate.Year() == time.Now().Year()
}
func verifyID(prev string, id string) bool {
return strings.EqualFold(prev, id)
}
func verifyStrings(toCompare string, real string) bool {
return strings.Contains(toCompare, real)
}
func getSum(row []bigquery.Value, sumCost float64, id string, prev string) float64 {
if verifyID(prev, id) { //if we are still in the same project continue
return row[0].(float64) + sumCost
}
return row[0].(float64)
}
func getDuration(projectid string, previous string, sum float64, endTime time.Time, startTime time.Time) float64 {
if verifyID(previous, projectid) {
diff := endTime.Sub(startTime).Hours() + sum
return diff
}
return sum
}
func getVM(sumVM float64, prev string, id string) float64 {
if verifyID(prev, id) {
return sumVM + 1
}
return 0
}
func verifyMem(valMem string) float64 {
example, err := strconv.ParseFloat(fmt.Sprint(valMem[len(valMem)-5:len(valMem)-1]), 64)
if err == nil {
// TODO: Handle error.
}
return example
}
func getMem(sys []bigquery.Value, sumMem float64, prev string, id string) float64 {
if verifyID(prev, id) {
return verifyMem(fmt.Sprint(sys[2])) + sumMem
}
return 0
}