-
Notifications
You must be signed in to change notification settings - Fork 453
/
main.go
119 lines (107 loc) · 3.26 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
// Copyright 2021 OpenSSF Scorecard Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package main implements the BQ transfer job.
package main
import (
"bytes"
"context"
"flag"
"fmt"
"io"
"log"
"net/http"
"github.com/ossf/scorecard/v4/cron/config"
"github.com/ossf/scorecard/v4/cron/data"
)
func transferDataToBq(ctx context.Context,
bucketURL, projectID, datasetName, tableName string, completionThreshold float64, webhookURL string,
summary *data.BucketSummary,
) error {
for _, shards := range summary.Shards() {
if shards.IsTransferred() || !shards.IsCompleted(completionThreshold) {
continue
}
shardFileURI := data.GetBlobFilename("shard-*", shards.CreationTime())
if err := startDataTransferJob(ctx,
bucketURL, shardFileURI, projectID, datasetName, tableName,
shards.CreationTime()); err != nil {
return fmt.Errorf("error during StartDataTransferJob: %w", err)
}
if err := shards.MarkTransferred(ctx, bucketURL); err != nil {
return fmt.Errorf("error during MarkTransferred: %w", err)
}
if webhookURL == "" {
continue
}
//nolint: noctx, gosec // variable URL is ok here.
resp, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(shards.Metadata()))
if err != nil {
return fmt.Errorf("error during http.Post to %s: %w", webhookURL, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading resp.Body: %w", err)
}
log.Printf("Returned status: %s %s", resp.Status, body)
}
return nil
}
func getBQConfig() (projectID, datasetName, tableName string, err error) {
projectID, err = config.GetProjectID()
if err != nil {
return projectID, datasetName, tableName, fmt.Errorf("error getting ProjectId: %w", err)
}
datasetName, err = config.GetBigQueryDataset()
if err != nil {
return projectID, datasetName, tableName, fmt.Errorf("error getting BigQuery dataset: %w", err)
}
tableName, err = config.GetBigQueryTable()
if err != nil {
return projectID, datasetName, tableName, fmt.Errorf("error getting BigQuery table: %w", err)
}
return
}
func main() {
ctx := context.Background()
flag.Parse()
if err := config.ReadConfig(); err != nil {
panic(err)
}
bucketURL, err := config.GetResultDataBucketURL()
if err != nil {
panic(err)
}
webhookURL, err := config.GetWebhookURL()
if err != nil {
panic(err)
}
projectID, datasetName, tableName, err := getBQConfig()
if err != nil {
panic(err)
}
completionThreshold, err := config.GetCompletionThreshold()
if err != nil {
panic(err)
}
summary, err := data.GetBucketSummary(ctx, bucketURL)
if err != nil {
panic(err)
}
if err := transferDataToBq(ctx,
bucketURL, projectID, datasetName, tableName, completionThreshold, webhookURL,
summary); err != nil {
panic(err)
}
}