-
Notifications
You must be signed in to change notification settings - Fork 307
/
bqstreammanager.go
159 lines (136 loc) · 4.8 KB
/
bqstreammanager.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
//go:generate mockgen -destination=../../../mocks/services/streammanager/bqstream/mock_bqstream.go -package mock_bqstream github.com/rudderlabs/rudder-server/services/streammanager/bqstream BQClient
package bqstream
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"cloud.google.com/go/bigquery"
backendconfig "github.com/rudderlabs/rudder-server/config/backend-config"
"github.com/rudderlabs/rudder-server/services/streammanager/common"
"github.com/rudderlabs/rudder-server/utils/googleutils"
"github.com/rudderlabs/rudder-server/utils/logger"
"github.com/tidwall/gjson"
gbq "google.golang.org/api/bigquery/v2"
"google.golang.org/api/option"
)
type Config struct {
Credentials string `json:"credentials"`
ProjectId string `json:"projectId"`
DatasetId string `json:"datasetId"`
TableId string `json:"tableId"`
}
// https://stackoverflow.com/questions/55951812/insert-into-bigquery-without-a-well-defined-struct
type GenericRecord map[string]bigquery.Value
type BQClient interface {
Put(ctx context.Context, datasetID, tableID string, records []*GenericRecord) error
Close() error
}
type BQStreamProducer struct {
Opts common.Opts
Client BQClient
}
type Client struct {
bqClient *bigquery.Client
}
func (c *Client) Put(ctx context.Context, datasetID, tableID string, records []*GenericRecord) error {
bqInserter := c.bqClient.Dataset(datasetID).Table(tableID).Inserter()
return bqInserter.Put(ctx, records)
}
func (c *Client) Close() error {
return c.bqClient.Close()
}
func (rec GenericRecord) Save() (map[string]bigquery.Value, string, error) {
var insertID string
if columnVal, isInsertIdPresent := rec["insertId"]; isInsertIdPresent {
insertID = columnVal.(string)
delete(rec, "insertId")
}
return rec, insertID, nil
}
var pkgLogger logger.Logger
func Init() {
pkgLogger = logger.NewLogger().Child("streammanager").Child("bqstream")
}
func init() {
Init()
}
func NewProducer(destination *backendconfig.DestinationT, o common.Opts) (*BQStreamProducer, error) {
var config Config
jsonConfig, err := json.Marshal(destination.Config)
if err != nil {
return nil, fmt.Errorf("[BQStream] Error while marshalling destination config :: %w", err)
}
err = json.Unmarshal(jsonConfig, &config)
if err != nil {
return nil, createErr(err, "error in BQStream while unmarshalling destination config")
}
opts := []option.ClientOption{
option.WithScopes([]string{
gbq.BigqueryInsertdataScope,
}...),
}
if !googleutils.ShouldSkipCredentialsInit(config.Credentials) {
confCreds := []byte(config.Credentials)
if err = googleutils.CompatibleGoogleCredentialsJSON(confCreds); err != nil {
return nil, createErr(err, "incompatible credentials")
}
opts = append(opts, option.WithCredentialsJSON(confCreds))
}
bqClient, err := bigquery.NewClient(context.Background(), config.ProjectId, opts...)
if err != nil {
return nil, err
}
return &BQStreamProducer{Client: &Client{bqClient: bqClient}, Opts: o}, nil
}
func (producer *BQStreamProducer) Produce(jsonData json.RawMessage, _ interface{}) (statusCode int, respStatus, responseMessage string) {
client := producer.Client
if client == nil {
return http.StatusBadRequest, "Failure", "[BQStream] error :: invalid client"
}
parsedJSON := gjson.ParseBytes(jsonData)
dsId := parsedJSON.Get("datasetId").String()
tblId := parsedJSON.Get("tableId").String()
props := parsedJSON.Get("properties")
var genericRecs []*GenericRecord
if props.IsArray() {
err := json.Unmarshal([]byte(props.String()), &genericRecs)
if err != nil {
return http.StatusBadRequest, "Failure", createErr(err, "error in unmarshalling data").Error()
}
} else {
var genericRec *GenericRecord
err := json.Unmarshal([]byte(props.String()), &genericRec)
if err != nil {
return http.StatusBadRequest, "Failure", createErr(err, "error in unmarshalling data").Error()
}
genericRecs = append(genericRecs, genericRec)
}
ctx, cancel := context.WithTimeout(context.Background(), producer.Opts.Timeout)
defer cancel()
err := client.Put(ctx, dsId, tblId, genericRecs)
if err != nil {
if ctx.Err() != nil && errors.Is(err, context.DeadlineExceeded) {
return http.StatusGatewayTimeout, "Failure", createErr(err, "timeout in data insertion").Error()
}
return http.StatusBadRequest, "Failure", createErr(err, "error in data insertion").Error()
}
return http.StatusOK, "Success", `[BQStream] Successful insertion of data`
}
func (producer *BQStreamProducer) Close() error {
client := producer.Client
if client == nil {
return createErr(nil, "error while trying to close the client")
}
err := client.Close()
if err != nil {
return createErr(err, "error while closing the client")
}
return err
}
func createErr(err error, msg string) error {
fmtMsg := fmt.Errorf("[BQStream] error :: %v:: %w", msg, err).Error()
pkgLogger.Errorf(fmtMsg)
return fmt.Errorf(fmtMsg)
}