-
Notifications
You must be signed in to change notification settings - Fork 0
/
bigquery.go
59 lines (49 loc) · 1.53 KB
/
bigquery.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
//https://cloud.google.com/bigquery/docs/loading-data-local
//https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries
package adapters
import (
"context"
"github.com/masterxavierfox/goetl-postgres-bigquery/config"
log "github.com/sirupsen/logrus"
"google.golang.org/api/option"
"os"
"cloud.google.com/go/bigquery"
)
func bQUpload(datasetID, tableID, filename string) error{
config.LoadEnv()
credsFile := config.GOOGLE_APPLICATION_CREDENTIALS
//creds, err := ioutil.ReadFile(credsFile)
//if err != nil {
// log.Fatalf("Could note parse config file.")
//}
log.Info(":::: ========== BIG QUERY UPLOAD STARTED ========= ::::")
log.Info("\n -::Dataset ID | ",datasetID,"\n -::Table ID | ",tableID,"\n -::CSV File | ",filename,"\n -::PROJECT ID | ",config.BQ_PROJECT_ID,"\n")
ctx := context.Background()
client, err := bigquery.NewClient(ctx, config.BQ_PROJECT_ID,option.WithCredentialsFile(credsFile))
if err != nil {
return err
}
// [START big-query_load_from_file]
f, err := os.Open(filename)
if err != nil {
return err
}
source := bigquery.NewReaderSource(f)
source.AutoDetect = true // Allow BigQuery to determine schema.
source.SkipLeadingRows = 1 // CSV has a single header line.
loader := client.Dataset(datasetID).Table(tableID).LoaderFrom(source)
job, err := loader.Run(ctx)
if err != nil {
return err
}
status, err := job.Wait(ctx)
if err != nil {
return err
}
if err := status.Err(); err != nil {
return err
}
// [END bigquery_load_from_file]
DeleteArchive(filename)
return nil
}