-
Notifications
You must be signed in to change notification settings - Fork 500
/
client.go
82 lines (70 loc) · 2.33 KB
/
client.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
package client
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/cloudquery/cloudquery/plugins/destination/azblob/client/spec"
"github.com/cloudquery/filetypes/v4"
"github.com/cloudquery/plugin-sdk/v4/plugin"
"github.com/cloudquery/plugin-sdk/v4/writers/streamingbatchwriter"
"github.com/rs/zerolog"
)
type Client struct {
plugin.UnimplementedSource
streamingbatchwriter.IgnoreMigrateTable
streamingbatchwriter.UnimplementedDeleteStale
streamingbatchwriter.UnimplementedDeleteRecords
logger zerolog.Logger
spec *spec.Spec
*filetypes.Client
writer *streamingbatchwriter.StreamingBatchWriter
storageClient *azblob.Client
}
func New(ctx context.Context, logger zerolog.Logger, s []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
c := &Client{
logger: logger.With().Str("module", "azb").Logger(),
}
if opts.NoConnection {
return c, nil
}
if err := json.Unmarshal(s, &c.spec); err != nil {
return nil, fmt.Errorf("failed to unmarshal azblob spec: %w", err)
}
if err := c.spec.Validate(); err != nil {
return nil, err
}
c.spec.SetDefaults()
filetypesClient, err := filetypes.NewClient(&c.spec.FileSpec)
if err != nil {
return nil, fmt.Errorf("failed to create filetypes client: %w", err)
}
c.Client = filetypesClient
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, fmt.Errorf("failed to create Azure credential: %w", err)
}
c.storageClient, err = azblob.NewClient("https://"+c.spec.StorageAccount+".blob.core.windows.net", cred, nil)
if err != nil {
return nil, fmt.Errorf("failed to create Azure storage client: %w", err)
}
_, err = c.storageClient.UploadStream(ctx, c.spec.Container, "cq-test-file", strings.NewReader(""), nil)
if err != nil {
return nil, fmt.Errorf("failed to write test file to Azure: %w", err)
}
c.writer, err = streamingbatchwriter.New(c,
streamingbatchwriter.WithLogger(c.logger),
streamingbatchwriter.WithBatchSizeRows(*c.spec.BatchSize),
streamingbatchwriter.WithBatchSizeBytes(*c.spec.BatchSizeBytes),
streamingbatchwriter.WithBatchTimeout(c.spec.BatchTimeout.Duration()),
)
if err != nil {
return nil, err
}
return c, nil
}
func (c *Client) Close(ctx context.Context) error {
return c.writer.Close(ctx)
}