/
manager.go
150 lines (126 loc) · 3.41 KB
/
manager.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
package writer
import (
"context"
"io"
"sync"
"cloud.google.com/go/bigquery/storage/apiv1/storagepb"
mw "cloud.google.com/go/bigquery/storage/managedwriter"
"github.com/google/uuid"
"github.com/googleapis/gax-go/v2/apierror"
"github.com/m-mizutani/goerr"
"github.com/m-mizutani/swarm/pkg/domain/types"
"github.com/m-mizutani/swarm/pkg/utils"
"google.golang.org/protobuf/types/descriptorpb"
)
type Manager struct {
fac *factory
currentWriter *writer
mutex sync.Mutex
closeWG sync.WaitGroup
}
type factory struct {
mwClient *mw.Client
proto *descriptorpb.DescriptorProto
projectID types.GoogleProjectID
datasetID types.BQDatasetID
tableID types.BQTableID
}
func (x *factory) newWriter(ctx context.Context) (*writer, error) {
ms, err := x.mwClient.NewManagedStream(ctx,
mw.WithDestinationTable(
mw.TableParentFromParts(
x.projectID.String(),
x.datasetID.String(),
x.tableID.String(),
),
),
// mw.WithType(mw.CommittedStream),
mw.WithSchemaDescriptor(x.proto),
)
if err != nil {
return nil, goerr.Wrap(err, "failed to create managed stream")
}
w := &writer{
id: uuid.NewString(),
s: ms,
}
utils.CtxLogger(ctx).Debug("created new writer", "writer_id", w.id)
return w, nil
}
func NewManger(ctx context.Context, mwClient *mw.Client, proto *descriptorpb.DescriptorProto, projectID types.GoogleProjectID, datasetID types.BQDatasetID, tableID types.BQTableID) (*Manager, error) {
f := &factory{
mwClient: mwClient,
proto: proto,
projectID: projectID,
datasetID: datasetID,
tableID: tableID,
}
writer, err := f.newWriter(ctx)
if err != nil {
return nil, err
}
return &Manager{
fac: f,
currentWriter: writer,
}, nil
}
type writer struct {
id string
s *mw.ManagedStream
wg sync.WaitGroup
}
func (x *Manager) Renew(ctx context.Context) error {
x.mutex.Lock()
defer x.mutex.Unlock()
newWriter, err := x.fac.newWriter(ctx)
if err != nil {
return err
}
oldWriter := x.currentWriter
utils.CtxLogger(ctx).Debug("renewing writer", "writer_id", oldWriter.id)
x.closeWG.Add(1)
go func() {
defer x.closeWG.Done()
oldWriter.wg.Wait()
utils.SafeClose(oldWriter.s)
utils.CtxLogger(ctx).Debug("closed writer", "writer_id", oldWriter.id)
}()
x.currentWriter = newWriter
return nil
}
func (x *Manager) Close() error {
x.mutex.Lock()
defer x.mutex.Unlock()
if err := x.currentWriter.s.Close(); err != nil && err != io.EOF {
return goerr.Wrap(err, "failed to close managed stream").With("writer_id", x.currentWriter.id)
}
x.currentWriter = nil
x.closeWG.Wait()
return nil
}
func (x *Manager) Writer(ctx context.Context) *writer {
x.mutex.Lock()
defer x.mutex.Unlock()
x.currentWriter.wg.Add(1)
return x.currentWriter
}
func (x *writer) Append(ctx context.Context, rows [][]byte) error {
arResult, err := x.s.AppendRows(ctx, rows)
if err != nil {
return goerr.Wrap(err, "failed to append rows")
}
if _, err := arResult.FullResponse(ctx); err != nil {
if apiErr, ok := apierror.FromError(err); ok {
storageErr := &storagepb.StorageError{}
if e := apiErr.Details().ExtractProtoMessage(storageErr); e == nil && storageErr.Code == storagepb.StorageError_SCHEMA_MISMATCH_EXTRA_FIELDS {
utils.CtxLogger(ctx).Debug("schema does not matched, should retry")
return types.ErrSchemaNotMatched
}
}
return goerr.Wrap(err, "failed to get append result")
}
return nil
}
func (x *writer) Release() {
x.wg.Done()
}