/
copy_direct.go
94 lines (79 loc) · 2.39 KB
/
copy_direct.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package datablobstorage
import (
"context"
"io"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/molt/dbtable"
"github.com/cockroachdb/molt/fetch/internal/dataquery"
"github.com/cockroachdb/molt/testutils"
"github.com/jackc/pgx/v5"
"github.com/rs/zerolog"
)
// copyCRDBDirect represents a store in which any output is directly input
// into CockroachDB, instead of storing it as an intermediate file.
// This is only compatible with "COPY", and does not utilise IMPORT.
type copyCRDBDirect struct {
logger zerolog.Logger
target *pgx.Conn
}
const DirectCopyWriterMockErrMsg = "forced error for direct copy"
func (c *copyCRDBDirect) CreateFromReader(
ctx context.Context,
r io.Reader,
table dbtable.VerifiedTable,
iteration int,
fileExt string,
numRows chan int,
testingKnobs testutils.FetchTestingKnobs,
shardNum int,
) (Resource, error) {
// Drain the channel so we don't block.
go func() {
<-numRows
}()
conn, err := pgx.ConnectConfig(ctx, c.target.Config())
if err != nil {
return nil, err
}
// Set the session variables required for COPY
if err := SetCopyEnvVars(ctx, conn); err != nil {
return nil, err
}
if testingKnobs.FailedWriteToBucket.FailedBeforeReadFromPipe {
return nil, errors.New(DirectCopyWriterMockErrMsg)
}
c.logger.Debug().Int("batch", iteration).Msgf("csv batch starting")
copyFromQuery := dataquery.CopyFrom(table, false /*skipHeader*/)
c.logger.Debug().Msgf("running copy from query: %s", copyFromQuery)
if _, err := conn.PgConn().CopyFrom(ctx, r, copyFromQuery); err != nil {
return nil, errors.CombineErrors(err, conn.Close(ctx))
}
if testingKnobs.FailedWriteToBucket.FailedAfterReadFromPipe {
return nil, errors.New(DirectCopyWriterMockErrMsg)
}
c.logger.Debug().Int("batch", iteration).Msgf("csv batch complete")
return nil, conn.Close(ctx)
}
func (c *copyCRDBDirect) ListFromContinuationPoint(
ctx context.Context, table dbtable.VerifiedTable, fileName string,
) ([]Resource, error) {
return nil, nil
}
func (c *copyCRDBDirect) CanBeTarget() bool {
return false
}
func (c *copyCRDBDirect) DefaultFlushBatchSize() int {
return 1 * 1024 * 1024
}
func (c *copyCRDBDirect) Cleanup(ctx context.Context) error {
return nil
}
func (c *copyCRDBDirect) TelemetryName() string {
return "copy_direct"
}
func NewCopyCRDBDirect(logger zerolog.Logger, target *pgx.Conn) *copyCRDBDirect {
return ©CRDBDirect{
logger: logger,
target: target,
}
}