-
Notifications
You must be signed in to change notification settings - Fork 5.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
lightning: support extracting source/schema/table name to specific column #37790
Changes from 13 commits
7c2407d
48d5230
9407ccf
c4d00bb
e441446
9fc6f36
e047d9b
ce289be
437f9db
c8a98ab
d50ecd1
43cb4f6
972acd7
01717b8
79d3d42
0b9baa5
65322cc
689f231
d5d2185
befefba
f0e2406
72f33fb
9aa4393
d52d9e2
64e56bd
d1d054b
5f0eb0e
91e4172
90d8aae
750dc95
81648e0
f8fca3c
3479f1e
c73e1e2
6847ebc
fd448f9
73647ed
e9a1763
bfe361c
66516b2
0c4d822
bdce0bb
8826f72
b9df587
9c1cc44
1b82d71
e88b373
18dc3ad
4978bd8
87ef359
525b3e6
112901a
1d8424b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -648,9 +648,13 @@ func (p *PreRestoreInfoGetterImpl) sampleDataFromTable( | |
} | ||
|
||
initializedColumns := false | ||
var columnPermutation []int | ||
var kvSize uint64 = 0 | ||
var rowSize uint64 = 0 | ||
var ( | ||
columnPermutation []int | ||
kvSize uint64 = 0 | ||
rowSize uint64 = 0 | ||
extendCols []string | ||
extendVals []types.Datum | ||
) | ||
rowCount := 0 | ||
dataKVs := p.encBuilder.MakeEmptyRows() | ||
indexKVs := p.encBuilder.MakeEmptyRows() | ||
|
@@ -675,6 +679,21 @@ outloop: | |
return 0.0, false, errors.Trace(err) | ||
} | ||
} | ||
extendCols = sampleFile.ExtendData.Columns | ||
extendColsMap := make(map[string]struct{}) | ||
for _, extendCol := range extendCols { | ||
extendColsMap[extendCol] = struct{}{} | ||
} | ||
if len(columnNames) > 0 && len(extendCols) > 0 { | ||
for _, c := range columnNames { | ||
delete(extendColsMap, c) | ||
} | ||
} | ||
for i, c := range extendCols { | ||
if _, ok := extendColsMap[c]; ok { | ||
extendVals = append(extendVals, types.NewStringDatum(sampleFile.ExtendData.Values[i])) | ||
lance6716 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
} | ||
initializedColumns = true | ||
} | ||
case io.EOF: | ||
|
@@ -684,7 +703,20 @@ outloop: | |
return 0.0, false, errors.Trace(err) | ||
} | ||
lastRow := parser.LastRow() | ||
if rowCount == 0 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe move this branch into There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we move this logic to line 685 together? |
||
lastRowLen := len(lastRow.Row) | ||
extendColsMap := make(map[string]int) | ||
for i, c := range extendCols { | ||
extendColsMap[c] = lastRowLen + i | ||
} | ||
for i, col := range tableInfo.Columns { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if downstream forget to created extended columns, we might raise a friendly error in this function |
||
if p, ok := extendColsMap[col.Name.O]; ok { | ||
columnPermutation[i] = p | ||
lance6716 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
} | ||
} | ||
rowCount++ | ||
lastRow.Row = append(lastRow.Row, extendVals...) | ||
|
||
var dataChecksum, indexChecksum verification.KVChecksum | ||
kvs, encodeErr := kvEncoder.Encode(logTask.Logger, lastRow.Row, lastRow.RowID, columnPermutation, sampleFile.Path, offset) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not sure if we should expose it to user by toml, because this seems a DM only feature request