Skip to content

fix(index): enforce RFC-109 vector read correctness #19513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chrevanthreddy wants to merge 8 commits into
base: master
Choose a base branch
from
Open

fix(index): enforce RFC-109 vector read correctness #19513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
694caef
feat(metadata): add MDT row families and payload schema for vector in…
chrevanthreddy Jul 17, 2026
1645cfb
fix(metadata): align vector schema with freshness contract
Aug 3, 2026
7aa9ea7
fix(metadata): finalize vector generation manifest format
Aug 4, 2026
d7dffb2
feat(index): add vector index option surface and validation
chrevanthreddy Jul 16, 2026
cc805d8
feat(index): add RaBitQ encoder and multibit-aware scorer contract
chrevanthreddy Jul 17, 2026
6e11f97
fix(index): version RaBitQ factor semantics
Aug 4, 2026
1b8c646
feat(metadata): vector search core - candidate generation, pruning an…
Jul 15, 2026
cc09550
fix(index): enforce vector read correctness contract
Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions hudi-common/src/main/avro/HoodieMetadata.avsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,147 @@
}
],
"default" : null
},
{
"name": "VectorIndexMetadata",
"doc": "Typed metadata records for the MDT-backed IVF + RaBitQ vector index.",
"type": [
"null",
{
"type": "record",
"name": "HoodieVectorIndexActiveManifest",
"fields": [
{"name": "indexVersion", "type": "int", "doc": "Persisted vector-index format version."},
{"name": "activeGeneration", "type": ["null", "int"], "default": null,
"doc": "Reader-visible generation ordinal, or null before first activation."}
]
},
{
"type": "record",
"name": "HoodieVectorIndexManifest",
"fields": [
{"name": "indexVersion", "type": "int"},
{"name": "generationId", "type": "string", "doc": "Creating Hudi instant + optional human tag. The gen ordinal lives in the key."},
{"name": "state", "type": "string", "doc": "BUILDING | ACTIVE | RETIRED. Readers serve only the generation referenced by the singleton active manifest."},
{"name": "dim", "type": "int"},
{"name": "dimPadded", "type": "int", "doc": "nextPow2(dim) for hadamard quantizers, else dim. Authoritative for codeRowBytes."},
{"name": "codeRowBytes", "type": "int", "doc": "ceil(dimPadded/64)*8. Long-aligned per-plane row width."},
{"name": "bitsTotal", "type": "int", "doc": "B = 1 + numExPlanes."},
{"name": "numExPlanes", "type": "int"},
{"name": "numClusters", "type": "int"},
{"name": "shardCount", "type": "int", "doc": "Initial per-cluster posting shard geometry frozen for this generation."},
{"name": "fileGroupCount", "type": "int", "doc": "MDT file-group geometry frozen for this generation; readers and writers never re-derive it."},
{"name": "metric", "type": "string", "doc": "L2 | DOT | COSINE."},
{"name": "assumeNormalized", "type": "boolean"},
{"name": "residualEncoding", "type": "boolean", "default": false, "doc": "True when posting codes encode x - centroid and readers must use residual scoring."},
{"name": "vectorColumn", "type": "string", "doc": "Authoritative base-table vector column used for bootstrap and exact rerank."},
{"name": "targetBlockBytes", "type": "int", "doc": "Config input, e.g. 524288."},
{"name": "vectorsPerBlock", "type": "int", "doc": "Resolved N. Frozen per generation; readers never re-derive."},
{"name": "blockFormatVersion", "type": "int", "doc": "Expected posting-block layout version for this generation."},
{"name": "factorVersion", "type": "int", "doc": "RaBitQ scalar-factor semantics version. Readers reject unsupported values."},
{"name": "kappa", "type": "double", "doc": "Generation-scoped pass-1 error scale."},
{"name": "gMin", "type": "double", "doc": "Generation-scoped normalized alignment floor."},
{"name": "eps1Max", "type": "double", "doc": "Maximum permitted relative pass-1 error."},
{"name": "epsNRel", "type": "double", "doc": "Relative residual-norm floor."},
{"name": "centroidChunkCount", "type": "int", "doc": "Expected number of size-bounded centroid chunks."},
{"name": "centroidChecksum", "type": ["null", "string"], "default": null,
"doc": "Checksum of the canonical centroid payload used to reject incomplete or mismatched builds."},
{"name": "splitLimit", "type": "int"},
{"name": "mergeFloor", "type": "int"},
{"name": "bootstrapInstant", "type": ["null", "string"], "default": null,
"doc": "Source data-write instant included by the generation bootstrap baseline."},
{"name": "verifiedFrontier", "type": ["null", "string"], "default": null,
"doc": "Latest source data-write instant with verified contiguous marker coverage from the bootstrap baseline."},
{"name": "createdTs", "type": "long"}
]
},
{
"type": "record",
"name": "HoodieVectorIndexQuantizer",
"fields": [
{"name": "quantizerType", "type": "string", "doc": "rb-rotation-explicit | rb-rotation-hadamard."},
{"name": "randomSeed", "type": "long"},
{"name": "rotationBytes", "type": ["null", "bytes"], "default": null,
"doc": "Explicit path: rows of the D x D float32 LE orthogonal matrix. Null for hadamard."}
]
},
{
"type": "record",
"name": "HoodieVectorIndexCentroids",
"fields": [
{"name": "clusterIds", "type": "bytes", "doc": "k x u32 LE cluster ids in this chunk, parallel to centroidBytes rows."},
{"name": "centroidBytes", "type": "bytes", "doc": "k x dimPadded float32 LE, row-major, in rotated space."},
{"name": "clusterRadii", "type": "bytes", "doc": "k x float32 LE max residual norm per cluster."}
]
},
{
"type": "record",
"name": "HoodieVectorIndexPostingBlock",
"fields": [
{"name": "blockFormatVersion", "type": "int"},
{"name": "numVectors", "type": "int"},
{"name": "codeRowBytes", "type": "int", "doc": "Duplicated from manifest for self-description; MUST match."},
{"name": "signPlane", "type": "bytes", "doc": "S1: N x codeRowBytes, row-major. Bit plane B-1 (MSB)."},
{"name": "exPlanes", "type": "bytes", "doc": "S2: N x numExPlanes x codeRowBytes. Vector-major; planes MSB-first."},
{"name": "scalarFactors", "type": "bytes", "doc": "S3: fAdd1 | fRescale1 | err1 | fAddEx | fRescaleEx | residualNorm | optional vectorNorm, float32 LE SoA."},
{"name": "rowLocators", "type": "bytes", "doc": "S4: N x 8 B LE structs: fgDictIdx u16 | instantDictIdx u16 | rowPosition u32."},
{"name": "fileGroupDict", "type": {"type": "array", "items": "string"}},
{"name": "instantTimeDict", "type": {"type": "array", "items": "string"}},
{"name": "partitionDict", "type": {"type": "array", "items": "string"}, "doc": "Parallel indexing with fileGroupDict."},
{"name": "recordKeyOffsets", "type": "bytes", "doc": "S6a: (N+1) x u32 LE offsets into recordKeyBytes."},
{"name": "recordKeyBytes", "type": "bytes", "doc": "S6b: concatenated UTF-8 keys. Must remain final field."}
]
},
{
"type": "record",
"name": "HoodieVectorIndexPostingDelta",
"fields": [
{"name": "recordKey", "type": "string"},
{"name": "binaryCode", "type": "bytes", "doc": "MSB row followed by ex rows, identical row encoding and plane order as blocks."},
{"name": "fAdd1", "type": "float"},
{"name": "fRescale1", "type": "float"},
{"name": "err1", "type": "float"},
{"name": "fAddEx", "type": "float"},
{"name": "fRescaleEx", "type": "float"},
{"name": "residualNorm", "type": "float"},
{"name": "vectorNorm", "type": ["null", "float"], "default": null},
{"name": "fileGroupId", "type": "string"},
{"name": "partitionPath", "type": "string"},
{"name": "baseInstantTime", "type": "string"},
{"name": "rowPosition", "type": "long"}
]
},
{
"type": "record",
"name": "HoodieVectorIndexClusterStats",
"fields": [
{"name": "routingVersion", "type": "int", "doc": "Cache invalidation version for atomically published cluster routing changes."},
{"name": "shardCount", "type": "int"},
{"name": "fileGroupIds", "type": {"type": "array", "items": "string"}, "doc": "Candidate base-table file groups for this cluster."},
{"name": "liveCount", "type": "long"},
{"name": "deltaCount", "type": "long"},
{"name": "tombstoneCount", "type": "long"},
{"name": "lastRebalanceInstant", "type": ["null", "string"], "default": null},
{"name": "lastUpdatedTs", "type": "long"}
]
},
{
"type": "record",
"name": "HoodieVectorIndexSourceInstantMarker",
"fields": [
{"name": "dataInstant", "type": "string", "doc": "Completed source data-write instant incorporated by this generation."}
]
},
{
"type": "record",
"name": "HoodieVectorIndexTombstone",
"fields": [
{"name": "deleteInstant", "type": "string"},
{"name": "deleteReason", "type": "string", "doc": "SPLIT | MERGE | REBALANCE | GC | MANUAL."}
]
}
],
"default" : null
}
]
}
Loading