Skip to content

[python] Support raw fallback for global indexes #8310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 4 commits into from
Jun 21, 2026
Merged

[python] Support raw fallback for global indexes #8310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c701653
[python] Support raw fallback for global indexes
JingsongLi Jun 21, 2026
bb2460b
[python] Fix vector search test lint
JingsongLi Jun 21, 2026
2c55047
[python] Add mixed raw fallback tests
JingsongLi Jun 21, 2026
1e49ae2
[python] Fix mixed vector test formatting
JingsongLi Jun 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,77 @@ public void testBtreeIndexWrite() throws Exception {
testBtreeIndexWriteNull();
}

@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testBtreeRawFallbackWrite() throws Exception {
RowType rowType =
RowType.of(
new DataType[] {DataTypes.STRING(), DataTypes.STRING()},
new String[] {"k", "v"});
Options options = new Options();
Path tablePath = new Path(warehouse.toString() + "/default.db/test_btree_raw_fallback");
LocalFileIO.create().delete(tablePath, true);
options.set(PATH, tablePath.toString());
options.set(ROW_TRACKING_ENABLED, true);
options.set(DATA_EVOLUTION_ENABLED, true);
options.set(GLOBAL_INDEX_ENABLED, true);
TableSchema tableSchema =
SchemaUtils.forceCommit(
new SchemaManager(LocalFileIO.create(), tablePath),
new Schema(
rowType.getFields(),
Collections.emptyList(),
Collections.emptyList(),
options.toMap(),
""));
AppendOnlyFileStoreTable table =
new AppendOnlyFileStoreTable(
FileIOFinder.find(tablePath),
tablePath,
tableSchema,
CatalogEnvironment.empty());

BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
try (BatchTableWrite write = writeBuilder.newWrite();
BatchTableCommit commit = writeBuilder.newCommit()) {
write.write(
GenericRow.of(BinaryString.fromString("k1"), BinaryString.fromString("v1")));
write.write(
GenericRow.of(BinaryString.fromString("k2"), BinaryString.fromString("v2")));
write.write(
GenericRow.of(BinaryString.fromString("k3"), BinaryString.fromString("v3")));
commit.commit(write.prepareCommit());
}

SortedGlobalIndexBuilder builder =
new SortedGlobalIndexBuilder(table, "btree").withIndexField("k");
try (BatchTableCommit commit = writeBuilder.newCommit()) {
commit.commit(
builder.build(
builder.scan()
.map(org.apache.paimon.utils.Pair::getValue)
.orElseThrow(
() ->
new IllegalStateException(
"Expected scan result when building index."))
.get(0),
IOManager.create(warehouse.toString())));
}

try (BatchTableWrite write = writeBuilder.newWrite();
BatchTableCommit commit = writeBuilder.newCommit()) {
write.write(
GenericRow.of(BinaryString.fromString("k4"), BinaryString.fromString("v4")));
commit.commit(write.prepareCommit());
}

List<IndexManifestEntry> indexEntries =
table.indexManifestFileReader().read(table.latestSnapshot().get().indexManifest);
assertThat(indexEntries)
.singleElement()
.matches(entry -> entry.indexFile().rowCount() == 3);
}

@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testBitmapIndexWrite() throws Exception {
Expand Down
79 changes: 78 additions & 1 deletion paimon-python/dev/run_mixed_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,29 @@ run_btree_index_test() {
fi
}

run_btree_raw_fallback_test() {
echo -e "${YELLOW}=== Running BTree Raw Fallback Test (Java Write, Python Read) ===${NC}"

cd "$PROJECT_ROOT"

echo "Running Maven test for JavaPyE2ETest.testBtreeRawFallbackWrite..."
if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testBtreeRawFallbackWrite -pl paimon-core -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
echo -e "${GREEN}✓ Java test completed successfully${NC}"
else
echo -e "${RED}✗ Java test failed${NC}"
return 1
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for JavaPyReadWriteTest.test_read_btree_raw_fallback..."
if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_read_btree_raw_fallback -v; then
echo -e "${GREEN}✓ Python test completed successfully${NC}"
return 0
else
echo -e "${RED}✗ Python test failed${NC}"
return 1
fi
}

run_bitmap_index_test() {
echo -e "${YELLOW}=== Step 6b: Running Bitmap Index Test (Java Write, Python Read) ===${NC}"

Expand Down Expand Up @@ -541,6 +564,32 @@ run_vindex_vector_test() {
fi
}

run_vindex_vector_raw_fallback_test() {
echo -e "${YELLOW}=== Running paimon-vindex Vector Raw Fallback Test (Java Write, Python Read) ===${NC}"

cd "$PROJECT_ROOT"

echo "Running Maven test for JavaPyE2ETest.testVindexVectorRawFallbackWrite..."
if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorRawFallbackWrite -pl paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
echo -e "${GREEN}✓ Java test completed successfully${NC}"
else
echo -e "${RED}✗ Java test failed${NC}"
return 1
fi
cd "$PAIMON_PYTHON_DIR"
if ! ensure_paimon_vindex; then
return 1
fi
echo "Running Python test for JavaPyReadWriteTest.test_read_vindex_vector_raw_fallback..."
if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_read_vindex_vector_raw_fallback -v; then
echo -e "${GREEN}✓ Python test completed successfully${NC}"
return 0
else
echo -e "${RED}✗ Python test failed${NC}"
return 1
fi
}

run_compact_conflict_test() {
echo -e "${YELLOW}=== Running Compact Conflict Test (Java Write Base, Python Shard Update + Java Compact) ===${NC}"

Expand Down Expand Up @@ -796,6 +845,7 @@ main() {
local java_read_result=0
local pk_dv_result=0
local btree_index_result=0
local btree_raw_fallback_result=0
local bitmap_index_result=0
local compressed_global_index_result=0
local compressed_text_result=0
Expand All @@ -804,6 +854,7 @@ main() {
local lumina_vector_result=0
local lumina_vector_btree_result=0
local vindex_vector_result=0
local vindex_vector_raw_fallback_result=0
local compact_conflict_result=0
local blob_compact_conflict_result=0
local blob_alter_compact_result=0
Expand Down Expand Up @@ -873,6 +924,13 @@ main() {

echo ""

# Run BTree raw fallback test (Java write indexed + unindexed rows, Python read)
if ! run_btree_raw_fallback_test; then
btree_raw_fallback_result=1
fi

echo ""

# Run Bitmap index test (Java write, Python read)
if ! run_bitmap_index_test; then
bitmap_index_result=1
Expand Down Expand Up @@ -967,9 +1025,16 @@ main() {
if ! run_vindex_vector_test; then
vindex_vector_result=1
fi

echo ""

if ! run_vindex_vector_raw_fallback_test; then
vindex_vector_raw_fallback_result=1
fi
else
echo -e "${YELLOW}⏭ Skipping paimon-vindex Vector Index Test (requires Python >= 3.9, current: $PYTHON_VERSION)${NC}"
vindex_vector_result=0
vindex_vector_raw_fallback_result=0
fi

echo ""
Expand Down Expand Up @@ -1073,6 +1138,12 @@ main() {
echo -e "${RED}✗ BTree Index Test (Java Write, Python Read): FAILED${NC}"
fi

if [[ $btree_raw_fallback_result -eq 0 ]]; then
echo -e "${GREEN}✓ BTree Raw Fallback Test (Java Write, Python Read): PASSED${NC}"
else
echo -e "${RED}✗ BTree Raw Fallback Test (Java Write, Python Read): FAILED${NC}"
fi

if [[ $bitmap_index_result -eq 0 ]]; then
echo -e "${GREEN}✓ Bitmap Index Test (Java Write, Python Read): PASSED${NC}"
else
Expand Down Expand Up @@ -1145,6 +1216,12 @@ main() {
echo -e "${RED}✗ paimon-vindex Vector Index Test (Java Write, Python Read): FAILED${NC}"
fi

if [[ $vindex_vector_raw_fallback_result -eq 0 ]]; then
echo -e "${GREEN}✓ paimon-vindex Vector Raw Fallback Test (Java Write, Python Read): PASSED${NC}"
else
echo -e "${RED}✗ paimon-vindex Vector Raw Fallback Test (Java Write, Python Read): FAILED${NC}"
fi

if [[ $compact_conflict_result -eq 0 ]]; then
echo -e "${GREEN}✓ Compact Conflict Test (Java Write+Compact, Python Read): PASSED${NC}"
else
Expand Down Expand Up @@ -1198,7 +1275,7 @@ main() {
# Clean up warehouse directory after all tests
cleanup_warehouse

if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $bitmap_index_result -eq 0 && $compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 && $tantivy_fulltext_result -eq 0 && $lumina_vector_result -eq 0 && $lumina_vector_btree_result -eq 0 && $vindex_vector_result -eq 0 && $compact_conflict_result -eq 0 && $blob_compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 && $data_evolution_result -eq 0 && $data_evolution_py_write_result -eq 0 && $java_variant_write_py_read_result -eq 0 && $py_variant_write_java_read_result -eq 0 && $vector_append_table_result -eq 0 && $vector_dedicated_java_write_result -eq 0 && $vector_dedicated_py_write_result -eq 0 && $multi_vector_dedicated_java_write_result -eq 0 && $multi_vector_dedicated_py_write_result -eq 0 && $row_format_result -eq 0 ]]; then
if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $btree_raw_fallback_result -eq 0 && $bitmap_index_result -eq 0 && $compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 && $tantivy_fulltext_result -eq 0 && $lumina_vector_result -eq 0 && $lumina_vector_btree_result -eq 0 && $vindex_vector_result -eq 0 && $vindex_vector_raw_fallback_result -eq 0 && $compact_conflict_result -eq 0 && $blob_compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 && $data_evolution_result -eq 0 && $data_evolution_py_write_result -eq 0 && $java_variant_write_py_read_result -eq 0 && $py_variant_write_java_read_result -eq 0 && $vector_append_table_result -eq 0 && $vector_dedicated_java_write_result -eq 0 && $vector_dedicated_py_write_result -eq 0 && $multi_vector_dedicated_java_write_result -eq 0 && $multi_vector_dedicated_py_write_result -eq 0 && $row_format_result -eq 0 ]]; then
echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability verified.${NC}"
return 0
else
Expand Down
19 changes: 19 additions & 0 deletions paimon-python/pypaimon/common/options/core_options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ class GlobalIndexColumnUpdateAction(str, Enum):
DROP_PARTITION_INDEX = "DROP_PARTITION_INDEX"


class GlobalIndexSearchMode(str, Enum):
FAST = "fast"
FULL = "full"
DETAIL = "detail"


class CoreOptions:
"""Core options for Paimon tables."""
# File format constants
Expand Down Expand Up @@ -618,6 +624,16 @@ class CoreOptions:
.with_description("Whether to enable global index for scan.")
)

GLOBAL_INDEX_SEARCH_MODE: ConfigOption[GlobalIndexSearchMode] = (
ConfigOptions.key("global-index.search-mode")
.enum_type(GlobalIndexSearchMode)
.default_value(GlobalIndexSearchMode.FAST)
.with_description(
"Search mode for global index queries. "
"Supported values are 'fast', 'full', and 'detail'."
)
)

GLOBAL_INDEX_THREAD_NUM: ConfigOption[int] = (
ConfigOptions.key("global-index.thread-num")
.int_type()
Expand Down Expand Up @@ -1109,6 +1125,9 @@ def commit_max_retry_wait(self) -> int:
def global_index_enabled(self, default=None):
return self.options.get(CoreOptions.GLOBAL_INDEX_ENABLED, default)

def global_index_search_mode(self):
return self.options.get(CoreOptions.GLOBAL_INDEX_SEARCH_MODE)

def global_index_thread_num(self) -> Optional[int]:
return self.options.get(CoreOptions.GLOBAL_INDEX_THREAD_NUM)

Expand Down
Loading
Loading