[lance] Fix ArrayIndexOutOfBoundsException when reading vector search results with Lance format

paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java

@@ -183,7 +183,8 @@ private RecordReader<InternalRow> createReader(
                                         dataFilePathFactory,
                                         needMergeFiles.get(0),
                                         formatBuilder,
-                                        rowRanges));
+                                        rowRanges,
+                                        readRowType));
 
             } else {
                 suppliers.add(
@@ -308,7 +309,8 @@ private DataEvolutionFileReader createUnionReader(
                                         bunch,
                                         dataFilePathFactory,
                                         formatReaderMapping,
-                                        rowRanges));
+                                        rowRanges,
+                                        readRowType));
             }
         }
 
@@ -330,7 +332,8 @@ private FileRecordReader<InternalRow> createFileReader(
             DataFilePathFactory dataFilePathFactory,
             DataFileMeta file,
             Builder formatBuilder,
-            List<Range> rowRanges)
+            List<Range> rowRanges,
+            RowType readRowType)
             throws IOException {
         String formatIdentifier = DataFilePathFactory.formatIdentifier(file.fileName());
         long schemaId = file.schemaId();
@@ -345,23 +348,25 @@ private FileRecordReader<InternalRow> createFileReader(
                                                 ? schema
                                                 : schemaFetcher.apply(schemaId)));
         return createFileReader(
-                partition, file, dataFilePathFactory, formatReaderMapping, rowRanges);
+                partition, file, dataFilePathFactory, formatReaderMapping, rowRanges, readRowType);
     }
 
     private RecordReader<InternalRow> createFileReader(
             BinaryRow partition,
             FieldBunch bunch,
             DataFilePathFactory dataFilePathFactory,
             FormatReaderMapping formatReaderMapping,
-            List<Range> rowRanges)
+            List<Range> rowRanges,
+            RowType readRowType)
             throws IOException {
         if (bunch.files().size() == 1) {
             return createFileReader(
                     partition,
                     bunch.files().get(0),
                     dataFilePathFactory,
                     formatReaderMapping,
-                    rowRanges);
+                    rowRanges,
+                    readRowType);
         }
         List<ReaderSupplier<InternalRow>> readerSuppliers = new ArrayList<>();
         for (DataFileMeta file : bunch.files()) {
@@ -394,7 +399,8 @@ private FileRecordReader<InternalRow> createFileReader(
             DataFileMeta file,
             DataFilePathFactory dataFilePathFactory,
             FormatReaderMapping formatReaderMapping,
-            List<Range> rowRanges)
+            List<Range> rowRanges,
+            RowType readRowType)
             throws IOException {
         RoaringBitmap32 selection = file.toFileSelection(rowRanges);
         FormatReaderContext formatReaderContext =

paimon-lance/src/main/java/org/apache/paimon/format/lance/jni/LanceReader.java

@@ -65,6 +65,8 @@ public LanceReader(
                             .filter(fileFieldNames::contains)
                             .collect(Collectors.toList());
             if (existingFields.isEmpty()) {
+                // Read at least one column to get the correct row count.
+                // ArrowBatchReader maps by name; unmatched projected fields become null.
                 existingFields = Collections.singletonList(fileFieldNames.iterator().next());
             }
             this.arrowReader = reader.readAll(existingFields, ranges, batchSize);

paimon-lance/src/test/java/org/apache/paimon/format/lance/LanceVectorSearchTest.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.lance;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.source.VectorSearchBuilderTest;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.TraceableFileIO;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.paimon.options.CatalogOptions.WAREHOUSE;
+
+/** Test vector search with Lance file format. */
+public class LanceVectorSearchTest extends VectorSearchBuilderTest {
+
+    @BeforeEach
+    public void beforeEach() throws Catalog.DatabaseAlreadyExistException {
+        database = "default";
+        warehouse = new Path(TraceableFileIO.SCHEME + "://" + tempPath.toString());
+        Options options = new Options();
+        options.set(WAREHOUSE, warehouse.toUri().toString());
+        CatalogContext context = CatalogContext.create(options, new TraceableFileIO.Loader(), null);
+        catalog = CatalogFactory.createCatalog(context);
+        catalog.createDatabase(database, true);
+    }
+
+    @Override
+    protected Schema schemaDefault() {
+        return Schema.newBuilder()
+                .column("id", DataTypes.INT())
+                .column("vec", new ArrayType(DataTypes.FLOAT()))
+                .option(CoreOptions.BUCKET.key(), "-1")
+                .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+                .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+                .option(CoreOptions.FILE_FORMAT.key(), "lance")
+                .option("test.vector.dimension", "2")
+                .option("test.vector.metric", "l2")
+                .build();
+    }
+
+    @Disabled("Cosine metric uses Tantivy index which requires Hadoop dependencies")
+    @Test
+    @Override
+    public void testVectorSearchWithCosineMetric() {}
+}