From fe4daef13ba50b4c28038ed3691758479c489b95 Mon Sep 17 00:00:00 2001 From: majin1102 Date: Fri, 15 Aug 2025 16:31:32 +0800 Subject: [PATCH 1/8] init --- .../com/lancedb/lance/TransactionTest.java | 773 +----------------- .../lancedb/lance/operation/AppendTest.java | 101 +++ .../lance/operation/DataReplacementTest.java | 155 ++++ .../lancedb/lance/operation/DeleteTest.java | 77 ++ .../lancedb/lance/operation/MergeTest.java | 197 +++++ .../lance/operation/OperationTestBase.java | 95 +++ .../lance/operation/OverwriteTest.java | 97 +++ .../lancedb/lance/operation/ProjectTest.java | 73 ++ .../lance/operation/ReserveFragmentsTest.java | 98 +++ .../lancedb/lance/operation/RestoreTest.java | 74 ++ .../lancedb/lance/operation/RewriteTest.java | 92 +++ .../lance/operation/UpdateConfigTest.java | 123 +++ .../lancedb/lance/operation/UpdateTest.java | 93 +++ 13 files changed, 1299 insertions(+), 749 deletions(-) create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/AppendTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/DeleteTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/OverwriteTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/ProjectTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/ReserveFragmentsTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/RestoreTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/RewriteTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/UpdateConfigTest.java create mode 100644 java/core/src/test/java/com/lancedb/lance/operation/UpdateTest.java diff --git a/java/core/src/test/java/com/lancedb/lance/TransactionTest.java b/java/core/src/test/java/com/lancedb/lance/TransactionTest.java index 825f07877f4..544db9f9aac 100644 --- a/java/core/src/test/java/com/lancedb/lance/TransactionTest.java +++ b/java/core/src/test/java/com/lancedb/lance/TransactionTest.java @@ -1,774 +1,49 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package com.lancedb.lance; -import com.lancedb.lance.file.LanceFileWriter; -import com.lancedb.lance.fragment.DataFile; -import com.lancedb.lance.ipc.LanceScanner; import com.lancedb.lance.operation.Append; -import com.lancedb.lance.operation.DataReplacement; -import com.lancedb.lance.operation.Delete; -import com.lancedb.lance.operation.Merge; -import com.lancedb.lance.operation.Overwrite; -import com.lancedb.lance.operation.Project; -import com.lancedb.lance.operation.ReserveFragments; -import com.lancedb.lance.operation.Restore; -import com.lancedb.lance.operation.Rewrite; -import com.lancedb.lance.operation.RewriteGroup; -import com.lancedb.lance.operation.Update; -import com.lancedb.lance.operation.UpdateConfig; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ArrowReader; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class TransactionTest { - public static final int TEST_FILE_FORMAT_MAJOR_VERSION = 2; - public static final int TEST_FILE_FORMAT_MINOR_VERSION = 0; - private static Dataset dataset; - - @BeforeAll - static void setup() {} - - @AfterAll - static void tearDown() { - // Cleanup resources used by the tests - if (dataset != null) { - dataset.close(); - } - } - - @Test - void testProjection(@TempDir Path tempDir) { - String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); - String datasetPath = tempDir.resolve(testMethodName).toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - assertEquals(testDataset.getSchema(), dataset.getSchema()); - List fieldList = new ArrayList<>(testDataset.getSchema().getFields()); - Collections.reverse(fieldList); - Transaction txn1 = - dataset - .newTransactionBuilder() - .operation(Project.builder().schema(new Schema(fieldList)).build()) - .build(); - try (Dataset committedDataset = txn1.commit()) { - assertEquals(1, txn1.readVersion()); - assertEquals(1, dataset.version()); - assertEquals(2, committedDataset.version()); - assertEquals(new Schema(fieldList), committedDataset.getSchema()); - fieldList.remove(1); - Transaction txn2 = - committedDataset - .newTransactionBuilder() - .operation(Project.builder().schema(new Schema(fieldList)).build()) - .build(); - try (Dataset committedDataset2 = txn2.commit()) { - assertEquals(2, txn2.readVersion()); - assertEquals(2, committedDataset.version()); - assertEquals(3, committedDataset2.version()); - assertEquals(new Schema(fieldList), committedDataset2.getSchema()); - assertEquals(txn1, committedDataset.readTransaction().orElse(null)); - assertEquals(txn2, committedDataset2.readTransaction().orElse(null)); - } - } - } - } - - @Test - void testAppend(@TempDir Path tempDir) { - String datasetPath = tempDir.resolve("testAppend").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - // Commit fragment - int rowCount = 20; - FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) - .transactionProperties(Collections.singletonMap("key", "value")) - .build(); - assertEquals("value", transaction.transactionProperties().get("key")); - try (Dataset dataset = transaction.commit()) { - assertEquals(2, dataset.version()); - assertEquals(2, dataset.latestVersion()); - assertEquals(rowCount, dataset.countRows()); - assertThrows( - IllegalArgumentException.class, - () -> - dataset - .newTransactionBuilder() - .operation(Append.builder().fragments(new ArrayList<>()).build()) - .build() - .commit() - .close()); - assertEquals(transaction, dataset.readTransaction().orElse(null)); - } - } - } - - @Test - void testDelete(@TempDir Path tempDir) { - String datasetPath = tempDir.resolve("testDelete").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - // Commit fragment - int rowCount = 20; - FragmentMetadata fragmentMeta0 = testDataset.createNewFragment(rowCount); - FragmentMetadata fragmentMeta1 = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Arrays.asList(fragmentMeta0, fragmentMeta1)).build()) - .build(); - try (Dataset dataset = transaction.commit()) { - assertEquals(2, dataset.version()); - assertEquals(2, dataset.latestVersion()); - } - - dataset = Dataset.open(datasetPath, allocator); - - List deletedFragmentIds = - dataset.getFragments().stream() - .map(t -> Long.valueOf(t.getId())) - .collect(Collectors.toList()); - - Transaction delete = - dataset - .newTransactionBuilder() - .operation( - Delete.builder().deletedFragmentIds(deletedFragmentIds).predicate("1=1").build()) - .build(); - try (Dataset dataset = delete.commit()) { - Transaction txn = dataset.readTransaction().get(); - Delete execDelete = (Delete) txn.operation(); - assertEquals(delete.operation(), execDelete); - assertEquals(0, dataset.countRows()); - } - } - } - - @Test - void testOverwrite(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testOverwrite").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // Commit fragment - int rowCount = 20; - FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Overwrite.builder() - .fragments(Collections.singletonList(fragmentMeta)) - .schema(testDataset.getSchema()) - .build()) - .build(); - try (Dataset dataset = transaction.commit()) { - assertEquals(2, dataset.version()); - assertEquals(2, dataset.latestVersion()); - assertEquals(rowCount, dataset.countRows()); - Fragment fragment = dataset.getFragments().get(0); - - try (LanceScanner scanner = fragment.newScan()) { - Schema schemaRes = scanner.schema(); - assertEquals(testDataset.getSchema(), schemaRes); - } - } - - // Commit fragment again - rowCount = 40; - fragmentMeta = testDataset.createNewFragment(rowCount); - transaction = - dataset - .newTransactionBuilder() - .operation( - Overwrite.builder() - .fragments(Collections.singletonList(fragmentMeta)) - .schema(testDataset.getSchema()) - .configUpsertValues(Collections.singletonMap("config_key", "config_value")) - .build()) - .transactionProperties(Collections.singletonMap("key", "value")) - .build(); - assertEquals("value", transaction.transactionProperties().get("key")); - try (Dataset dataset = transaction.commit()) { - assertEquals(3, dataset.version()); - assertEquals(3, dataset.latestVersion()); - assertEquals(rowCount, dataset.countRows()); - assertEquals("config_value", dataset.getConfig().get("config_key")); - Fragment fragment = dataset.getFragments().get(0); - - try (LanceScanner scanner = fragment.newScan()) { - Schema schemaRes = scanner.schema(); - assertEquals(testDataset.getSchema(), schemaRes); - } - assertEquals(transaction, dataset.readTransaction().orElse(null)); - } - } - } - - @Test - void testUpdate(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testUpdate").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // Commit fragment - int rowCount = 20; - FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) - .build(); - - try (Dataset dataset = transaction.commit()) { - assertEquals(2, dataset.version()); - assertEquals(2, dataset.latestVersion()); - assertEquals(rowCount, dataset.countRows()); - assertThrows( - IllegalArgumentException.class, - () -> - dataset - .newTransactionBuilder() - .operation(Append.builder().fragments(new ArrayList<>()).build()) - .build() - .commit() - .close()); - } - - dataset = Dataset.open(datasetPath, allocator); - // Update fragments - rowCount = 40; - FragmentMetadata newFragment = testDataset.createNewFragment(rowCount); - transaction = - dataset - .newTransactionBuilder() - .operation( - Update.builder() - .removedFragmentIds( - Collections.singletonList( - Long.valueOf(dataset.getFragments().get(0).getId()))) - .newFragments(Collections.singletonList(newFragment)) - .build()) - .build(); - - try (Dataset dataset = transaction.commit()) { - assertEquals(3, dataset.version()); - assertEquals(3, dataset.latestVersion()); - assertEquals(rowCount, dataset.countRows()); - - Transaction txn = dataset.readTransaction().orElse(null); - assertEquals(transaction, txn); - } - } - } - @Test - void testUpdateConfig(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testUpdateConfig").toString(); + public void testTransaction(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testTransaction").toString(); try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // Test 1: Update configuration values using upsertValues - Map upsertValues = new HashMap<>(); - upsertValues.put("key1", "value1"); - upsertValues.put("key2", "value2"); - - Transaction transaction = - dataset - .newTransactionBuilder() - .operation(UpdateConfig.builder().upsertValues(upsertValues).build()) - .build(); - try (Dataset updatedDataset = transaction.commit()) { - assertEquals(2, updatedDataset.version()); - assertEquals("value1", updatedDataset.getConfig().get("key1")); - assertEquals("value2", updatedDataset.getConfig().get("key2")); - - // Test 2: Delete configuration keys using deleteKeys - List deleteKeys = Collections.singletonList("key1"); - transaction = - updatedDataset - .newTransactionBuilder() - .operation(UpdateConfig.builder().deleteKeys(deleteKeys).build()) - .build(); - try (Dataset updatedDataset2 = transaction.commit()) { - assertEquals(3, updatedDataset2.version()); - assertNull(updatedDataset2.getConfig().get("key1")); - assertEquals("value2", updatedDataset2.getConfig().get("key2")); - - // Test 3: Update schema metadata using schemaMetadata - Map schemaMetadata = new HashMap<>(); - schemaMetadata.put("schema_key1", "schema_value1"); - schemaMetadata.put("schema_key2", "schema_value2"); - - transaction = - updatedDataset2 - .newTransactionBuilder() - .operation(UpdateConfig.builder().schemaMetadata(schemaMetadata).build()) - .build(); - try (Dataset updatedDataset3 = transaction.commit()) { - assertEquals(4, updatedDataset3.version()); - assertEquals( - "schema_value1", updatedDataset3.getLanceSchema().metadata().get("schema_key1")); - assertEquals( - "schema_value2", updatedDataset3.getLanceSchema().metadata().get("schema_key2")); - - // Test 4: Update field metadata using fieldMetadata - Map> fieldMetadata = new HashMap<>(); - Map field0Metadata = new HashMap<>(); - field0Metadata.put("field0_key1", "field0_value1"); - - Map field1Metadata = new HashMap<>(); - field1Metadata.put("field1_key1", "field1_value1"); - field1Metadata.put("field1_key2", "field1_value2"); - - fieldMetadata.put(0, field0Metadata); - fieldMetadata.put(1, field1Metadata); - - transaction = - updatedDataset3 - .newTransactionBuilder() - .operation(UpdateConfig.builder().fieldMetadata(fieldMetadata).build()) - .build(); - try (Dataset updatedDataset4 = transaction.commit()) { - assertEquals(5, updatedDataset4.version()); - - // Verify field metadata for field 0 - Map fieldMetadata0 = - updatedDataset4.getLanceSchema().fields().get(0).getMetadata(); - assertEquals("field0_value1", fieldMetadata0.get("field0_key1")); - - // Verify field metadata for field 1 - Map field1Result = - updatedDataset4.getLanceSchema().fields().get(1).getMetadata(); - assertEquals("field1_value1", field1Result.get("field1_key1")); - assertEquals("field1_value2", field1Result.get("field1_key2")); - } - } - } - } - } - } - - @Test - void testMerge(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testMerge").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - int rowCount = 20; - FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) - .build(); - try (Dataset dataset = transaction.commit()) { - assertEquals(2, dataset.version()); - assertEquals(rowCount, dataset.countRows()); - - Schema newSchema = testDataset.getSchema(); - FragmentMetadata newFragmentMeta = testDataset.createNewFragment(rowCount); - - Transaction mergeTransaction = - dataset - .newTransactionBuilder() - .operation( - Merge.builder() - .fragments(Collections.singletonList(newFragmentMeta)) - .schema(newSchema) - .build()) - .transactionProperties(Collections.singletonMap("key", "value")) - .build(); - assertEquals("value", mergeTransaction.transactionProperties().get("key")); - try (Dataset mergedDataset = mergeTransaction.commit()) { - assertEquals(3, mergedDataset.version()); - assertEquals(3, mergedDataset.latestVersion()); - assertEquals(rowCount, mergedDataset.countRows()); - - assertEquals(newSchema, mergedDataset.getSchema()); - - Fragment fragment = mergedDataset.getFragments().get(0); - try (LanceScanner scanner = fragment.newScan()) { - Schema schemaRes = scanner.schema(); - assertEquals(newSchema, schemaRes); - } - - assertEquals(mergeTransaction, mergedDataset.readTransaction().orElse(null)); - } - } - } - } - - @Test - void testRewrite(@TempDir Path tempDir) { - String datasetPath = tempDir.resolve("testRewrite").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // First, append some data - int rowCount = 20; - FragmentMetadata fragmentMeta1 = testDataset.createNewFragment(rowCount); - FragmentMetadata fragmentMeta2 = testDataset.createNewFragment(rowCount); - - Transaction appendTx = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Arrays.asList(fragmentMeta1, fragmentMeta2)).build()) - .build(); - - try (Dataset datasetWithData = appendTx.commit()) { - assertEquals(2, datasetWithData.version()); - assertEquals(rowCount * 2, datasetWithData.countRows()); - - // Now create a rewrite operation - List groups = new ArrayList<>(); - - // Create a rewrite group with old fragments and new fragments - List oldFragments = new ArrayList<>(); - oldFragments.add(fragmentMeta1); - - List newFragments = new ArrayList<>(); - FragmentMetadata newFragmentMeta = testDataset.createNewFragment(rowCount); - newFragments.add(newFragmentMeta); - - RewriteGroup group = - RewriteGroup.builder().oldFragments(oldFragments).newFragments(newFragments).build(); - - groups.add(group); - - // Create and commit the rewrite transaction - Transaction rewriteTx = - datasetWithData - .newTransactionBuilder() - .operation(Rewrite.builder().groups(groups).build()) - .build(); - - try (Dataset rewrittenDataset = rewriteTx.commit()) { - assertEquals(3, rewrittenDataset.version()); - // The row count should remain the same since we're just rewriting - assertEquals(rowCount * 2, rewrittenDataset.countRows()); - - // Verify that the transaction was recorded - assertEquals(rewriteTx, rewrittenDataset.readTransaction().orElse(null)); - } - } - } - } - - @Test - void testRestore(@TempDir Path tempDir) { - String datasetPath = tempDir.resolve("testRestore").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // Record the initial version - long initialVersion = dataset.version(); - - // Append data to create a new version - int rowCount = 20; - FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); - Transaction transaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) - .build(); - try (Dataset modifiedDataset = transaction.commit()) { - // Verify the dataset was modified - long newVersion = modifiedDataset.version(); - assertEquals(initialVersion + 1, newVersion); - assertEquals(rowCount, modifiedDataset.countRows()); - - // Restore to the initial version - Transaction restoreTransaction = - modifiedDataset - .newTransactionBuilder() - .operation(new Restore.Builder().version(initialVersion).build()) - .build(); - try (Dataset restoredDataset = restoreTransaction.commit()) { - // Verify the dataset was restored to the initial version, but the version increases - assertEquals(initialVersion + 2, restoredDataset.version()); - // Initial dataset had 0 rows - assertEquals(0, restoredDataset.countRows()); - assertEquals(restoreTransaction, restoredDataset.readTransaction().orElse(null)); - } - } - } - } - - @Test - void testReserveFragments(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testReserveFragments").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // Create an initial fragment to establish a baseline fragment ID - FragmentMetadata initialFragmentMeta = testDataset.createNewFragment(10); - Transaction appendTransaction = - dataset - .newTransactionBuilder() - .operation( - Append.builder() - .fragments(Collections.singletonList(initialFragmentMeta)) - .build()) - .build(); - try (Dataset datasetWithFragment = appendTransaction.commit()) { - // Reserve fragment IDs - int numFragmentsToReserve = 5; - Transaction reserveTransaction = - datasetWithFragment - .newTransactionBuilder() - .operation( - new ReserveFragments.Builder().numFragments(numFragmentsToReserve).build()) - .build(); - try (Dataset datasetWithReservedFragments = reserveTransaction.commit()) { - // Create a new fragment and verify its ID reflects the reservation - FragmentMetadata newFragmentMeta = testDataset.createNewFragment(10); - Transaction appendTransaction2 = - datasetWithReservedFragments - .newTransactionBuilder() - .operation( - Append.builder() - .fragments(Collections.singletonList(newFragmentMeta)) - .build()) - .build(); - try (Dataset finalDataset = appendTransaction2.commit()) { - // Verify the fragment IDs were properly reserved - // The new fragment should have an ID that's at least numFragmentsToReserve higher - // than it would have been without the reservation - List fragments = finalDataset.getFragments(); - assertEquals(2, fragments.size()); - - // The first fragment ID is typically 0, and the second would normally be 1 - // But after reserving 5 fragments, the second fragment ID should be at least 6 - Fragment firstFragment = fragments.get(0); - Fragment secondFragment = fragments.get(1); - - // Check that the second fragment has a significantly higher ID than the first - // This is an indirect way to verify that fragment IDs were reserved - Assertions.assertNotEquals( - firstFragment.metadata().getId() + 1, secondFragment.getId()); - - // Verify the transaction is recorded - assertEquals( - reserveTransaction, datasetWithReservedFragments.readTransaction().orElse(null)); - } - } - } - } - } - - @Test - void testDataReplacement(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testDataReplacement").toString(); - try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - - // step 1. create a dataset with schema: id: int, name: varchar - TestUtils.SimpleTestDataset testDataset = - new TestUtils.SimpleTestDataset(allocator, datasetPath); - dataset = testDataset.createEmptyDataset(); - - // step 2. create a new VectorSchemaRoot with only id values and append it to the dataset - int rowCount = 20; - Schema idOnlySchema = - new Schema( - Collections.singletonList(Field.nullable("id", new ArrowType.Int(32, true))), null); - - try (VectorSchemaRoot idRoot = VectorSchemaRoot.create(idOnlySchema, allocator)) { - idRoot.allocateNew(); - IntVector idVector = (IntVector) idRoot.getVector("id"); - for (int i = 0; i < rowCount; i++) { - idVector.setSafe(i, i); - } - idRoot.setRowCount(rowCount); - - List fragmentMetas = - Fragment.create(datasetPath, allocator, idRoot, new WriteParams.Builder().build()); + TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath); + try (Dataset dataset = testDataset.createEmptyDataset()) { + FragmentMetadata fragmentMeta = testDataset.createNewFragment(20); + Map properties = new HashMap<>(); + properties.put("transactionType", "APPEND"); + properties.put("createdBy", "testUser"); Transaction appendTxn = dataset .newTransactionBuilder() - .operation(Append.builder().fragments(fragmentMetas).build()) + .operation(Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .transactionProperties(properties) .build(); + try (Dataset committedDataset = appendTxn.commit()) { + assertEquals(2, committedDataset.version()); + assertEquals(2, committedDataset.latestVersion()); + assertEquals(20, committedDataset.countRows()); + assertEquals(dataset.version(), appendTxn.readVersion()); + assertNotNull(appendTxn.uuid()); - try (Dataset initDataset = appendTxn.commit()) { - assertEquals(2, initDataset.version()); - assertEquals(rowCount, initDataset.countRows()); - - // step 3. use dataset.addColumn to add a new column named as address with all null values - Field addressField = Field.nullable("address", new ArrowType.Utf8()); - Schema addressSchema = new Schema(Collections.singletonList(addressField), null); - initDataset.addColumns(addressSchema); - - try (LanceScanner scanner = initDataset.newScan()) { - try (ArrowReader resultReader = scanner.scanBatches()) { - assertTrue(resultReader.loadNextBatch()); - VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); - assertEquals(rowCount, initDataset.countRows()); - assertEquals(rowCount, batch.getRowCount()); - - // verify all null values - VarCharVector resultNameVector = (VarCharVector) batch.getVector("address"); - for (int i = 0; i < rowCount; i++) { - Assertions.assertTrue(resultNameVector.isNull(i)); - } - } - } - - // step 4. use DataReplacement transaction to replace null values - try (VectorSchemaRoot replaceVectorRoot = - VectorSchemaRoot.create(addressSchema, allocator)) { - replaceVectorRoot.allocateNew(); - VarCharVector addressVector = (VarCharVector) replaceVectorRoot.getVector("address"); - - for (int i = 0; i < rowCount; i++) { - String name = "District " + i; - addressVector.setSafe(i, name.getBytes(StandardCharsets.UTF_8)); - } - replaceVectorRoot.setRowCount(rowCount); - - DataFile datafile = - createDataFile(dataset.allocator(), datasetPath, replaceVectorRoot, 2); - List replacementGroups = - Collections.singletonList( - new DataReplacement.DataReplacementGroup( - fragmentMetas.get(0).getId(), datafile)); - Transaction replaceTxn = - initDataset - .newTransactionBuilder() - .operation(DataReplacement.builder().replacements(replacementGroups).build()) - .build(); - - try (Dataset datasetWithAddress = replaceTxn.commit()) { - assertEquals(4, datasetWithAddress.version()); - assertEquals(rowCount, datasetWithAddress.countRows()); - - try (LanceScanner scanner = datasetWithAddress.newScan()) { - try (ArrowReader resultReader = scanner.scanBatches()) { - assertTrue(resultReader.loadNextBatch()); - VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); - assertEquals(rowCount, datasetWithAddress.countRows()); - assertEquals(rowCount, batch.getRowCount()); - - // verify all address values not null - VarCharVector resultNameVector = (VarCharVector) batch.getVector("address"); - for (int i = 0; i < rowCount; i++) { - Assertions.assertFalse(resultNameVector.isNull(i)); - String expectedName = "District " + i; - String actualName = new String(resultNameVector.get(i), StandardCharsets.UTF_8); - assertEquals(expectedName, actualName); - } - } - } - } - } + // Verify transaction properties + Map txnProps = appendTxn.transactionProperties(); + assertEquals("APPEND", txnProps.get("transactionType")); + assertEquals("testUser", txnProps.get("createdBy")); } } } } - - /** - * Helper method to create a DataFile from a VectorSchemaRoot. This implementation uses - * LanceFileWriter to ensure compatibility with Lance format. - */ - private DataFile createDataFile( - BufferAllocator allocator, String basePath, VectorSchemaRoot root, int fieldIndex) { - // Create a unique file path for the data file - String fileName = UUID.randomUUID() + ".lance"; - String filePath = basePath + "/data/" + fileName; - - // Create parent directories if they don't exist - File file = new File(filePath); - - // Use LanceFileWriter to write the data - try (LanceFileWriter writer = LanceFileWriter.open(filePath, allocator, null)) { - writer.write(root); - } catch (Exception e) { - throw new RuntimeException(e); - } - - // Create a DataFile object with the field index - // The fields array contains the indices of the fields in the schema - // The columnIndices array contains the indices of the columns in the file - // Use a stable file format version - return new DataFile( - fileName, - new int[] {fieldIndex}, // Field index in the schema - new int[] {0}, // Column index in the file (always 0 for single column) - TEST_FILE_FORMAT_MAJOR_VERSION, // File major version - TEST_FILE_FORMAT_MINOR_VERSION, // File minor version - file.length() // File size in bytes (now contains actual data) - ); - } } diff --git a/java/core/src/test/java/com/lancedb/lance/operation/AppendTest.java b/java/core/src/test/java/com/lancedb/lance/operation/AppendTest.java new file mode 100644 index 00000000000..6d7cf0b59a8 --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/AppendTest.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class AppendTest extends OperationTestBase { + + @Test + void testAppendSingleFragment(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testAppendSingleFragment").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + int rowCount = 10; + try (Dataset result = createAndAppendRows(testDataset, rowCount)) { + assertEquals(2, result.version()); + assertEquals(rowCount, result.countRows()); + } + } + } + + @Test + void testAppendMultipleFragments(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testAppendMultipleFragments").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + int rowCount = 10; + List fragments = + Arrays.asList( + testDataset.createNewFragment(rowCount), + testDataset.createNewFragment(rowCount), + testDataset.createNewFragment(rowCount)); + + Transaction transaction = + dataset + .newTransactionBuilder() + .operation(Append.builder().fragments(fragments).build()) + .build(); + + try (Dataset dataset = transaction.commit()) { + assertEquals(2, dataset.version()); + assertEquals(rowCount * 3, dataset.countRows()); + assertEquals(3, dataset.getFragments().size()); + assertEquals(transaction, dataset.readTransaction().orElse(null)); + } + } + } + + @Test + void testAppendEmptyFragmentList(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testAppendEmptyFragmentList").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + try (Dataset dataset = testDataset.createEmptyDataset()) { + assertThrows( + IllegalArgumentException.class, + () -> { + Transaction transaction = + dataset + .newTransactionBuilder() + .operation(Append.builder().fragments(new ArrayList<>()).build()) + .build(); + transaction.commit().close(); + }); + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java new file mode 100644 index 00000000000..ce7322b8d3a --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.Fragment; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; +import com.lancedb.lance.WriteParams; +import com.lancedb.lance.fragment.DataFile; +import com.lancedb.lance.ipc.LanceScanner; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DataReplacementTest extends OperationTestBase { + + @Test + void testDataReplacement(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testDataReplacement").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + + // step 1. create a dataset with schema: id: int, name: varchar + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // step 2. create a new VectorSchemaRoot with only id values and append it to the dataset + int rowCount = 20; + Schema idOnlySchema = + new Schema( + Collections.singletonList(Field.nullable("id", new ArrowType.Int(32, true))), null); + + try (VectorSchemaRoot idRoot = VectorSchemaRoot.create(idOnlySchema, allocator)) { + idRoot.allocateNew(); + IntVector idVector = (IntVector) idRoot.getVector("id"); + for (int i = 0; i < rowCount; i++) { + idVector.setSafe(i, i); + } + idRoot.setRowCount(rowCount); + + List fragmentMetas = + Fragment.create(datasetPath, allocator, idRoot, new WriteParams.Builder().build()); + + Transaction appendTxn = + dataset + .newTransactionBuilder() + .operation(Append.builder().fragments(fragmentMetas).build()) + .build(); + + try (Dataset initDataset = appendTxn.commit()) { + assertEquals(2, initDataset.version()); + assertEquals(rowCount, initDataset.countRows()); + + // step 3. use dataset.addColumn to add a new column named as address with all null values + Field addressField = Field.nullable("address", new ArrowType.Utf8()); + Schema addressSchema = new Schema(Collections.singletonList(addressField), null); + initDataset.addColumns(addressSchema); + + try (LanceScanner scanner = initDataset.newScan()) { + try (ArrowReader resultReader = scanner.scanBatches()) { + assertTrue(resultReader.loadNextBatch()); + VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); + assertEquals(rowCount, initDataset.countRows()); + assertEquals(rowCount, batch.getRowCount()); + + // verify all null values + VarCharVector resultNameVector = (VarCharVector) batch.getVector("address"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertTrue(resultNameVector.isNull(i)); + } + } + } + + // step 4. use DataReplacement transaction to replace null values + try (VectorSchemaRoot replaceVectorRoot = + VectorSchemaRoot.create(addressSchema, allocator)) { + replaceVectorRoot.allocateNew(); + VarCharVector addressVector = (VarCharVector) replaceVectorRoot.getVector("address"); + + for (int i = 0; i < rowCount; i++) { + String name = "District " + i; + addressVector.setSafe(i, name.getBytes(StandardCharsets.UTF_8)); + } + replaceVectorRoot.setRowCount(rowCount); + + DataFile datafile = + writeLanceDataFile(dataset.allocator(), datasetPath, replaceVectorRoot, 2); + List replacementGroups = + Collections.singletonList( + new DataReplacement.DataReplacementGroup( + fragmentMetas.get(0).getId(), datafile)); + Transaction replaceTxn = + initDataset + .newTransactionBuilder() + .operation(DataReplacement.builder().replacements(replacementGroups).build()) + .build(); + + try (Dataset datasetWithAddress = replaceTxn.commit()) { + assertEquals(4, datasetWithAddress.version()); + assertEquals(rowCount, datasetWithAddress.countRows()); + + try (LanceScanner scanner = datasetWithAddress.newScan()) { + try (ArrowReader resultReader = scanner.scanBatches()) { + assertTrue(resultReader.loadNextBatch()); + VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); + assertEquals(rowCount, datasetWithAddress.countRows()); + assertEquals(rowCount, batch.getRowCount()); + + // verify all address values not null + VarCharVector resultNameVector = (VarCharVector) batch.getVector("address"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertFalse(resultNameVector.isNull(i)); + String expectedName = "District " + i; + String actualName = new String(resultNameVector.get(i), StandardCharsets.UTF_8); + assertEquals(expectedName, actualName); + } + } + } + } + } + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/DeleteTest.java b/java/core/src/test/java/com/lancedb/lance/operation/DeleteTest.java new file mode 100644 index 00000000000..3906224dd2c --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/DeleteTest.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DeleteTest extends OperationTestBase { + + @Test + void testDelete(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testDelete").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + // Commit fragment + int rowCount = 20; + FragmentMetadata fragmentMeta0 = testDataset.createNewFragment(rowCount); + FragmentMetadata fragmentMeta1 = testDataset.createNewFragment(rowCount); + Transaction transaction = + dataset + .newTransactionBuilder() + .operation( + Append.builder().fragments(Arrays.asList(fragmentMeta0, fragmentMeta1)).build()) + .build(); + try (Dataset dataset = transaction.commit()) { + assertEquals(2, dataset.version()); + assertEquals(2, dataset.latestVersion()); + } + + dataset = Dataset.open(datasetPath, allocator); + + List deletedFragmentIds = + dataset.getFragments().stream() + .map(t -> Long.valueOf(t.getId())) + .collect(Collectors.toList()); + + Transaction delete = + dataset + .newTransactionBuilder() + .operation( + Delete.builder().deletedFragmentIds(deletedFragmentIds).predicate("1=1").build()) + .build(); + try (Dataset dataset = delete.commit()) { + Transaction txn = dataset.readTransaction().get(); + Delete execDelete = (Delete) txn.operation(); + assertEquals(delete.operation(), execDelete); + assertEquals(0, dataset.countRows()); + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java new file mode 100644 index 00000000000..7d17d906629 --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java @@ -0,0 +1,197 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; +import com.lancedb.lance.fragment.DataFile; +import com.lancedb.lance.ipc.LanceScanner; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; + +public class MergeTest extends OperationTestBase { + + @Test + void testMergeNewColumn(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testMergeNewColumn").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + int rowCount = 15; + try (Dataset initialDataset = createAndAppendRows(testDataset, 15)) { + // Add a new column with different data type + Field ageField = Field.nullable("age", new ArrowType.Int(32, true)); + Schema evolvedSchema = + new Schema( + Arrays.asList( + Field.nullable("id", new ArrowType.Int(32, true)), + Field.nullable("name", new ArrowType.Utf8()), + Field.nullable("age", new ArrowType.Int(32, true))), + null); + + try (VectorSchemaRoot ageRoot = + VectorSchemaRoot.create( + new Schema(Collections.singletonList(ageField), null), allocator)) { + ageRoot.allocateNew(); + IntVector ageVector = (IntVector) ageRoot.getVector("age"); + + for (int i = 0; i < rowCount; i++) { + ageVector.setSafe(i, 20 + i); + } + ageRoot.setRowCount(rowCount); + + DataFile ageDataFile = + writeLanceDataFile( + dataset.allocator(), datasetPath, ageRoot, 2 // field index for age column + ); + + FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata(); + FragmentMetadata evolvedFragment = + new FragmentMetadata( + fragmentMeta.getId(), + Collections.singletonList(ageDataFile), + fragmentMeta.getPhysicalRows(), + fragmentMeta.getDeletionFile(), + fragmentMeta.getRowIdMeta()); + + Transaction mergeTransaction = + initialDataset + .newTransactionBuilder() + .operation( + Merge.builder() + .fragments(Collections.singletonList(evolvedFragment)) + .schema(evolvedSchema) + .build()) + .build(); + + try (Dataset evolvedDataset = mergeTransaction.commit()) { + Assertions.assertEquals(3, evolvedDataset.version()); + Assertions.assertEquals(rowCount, evolvedDataset.countRows()); + Assertions.assertEquals(evolvedSchema, evolvedDataset.getSchema()); + Assertions.assertEquals(3, evolvedDataset.getSchema().getFields().size()); + // Verify merged data + try (LanceScanner scanner = evolvedDataset.newScan()) { + try (ArrowReader resultReader = scanner.scanBatches()) { + Assertions.assertTrue(resultReader.loadNextBatch()); + VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); + Assertions.assertEquals(rowCount, batch.getRowCount()); + Assertions.assertEquals(3, batch.getSchema().getFields().size()); + // Verify age column + IntVector ageResultVector = (IntVector) batch.getVector("age"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertEquals(20 + i, ageResultVector.get(i)); + } + } + } + } + } + } + } + } + + @Test + void testMergeExistingColumn(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testMergeExistingColumn").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + // Test merging with existing column updates + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + int rowCount = 10; + try (Dataset initialDataset = createAndAppendRows(testDataset, rowCount)) { + // Create updated name column data + Field nameField = Field.nullable("name", new ArrowType.Utf8()); + Schema nameSchema = new Schema(Collections.singletonList(nameField), null); + + try (VectorSchemaRoot updatedNameRoot = VectorSchemaRoot.create(nameSchema, allocator)) { + updatedNameRoot.allocateNew(); + VarCharVector nameVector = (VarCharVector) updatedNameRoot.getVector("name"); + + for (int i = 0; i < rowCount; i++) { + String updatedName = "UpdatedName_" + i; + nameVector.setSafe(i, updatedName.getBytes(StandardCharsets.UTF_8)); + } + updatedNameRoot.setRowCount(rowCount); + + // Create DataFile for updated column + DataFile updatedNameDataFile = + writeLanceDataFile( + dataset.allocator(), + datasetPath, + updatedNameRoot, + 1 // field index for name column + ); + + // Perform merge with updated column + FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata(); + FragmentMetadata updatedFragment = + new FragmentMetadata( + fragmentMeta.getId(), + Collections.singletonList(updatedNameDataFile), + fragmentMeta.getPhysicalRows(), + fragmentMeta.getDeletionFile(), + fragmentMeta.getRowIdMeta()); + + Transaction mergeTransaction = + initialDataset + .newTransactionBuilder() + .operation( + Merge.builder() + .fragments(Collections.singletonList(updatedFragment)) + .schema(testDataset.getSchema()) + .build()) + .build(); + + try (Dataset mergedDataset = mergeTransaction.commit()) { + Assertions.assertEquals(3, mergedDataset.version()); + Assertions.assertEquals(rowCount, mergedDataset.countRows()); + + // Verify updated data + try (LanceScanner scanner = mergedDataset.newScan()) { + try (ArrowReader resultReader = scanner.scanBatches()) { + Assertions.assertTrue(resultReader.loadNextBatch()); + VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); + + VarCharVector nameResultVector = (VarCharVector) batch.getVector("name"); + for (int i = 0; i < rowCount; i++) { + String expectedName = "UpdatedName_" + i; + String actualName = new String(nameResultVector.get(i), StandardCharsets.UTF_8); + Assertions.assertEquals(expectedName, actualName); + } + } + } + } + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java new file mode 100644 index 00000000000..a974ab723f0 --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; +import com.lancedb.lance.file.LanceFileWriter; +import com.lancedb.lance.fragment.DataFile; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +import java.io.File; +import java.util.Collections; +import java.util.UUID; + +public class OperationTestBase { + + public static final int TEST_FILE_FORMAT_MAJOR_VERSION = 2; + public static final int TEST_FILE_FORMAT_MINOR_VERSION = 0; + protected static Dataset dataset; + + @BeforeAll + static void setup() {} + + @AfterAll + static void tearDown() { + // Cleanup resources used by the tests + if (dataset != null) { + dataset.close(); + } + } + + /** Helper method to append simple data to a dataset. */ + protected Dataset createAndAppendRows(TestUtils.SimpleTestDataset suite, int rowCount) { + dataset = suite.createEmptyDataset(); + FragmentMetadata fragmentMeta = suite.createNewFragment(rowCount); + + Transaction appendTxn = + dataset + .newTransactionBuilder() + .operation(Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build(); + return appendTxn.commit(); + } + + /** + * Helper method to create a DataFile from a VectorSchemaRoot. This implementation uses + * LanceFileWriter to ensure compatibility with Lance format. + */ + protected DataFile writeLanceDataFile( + BufferAllocator allocator, String basePath, VectorSchemaRoot root, int fieldIndex) { + // Create a unique file path for the data file + String fileName = UUID.randomUUID() + ".lance"; + String filePath = basePath + "/data/" + fileName; + + // Create parent directories if they don't exist + File file = new File(filePath); + + // Use LanceFileWriter to write the data + try (LanceFileWriter writer = LanceFileWriter.open(filePath, allocator, null)) { + writer.write(root); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // Create a DataFile object with the field index + // The fields array contains the indices of the fields in the schema + // The columnIndices array contains the indices of the columns in the file + // Use a stable file format version + return new DataFile( + fileName, + new int[] {fieldIndex}, // Field index in the schema + new int[] {0}, // Column index in the file (always 0 for single column) + TEST_FILE_FORMAT_MAJOR_VERSION, // File major version + TEST_FILE_FORMAT_MINOR_VERSION, // File minor version + file.length() // File size in bytes (now contains actual data) + ); + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OverwriteTest.java b/java/core/src/test/java/com/lancedb/lance/operation/OverwriteTest.java new file mode 100644 index 00000000000..145b81539ff --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/OverwriteTest.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.Fragment; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; +import com.lancedb.lance.ipc.LanceScanner; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class OverwriteTest extends OperationTestBase { + + @Test + void testOverwrite(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testOverwrite").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Commit fragment + int rowCount = 20; + FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); + Transaction transaction = + dataset + .newTransactionBuilder() + .operation( + Overwrite.builder() + .fragments(Collections.singletonList(fragmentMeta)) + .schema(testDataset.getSchema()) + .build()) + .build(); + try (Dataset dataset = transaction.commit()) { + assertEquals(2, dataset.version()); + assertEquals(2, dataset.latestVersion()); + assertEquals(rowCount, dataset.countRows()); + Fragment fragment = dataset.getFragments().get(0); + + try (LanceScanner scanner = fragment.newScan()) { + Schema schemaRes = scanner.schema(); + assertEquals(testDataset.getSchema(), schemaRes); + } + } + + // Commit fragment again + rowCount = 40; + fragmentMeta = testDataset.createNewFragment(rowCount); + transaction = + dataset + .newTransactionBuilder() + .operation( + Overwrite.builder() + .fragments(Collections.singletonList(fragmentMeta)) + .schema(testDataset.getSchema()) + .configUpsertValues(Collections.singletonMap("config_key", "config_value")) + .build()) + .transactionProperties(Collections.singletonMap("key", "value")) + .build(); + assertEquals("value", transaction.transactionProperties().get("key")); + try (Dataset dataset = transaction.commit()) { + assertEquals(3, dataset.version()); + assertEquals(3, dataset.latestVersion()); + assertEquals(rowCount, dataset.countRows()); + assertEquals("config_value", dataset.getConfig().get("config_key")); + Fragment fragment = dataset.getFragments().get(0); + + try (LanceScanner scanner = fragment.newScan()) { + Schema schemaRes = scanner.schema(); + assertEquals(testDataset.getSchema(), schemaRes); + } + assertEquals(transaction, dataset.readTransaction().orElse(null)); + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/ProjectTest.java b/java/core/src/test/java/com/lancedb/lance/operation/ProjectTest.java new file mode 100644 index 00000000000..3a564040bdf --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/ProjectTest.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ProjectTest extends OperationTestBase { + + @Test + void testProjection(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testProjection").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + assertEquals(testDataset.getSchema(), dataset.getSchema()); + List fieldList = new ArrayList<>(testDataset.getSchema().getFields()); + Collections.reverse(fieldList); + Transaction txn1 = + dataset + .newTransactionBuilder() + .operation(Project.builder().schema(new Schema(fieldList)).build()) + .build(); + try (Dataset committedDataset = txn1.commit()) { + assertEquals(1, txn1.readVersion()); + assertEquals(1, dataset.version()); + assertEquals(2, committedDataset.version()); + assertEquals(new Schema(fieldList), committedDataset.getSchema()); + fieldList.remove(1); + Transaction txn2 = + committedDataset + .newTransactionBuilder() + .operation(Project.builder().schema(new Schema(fieldList)).build()) + .build(); + try (Dataset committedDataset2 = txn2.commit()) { + assertEquals(2, txn2.readVersion()); + assertEquals(2, committedDataset.version()); + assertEquals(3, committedDataset2.version()); + assertEquals(new Schema(fieldList), committedDataset2.getSchema()); + assertEquals(txn1, committedDataset.readTransaction().orElse(null)); + assertEquals(txn2, committedDataset2.readTransaction().orElse(null)); + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/ReserveFragmentsTest.java b/java/core/src/test/java/com/lancedb/lance/operation/ReserveFragmentsTest.java new file mode 100644 index 00000000000..0ce908a55b6 --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/ReserveFragmentsTest.java @@ -0,0 +1,98 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.Fragment; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ReserveFragmentsTest extends OperationTestBase { + + @Test + void testReserveFragments(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testReserveFragments").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Create an initial fragment to establish a baseline fragment ID + FragmentMetadata initialFragmentMeta = testDataset.createNewFragment(10); + Transaction appendTransaction = + dataset + .newTransactionBuilder() + .operation( + Append.builder() + .fragments(Collections.singletonList(initialFragmentMeta)) + .build()) + .build(); + try (Dataset datasetWithFragment = appendTransaction.commit()) { + // Reserve fragment IDs + int numFragmentsToReserve = 5; + Transaction reserveTransaction = + datasetWithFragment + .newTransactionBuilder() + .operation( + new ReserveFragments.Builder().numFragments(numFragmentsToReserve).build()) + .build(); + try (Dataset datasetWithReservedFragments = reserveTransaction.commit()) { + // Create a new fragment and verify its ID reflects the reservation + FragmentMetadata newFragmentMeta = testDataset.createNewFragment(10); + Transaction appendTransaction2 = + datasetWithReservedFragments + .newTransactionBuilder() + .operation( + Append.builder() + .fragments(Collections.singletonList(newFragmentMeta)) + .build()) + .build(); + try (Dataset finalDataset = appendTransaction2.commit()) { + // Verify the fragment IDs were properly reserved + // The new fragment should have an ID that's at least numFragmentsToReserve higher + // than it would have been without the reservation + List fragments = finalDataset.getFragments(); + assertEquals(2, fragments.size()); + + // The first fragment ID is typically 0, and the second would normally be 1 + // But after reserving 5 fragments, the second fragment ID should be at least 6 + Fragment firstFragment = fragments.get(0); + Fragment secondFragment = fragments.get(1); + + // Check that the second fragment has a significantly higher ID than the first + // This is an indirect way to verify that fragment IDs were reserved + Assertions.assertNotEquals( + firstFragment.metadata().getId() + 1, secondFragment.getId()); + + // Verify the transaction is recorded + assertEquals( + reserveTransaction, datasetWithReservedFragments.readTransaction().orElse(null)); + } + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/RestoreTest.java b/java/core/src/test/java/com/lancedb/lance/operation/RestoreTest.java new file mode 100644 index 00000000000..40814cf51dd --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/RestoreTest.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RestoreTest extends OperationTestBase { + + @Test + void testRestore(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testRestore").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Record the initial version + long initialVersion = dataset.version(); + + // Append data to create a new version + int rowCount = 20; + FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); + Transaction transaction = + dataset + .newTransactionBuilder() + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build(); + try (Dataset modifiedDataset = transaction.commit()) { + // Verify the dataset was modified + long newVersion = modifiedDataset.version(); + assertEquals(initialVersion + 1, newVersion); + assertEquals(rowCount, modifiedDataset.countRows()); + + // Restore to the initial version + Transaction restoreTransaction = + modifiedDataset + .newTransactionBuilder() + .operation(new Restore.Builder().version(initialVersion).build()) + .build(); + try (Dataset restoredDataset = restoreTransaction.commit()) { + // Verify the dataset was restored to the initial version, but the version increases + assertEquals(initialVersion + 2, restoredDataset.version()); + // Initial dataset had 0 rows + assertEquals(0, restoredDataset.countRows()); + assertEquals(restoreTransaction, restoredDataset.readTransaction().orElse(null)); + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/RewriteTest.java b/java/core/src/test/java/com/lancedb/lance/operation/RewriteTest.java new file mode 100644 index 00000000000..09dbead2cf2 --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/RewriteTest.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RewriteTest extends OperationTestBase { + + @Test + void testRewrite(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testRewrite").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // First, append some data + int rowCount = 20; + FragmentMetadata fragmentMeta1 = testDataset.createNewFragment(rowCount); + FragmentMetadata fragmentMeta2 = testDataset.createNewFragment(rowCount); + + Transaction appendTx = + dataset + .newTransactionBuilder() + .operation( + Append.builder().fragments(Arrays.asList(fragmentMeta1, fragmentMeta2)).build()) + .build(); + + try (Dataset datasetWithData = appendTx.commit()) { + assertEquals(2, datasetWithData.version()); + assertEquals(rowCount * 2, datasetWithData.countRows()); + + // Now create a rewrite operation + List groups = new ArrayList<>(); + + // Create a rewrite group with old fragments and new fragments + List oldFragments = new ArrayList<>(); + oldFragments.add(fragmentMeta1); + + List newFragments = new ArrayList<>(); + FragmentMetadata newFragmentMeta = testDataset.createNewFragment(rowCount); + newFragments.add(newFragmentMeta); + + RewriteGroup group = + RewriteGroup.builder().oldFragments(oldFragments).newFragments(newFragments).build(); + + groups.add(group); + + // Create and commit the rewrite transaction + Transaction rewriteTx = + datasetWithData + .newTransactionBuilder() + .operation(Rewrite.builder().groups(groups).build()) + .build(); + + try (Dataset rewrittenDataset = rewriteTx.commit()) { + assertEquals(3, rewrittenDataset.version()); + // The row count should remain the same since we're just rewriting + assertEquals(rowCount * 2, rewrittenDataset.countRows()); + + // Verify that the transaction was recorded + assertEquals(rewriteTx, rewrittenDataset.readTransaction().orElse(null)); + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/UpdateConfigTest.java b/java/core/src/test/java/com/lancedb/lance/operation/UpdateConfigTest.java new file mode 100644 index 00000000000..56073b25f8c --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/UpdateConfigTest.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class UpdateConfigTest extends OperationTestBase { + + @Test + void testUpdateConfig(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testUpdateConfig").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Test 1: Update configuration values using upsertValues + Map upsertValues = new HashMap<>(); + upsertValues.put("key1", "value1"); + upsertValues.put("key2", "value2"); + + Transaction transaction = + dataset + .newTransactionBuilder() + .operation(UpdateConfig.builder().upsertValues(upsertValues).build()) + .build(); + try (Dataset updatedDataset = transaction.commit()) { + assertEquals(2, updatedDataset.version()); + assertEquals("value1", updatedDataset.getConfig().get("key1")); + assertEquals("value2", updatedDataset.getConfig().get("key2")); + + // Test 2: Delete configuration keys using deleteKeys + List deleteKeys = Collections.singletonList("key1"); + transaction = + updatedDataset + .newTransactionBuilder() + .operation(UpdateConfig.builder().deleteKeys(deleteKeys).build()) + .build(); + try (Dataset updatedDataset2 = transaction.commit()) { + assertEquals(3, updatedDataset2.version()); + assertNull(updatedDataset2.getConfig().get("key1")); + assertEquals("value2", updatedDataset2.getConfig().get("key2")); + + // Test 3: Update schema metadata using schemaMetadata + Map schemaMetadata = new HashMap<>(); + schemaMetadata.put("schema_key1", "schema_value1"); + schemaMetadata.put("schema_key2", "schema_value2"); + + transaction = + updatedDataset2 + .newTransactionBuilder() + .operation(UpdateConfig.builder().schemaMetadata(schemaMetadata).build()) + .build(); + try (Dataset updatedDataset3 = transaction.commit()) { + assertEquals(4, updatedDataset3.version()); + assertEquals( + "schema_value1", updatedDataset3.getLanceSchema().metadata().get("schema_key1")); + assertEquals( + "schema_value2", updatedDataset3.getLanceSchema().metadata().get("schema_key2")); + + // Test 4: Update field metadata using fieldMetadata + Map> fieldMetadata = new HashMap<>(); + Map field0Metadata = new HashMap<>(); + field0Metadata.put("field0_key1", "field0_value1"); + + Map field1Metadata = new HashMap<>(); + field1Metadata.put("field1_key1", "field1_value1"); + field1Metadata.put("field1_key2", "field1_value2"); + + fieldMetadata.put(0, field0Metadata); + fieldMetadata.put(1, field1Metadata); + + transaction = + updatedDataset3 + .newTransactionBuilder() + .operation(UpdateConfig.builder().fieldMetadata(fieldMetadata).build()) + .build(); + try (Dataset updatedDataset4 = transaction.commit()) { + assertEquals(5, updatedDataset4.version()); + + // Verify field metadata for field 0 + Map fieldMetadata0 = + updatedDataset4.getLanceSchema().fields().get(0).getMetadata(); + assertEquals("field0_value1", fieldMetadata0.get("field0_key1")); + + // Verify field metadata for field 1 + Map field1Result = + updatedDataset4.getLanceSchema().fields().get(1).getMetadata(); + assertEquals("field1_value1", field1Result.get("field1_key1")); + assertEquals("field1_value2", field1Result.get("field1_key2")); + } + } + } + } + } + } +} diff --git a/java/core/src/test/java/com/lancedb/lance/operation/UpdateTest.java b/java/core/src/test/java/com/lancedb/lance/operation/UpdateTest.java new file mode 100644 index 00000000000..8f753d41fcf --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lance/operation/UpdateTest.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lance.operation; + +import com.lancedb.lance.Dataset; +import com.lancedb.lance.FragmentMetadata; +import com.lancedb.lance.TestUtils; +import com.lancedb.lance.Transaction; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UpdateTest extends OperationTestBase { + + @Test + void testUpdate(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testUpdate").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Commit fragment + int rowCount = 20; + FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount); + Transaction transaction = + dataset + .newTransactionBuilder() + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build(); + + try (Dataset dataset = transaction.commit()) { + assertEquals(2, dataset.version()); + assertEquals(2, dataset.latestVersion()); + assertEquals(rowCount, dataset.countRows()); + assertThrows( + IllegalArgumentException.class, + () -> + dataset + .newTransactionBuilder() + .operation(Append.builder().fragments(new ArrayList<>()).build()) + .build() + .commit() + .close()); + } + + dataset = Dataset.open(datasetPath, allocator); + // Update fragments + rowCount = 40; + FragmentMetadata newFragment = testDataset.createNewFragment(rowCount); + transaction = + dataset + .newTransactionBuilder() + .operation( + Update.builder() + .removedFragmentIds( + Collections.singletonList( + Long.valueOf(dataset.getFragments().get(0).getId()))) + .newFragments(Collections.singletonList(newFragment)) + .build()) + .build(); + + try (Dataset dataset = transaction.commit()) { + assertEquals(3, dataset.version()); + assertEquals(3, dataset.latestVersion()); + assertEquals(rowCount, dataset.countRows()); + + Transaction txn = dataset.readTransaction().orElse(null); + assertEquals(transaction, txn); + } + } + } +} From 4d9fef74d27df6be4e64a6bd9854a6aac1a46d98 Mon Sep 17 00:00:00 2001 From: majin1102 Date: Fri, 15 Aug 2025 21:56:36 +0800 Subject: [PATCH 2/8] mvn spotless --- .../com/lancedb/lance/TransactionTest.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/java/core/src/test/java/com/lancedb/lance/TransactionTest.java b/java/core/src/test/java/com/lancedb/lance/TransactionTest.java index 544db9f9aac..c681ccf3d29 100644 --- a/java/core/src/test/java/com/lancedb/lance/TransactionTest.java +++ b/java/core/src/test/java/com/lancedb/lance/TransactionTest.java @@ -1,13 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.lancedb.lance; import com.lancedb.lance.operation.Append; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.arrow.memory.RootAllocator; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -18,7 +33,8 @@ public class TransactionTest { public void testTransaction(@TempDir Path tempDir) { String datasetPath = tempDir.resolve("testTransaction").toString(); try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { - TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath); + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); try (Dataset dataset = testDataset.createEmptyDataset()) { FragmentMetadata fragmentMeta = testDataset.createNewFragment(20); @@ -28,7 +44,8 @@ public void testTransaction(@TempDir Path tempDir) { Transaction appendTxn = dataset .newTransactionBuilder() - .operation(Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) .transactionProperties(properties) .build(); try (Dataset committedDataset = appendTxn.commit()) { From 66fde395cdfaa6ccb194797eafe3199ce2ed31f6 Mon Sep 17 00:00:00 2001 From: majin1102 Date: Wed, 20 Aug 2025 11:14:04 +0800 Subject: [PATCH 3/8] fix merge test --- .../lance/operation/DataReplacementTest.java | 2 +- .../lancedb/lance/operation/MergeTest.java | 107 ++++++++++++++++-- .../lance/operation/OperationTestBase.java | 6 +- 3 files changed, 104 insertions(+), 11 deletions(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java index ce7322b8d3a..b2a1dc46364 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java @@ -114,7 +114,7 @@ void testDataReplacement(@TempDir Path tempDir) throws Exception { replaceVectorRoot.setRowCount(rowCount); DataFile datafile = - writeLanceDataFile(dataset.allocator(), datasetPath, replaceVectorRoot, 2); + writeLanceDataFile(dataset.allocator(), datasetPath, replaceVectorRoot, new int[]{2}, new int[]{0}); List replacementGroups = Collections.singletonList( new DataReplacement.DataReplacementGroup( diff --git a/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java index 7d17d906629..319fcf92059 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java @@ -36,6 +36,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class MergeTest extends OperationTestBase { @@ -55,7 +56,7 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { Arrays.asList( Field.nullable("id", new ArrowType.Int(32, true)), Field.nullable("name", new ArrowType.Utf8()), - Field.nullable("age", new ArrowType.Int(32, true))), + ageField), null); try (VectorSchemaRoot ageRoot = @@ -71,14 +72,20 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { DataFile ageDataFile = writeLanceDataFile( - dataset.allocator(), datasetPath, ageRoot, 2 // field index for age column + dataset.allocator(), + datasetPath, + ageRoot, + new int[] {2}, + new int[] {0} // field index for age column ); FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata(); + List dataFiles = fragmentMeta.getFiles(); + dataFiles.add(ageDataFile); FragmentMetadata evolvedFragment = new FragmentMetadata( fragmentMeta.getId(), - Collections.singletonList(ageDataFile), + dataFiles, fragmentMeta.getPhysicalRows(), fragmentMeta.getDeletionFile(), fragmentMeta.getRowIdMeta()); @@ -110,6 +117,89 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { for (int i = 0; i < rowCount; i++) { Assertions.assertEquals(20 + i, ageResultVector.get(i)); } + IntVector idResultVector = (IntVector) batch.getVector("id"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertEquals(i, idResultVector.get(i)); + } + } + } + } + } + } + } + } + + @Test + void testReplaceAsDiffColumns(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testMergeNewColumn").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + int rowCount = 15; + try (Dataset initialDataset = createAndAppendRows(testDataset, 15)) { + // Add a new column with different data type + Field ageField = Field.nullable("age", new ArrowType.Int(32, true)); + Field idField = Field.notNullable("id", new ArrowType.Int(32, true)); + List fields = Arrays.asList(idField, ageField); + Schema evolvedSchema = new Schema(fields, null); + + try (VectorSchemaRoot ageRoot = + VectorSchemaRoot.create(new Schema(fields, null), allocator)) { + ageRoot.allocateNew(); + IntVector ageVector = (IntVector) ageRoot.getVector("age"); + IntVector idVector = (IntVector) ageRoot.getVector("id"); + + for (int i = 0; i < rowCount; i++) { + ageVector.setSafe(i, 20 + i); + idVector.setSafe(i, i); + } + ageRoot.setRowCount(rowCount); + + DataFile ageDataFile = + writeLanceDataFile( + dataset.allocator(), datasetPath, ageRoot, new int[] {0, 1}, new int[] {0, 1}); + + FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata(); + FragmentMetadata evolvedFragment = + new FragmentMetadata( + fragmentMeta.getId(), + Collections.singletonList(ageDataFile), + fragmentMeta.getPhysicalRows(), + fragmentMeta.getDeletionFile(), + fragmentMeta.getRowIdMeta()); + + Transaction mergeTransaction = + initialDataset + .newTransactionBuilder() + .operation( + Merge.builder() + .fragments(Collections.singletonList(evolvedFragment)) + .schema(evolvedSchema) + .build()) + .build(); + + try (Dataset evolvedDataset = mergeTransaction.commit()) { + Assertions.assertEquals(3, evolvedDataset.version()); + Assertions.assertEquals(rowCount, evolvedDataset.countRows()); + Assertions.assertEquals(evolvedSchema, evolvedDataset.getSchema()); + Assertions.assertEquals(2, evolvedDataset.getSchema().getFields().size()); + // Verify merged data + try (LanceScanner scanner = evolvedDataset.newScan()) { + try (ArrowReader resultReader = scanner.scanBatches()) { + Assertions.assertTrue(resultReader.loadNextBatch()); + VectorSchemaRoot batch = resultReader.getVectorSchemaRoot(); + Assertions.assertEquals(rowCount, batch.getRowCount()); + Assertions.assertEquals(2, batch.getSchema().getFields().size()); + // Verify age column + IntVector ageResultVector = (IntVector) batch.getVector("age"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertEquals(20 + i, ageResultVector.get(i)); + } + IntVector idResultVector = (IntVector) batch.getVector("id"); + for (int i = 0; i < rowCount; i++) { + Assertions.assertEquals(i, idResultVector.get(i)); + } } } } @@ -148,15 +238,18 @@ void testMergeExistingColumn(@TempDir Path tempDir) throws Exception { dataset.allocator(), datasetPath, updatedNameRoot, - 1 // field index for name column + new int[] {1}, // field index for name column + new int[] {0} // column indices ); // Perform merge with updated column FragmentMetadata fragmentMeta = initialDataset.getFragment(0).metadata(); - FragmentMetadata updatedFragment = + List dataFiles = fragmentMeta.getFiles(); + dataFiles.add(updatedNameDataFile); + FragmentMetadata evolvedFragment = new FragmentMetadata( fragmentMeta.getId(), - Collections.singletonList(updatedNameDataFile), + dataFiles, fragmentMeta.getPhysicalRows(), fragmentMeta.getDeletionFile(), fragmentMeta.getRowIdMeta()); @@ -166,7 +259,7 @@ void testMergeExistingColumn(@TempDir Path tempDir) throws Exception { .newTransactionBuilder() .operation( Merge.builder() - .fragments(Collections.singletonList(updatedFragment)) + .fragments(Collections.singletonList(evolvedFragment)) .schema(testDataset.getSchema()) .build()) .build(); diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index a974ab723f0..9391cd6fd4b 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -64,7 +64,7 @@ protected Dataset createAndAppendRows(TestUtils.SimpleTestDataset suite, int row * LanceFileWriter to ensure compatibility with Lance format. */ protected DataFile writeLanceDataFile( - BufferAllocator allocator, String basePath, VectorSchemaRoot root, int fieldIndex) { + BufferAllocator allocator, String basePath, VectorSchemaRoot root, int[] fieldIndexes, int[] columnIndices) { // Create a unique file path for the data file String fileName = UUID.randomUUID() + ".lance"; String filePath = basePath + "/data/" + fileName; @@ -85,8 +85,8 @@ protected DataFile writeLanceDataFile( // Use a stable file format version return new DataFile( fileName, - new int[] {fieldIndex}, // Field index in the schema - new int[] {0}, // Column index in the file (always 0 for single column) + fieldIndexes, // Field index in the schema + columnIndices, // Just the same with fieldIndex for easy test TEST_FILE_FORMAT_MAJOR_VERSION, // File major version TEST_FILE_FORMAT_MINOR_VERSION, // File minor version file.length() // File size in bytes (now contains actual data) From 6bbcde5fc1d461c59741b94997fe6d37cb213d5a Mon Sep 17 00:00:00 2001 From: majin1102 Date: Thu, 21 Aug 2025 01:11:10 +0800 Subject: [PATCH 4/8] mvn spotless --- .../com/lancedb/lance/operation/DataReplacementTest.java | 7 ++++++- .../com/lancedb/lance/operation/OperationTestBase.java | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java index b2a1dc46364..7b5696726b4 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/DataReplacementTest.java @@ -114,7 +114,12 @@ void testDataReplacement(@TempDir Path tempDir) throws Exception { replaceVectorRoot.setRowCount(rowCount); DataFile datafile = - writeLanceDataFile(dataset.allocator(), datasetPath, replaceVectorRoot, new int[]{2}, new int[]{0}); + writeLanceDataFile( + dataset.allocator(), + datasetPath, + replaceVectorRoot, + new int[] {2}, + new int[] {0}); List replacementGroups = Collections.singletonList( new DataReplacement.DataReplacementGroup( diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index 9391cd6fd4b..0cd76eafe5d 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -64,7 +64,11 @@ protected Dataset createAndAppendRows(TestUtils.SimpleTestDataset suite, int row * LanceFileWriter to ensure compatibility with Lance format. */ protected DataFile writeLanceDataFile( - BufferAllocator allocator, String basePath, VectorSchemaRoot root, int[] fieldIndexes, int[] columnIndices) { + BufferAllocator allocator, + String basePath, + VectorSchemaRoot root, + int[] fieldIndexes, + int[] columnIndices) { // Create a unique file path for the data file String fileName = UUID.randomUUID() + ".lance"; String filePath = basePath + "/data/" + fileName; From bbd80f04ba2af4cda3a80b206eb6fb5916a55cc5 Mon Sep 17 00:00:00 2001 From: majin1102 Date: Mon, 25 Aug 2025 16:22:35 +0800 Subject: [PATCH 5/8] adapt shallow_clone --- .../java/com/lancedb/lance/operation/OperationTestBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index 0cd76eafe5d..4cd64906847 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -93,7 +93,8 @@ protected DataFile writeLanceDataFile( columnIndices, // Just the same with fieldIndex for easy test TEST_FILE_FORMAT_MAJOR_VERSION, // File major version TEST_FILE_FORMAT_MINOR_VERSION, // File minor version - file.length() // File size in bytes (now contains actual data) + file.length(), // File size in bytes (now contains actual data) + null ); } } From 389ec7831bd03a43c33376ce58feed65531117fa Mon Sep 17 00:00:00 2001 From: majin1102 Date: Mon, 25 Aug 2025 16:24:35 +0800 Subject: [PATCH 6/8] mvn spotless --- .../java/com/lancedb/lance/operation/OperationTestBase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index 4cd64906847..fb62644e7d1 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -94,7 +94,6 @@ protected DataFile writeLanceDataFile( TEST_FILE_FORMAT_MAJOR_VERSION, // File major version TEST_FILE_FORMAT_MINOR_VERSION, // File minor version file.length(), // File size in bytes (now contains actual data) - null - ); + null); } } From e5604ddd29614d41791fe7d1f3b1f0ab0b4fd1cc Mon Sep 17 00:00:00 2001 From: majin1102 Date: Mon, 25 Aug 2025 21:32:12 +0800 Subject: [PATCH 7/8] address comments --- .../test/java/com/lancedb/lance/operation/MergeTest.java | 2 +- .../com/lancedb/lance/operation/OperationTestBase.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java index 319fcf92059..9d8ef801d30 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/MergeTest.java @@ -131,7 +131,7 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { @Test void testReplaceAsDiffColumns(@TempDir Path tempDir) throws Exception { - String datasetPath = tempDir.resolve("testMergeNewColumn").toString(); + String datasetPath = tempDir.resolve("testReplaceAsDiffColumns").toString(); try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath); diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index fb62644e7d1..0a10cf7437f 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -28,18 +28,20 @@ import java.io.File; import java.util.Collections; import java.util.UUID; +import org.junit.jupiter.api.TestInstance; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class OperationTestBase { public static final int TEST_FILE_FORMAT_MAJOR_VERSION = 2; public static final int TEST_FILE_FORMAT_MINOR_VERSION = 0; - protected static Dataset dataset; + protected Dataset dataset; @BeforeAll - static void setup() {} + void setup() {} @AfterAll - static void tearDown() { + void tearDown() { // Cleanup resources used by the tests if (dataset != null) { dataset.close(); From 740799b43b08c25907c8c99707d677b8ccfe216b Mon Sep 17 00:00:00 2001 From: majin1102 Date: Mon, 25 Aug 2025 21:35:22 +0800 Subject: [PATCH 8/8] spotless --- .../java/com/lancedb/lance/operation/OperationTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java index 0a10cf7437f..f71f155e9b0 100644 --- a/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java +++ b/java/core/src/test/java/com/lancedb/lance/operation/OperationTestBase.java @@ -24,11 +24,11 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; import java.io.File; import java.util.Collections; import java.util.UUID; -import org.junit.jupiter.api.TestInstance; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class OperationTestBase {