diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java new file mode 100644 index 0000000000..3db3280a65 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java @@ -0,0 +1,76 @@ +/* + * DataInKeySpacePath.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.foundationdb.tuple.TupleHelpers; + +import java.util.concurrent.CompletableFuture; + +/** + * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}. + */ +public class DataInKeySpacePath { + + final CompletableFuture resolvedPath; + final KeyValue rawKeyValue; + + public DataInKeySpacePath(KeySpacePath path, KeyValue rawKeyValue, FDBRecordContext context) { + this.rawKeyValue = rawKeyValue; + + // Convert the raw key to a Tuple and resolve it starting from the provided path + Tuple keyTuple = Tuple.fromBytes(rawKeyValue.getKey()); + + // First resolve the provided path to get its resolved form + this.resolvedPath = path.toResolvedPathAsync(context).thenCompose(resolvedPath -> { + // Now use the resolved path to find the child for the key + // We need to figure out how much of the key corresponds to the resolved path + Tuple pathTuple = resolvedPath.toTuple(); + int pathLength = pathTuple.size(); + + // The remaining part of the key should be resolved from the resolved path's directory + if (keyTuple.size() > pathLength) { + // There's more in the key than just the path, so resolve the rest + if (resolvedPath.getDirectory().getSubdirectories().isEmpty()) { + return CompletableFuture.completedFuture( + new ResolvedKeySpacePath(resolvedPath.getParent(), resolvedPath.toPath(), + resolvedPath.getResolvedPathValue(), + TupleHelpers.subTuple(keyTuple, pathTuple.size(), keyTuple.size()))); + } else { + return resolvedPath.getDirectory().findChildForKey(context, resolvedPath, keyTuple, keyTuple.size(), pathLength); + } + } else { + // The key exactly matches the path + return CompletableFuture.completedFuture(resolvedPath); + } + }); + } + + public CompletableFuture getResolvedPath() { + return resolvedPath; + } + + public KeyValue getRawKeyValue() { + return rawKeyValue; + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index d52861ec69..501079240d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -566,4 +566,20 @@ default List listSubdirectory(@Nonnull FDBRecordContext co */ @API(API.Status.UNSTABLE) String toString(@Nonnull Tuple tuple); + + /** + * Export all data stored under this KeySpacePath and return it in a RecordCursor. + * This method scans all keys that have this path as a prefix and returns the key-value pairs. + * Supports continuation to resume scanning from a previous position. + * + * @param context the transaction context in which to perform the data export + * @param continuation optional continuation from a previous export operation, or null to start from the beginning + * @param scanProperties properties controlling how the scan should be performed + * @return a RecordCursor that iterates over all KeyValue pairs under this path + */ + @API(API.Status.UNSTABLE) + @Nonnull + RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 7102da8936..fc8fa45ab5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -26,10 +26,14 @@ import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.ValueRange; +import com.apple.foundationdb.record.cursors.LazyCursor; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor; +import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; import com.google.common.collect.Lists; + import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; @@ -337,6 +341,21 @@ public String toString() { return toString(null); } + @Nonnull + @Override + public RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties) { + return new LazyCursor<>(toTupleAsync(context) + .thenApply(tuple -> KeyValueCursor.Builder.withSubspace(new Subspace(tuple)) + .setContext(context) + .setContinuation(continuation) + .setScanProperties(scanProperties) + .build()), + context.getExecutor()) + .map(keyValue -> new DataInKeySpacePath(this, keyValue, context)); + } + /** * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java index 769649b8da..54d794f2a7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java @@ -226,4 +226,12 @@ public String toString() { public String toString(@Nonnull Tuple t) { return inner.toString(t); } + + @Nonnull + @Override + public RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties) { + return inner.exportAllData(context, continuation, scanProperties); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java new file mode 100644 index 0000000000..1390e3a755 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java @@ -0,0 +1,333 @@ +/* + * DataInKeySpacePathTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.foundationdb.tuple.TupleHelpers; +import com.apple.test.Tags; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link DataInKeySpacePath}. + */ +@Tag(Tags.RequiresFDB) +class DataInKeySpacePathTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3, 4, 5}) + void resolution() { + // Include some extra children to make sure resolution doesn't get confused + final String companyUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, companyUuid) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("team_id", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("employee_uuid", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL, null)) + .addSubdirectory(new KeySpaceDirectory("metaData", KeyType.LONG, 0)))) + .addSubdirectory(new KeySpaceDirectory("buildings", KeyType.STRING))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + UUID employeeId = UUID.randomUUID(); + KeySpacePath employeePath = root.path("company") + .add("department", "engineering") + .add("team_id", 42L) + .add("employee_uuid", employeeId) + .add("active", true) + .add("data"); + + // Add additional tuple elements after the KeySpacePath + final Tuple remainderTuple = Tuple.from("salary", 75000L, "start_date", "2023-01-15"); + byte[] keyBytes = employeePath.toSubspace(context).pack(remainderTuple); + byte[] valueBytes = Tuple.from("employee_record").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the company-level path + KeySpacePath companyPath = root.path("company"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(companyPath, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path + ResolvedKeySpacePath activeLevel = assertNameAndValue(resolved, "data", null); + ResolvedKeySpacePath uuidLevel = assertNameAndValue(activeLevel, "active", true); + ResolvedKeySpacePath teamLevel = assertNameAndValue(uuidLevel, "employee_uuid", employeeId); + ResolvedKeySpacePath deptLevel = assertNameAndValue(teamLevel, "team_id", 42L); + ResolvedKeySpacePath companyLevel = assertNameAndValue(deptLevel, "department", "engineering"); + assertNull(assertNameAndValue(companyLevel, "company", companyUuid)); + + // Verify the resolved path recreates the KeySpacePath portion + assertEquals(TupleHelpers.subTuple(Tuple.fromBytes(keyBytes), 0, 6), resolved.toTuple()); + + // Verify that the remainder contains the additional tuple elements + assertEquals(remainderTuple, resolved.getRemainder()); + + context.commit(); + } + } + + @Test + void pathWithConstantValues() { + final String appUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("application", KeyType.STRING, appUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("environment", KeyType.STRING, "production") + .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath dataPath = root.path("application") + .add("version") // Uses constant value 1L + .add("environment") // Uses constant value "production" + .add("data", "user_records"); + // Add additional tuple elements after the KeySpacePath + byte[] keyBytes = dataPath.toSubspace(context).pack( + Tuple.from("config_id", 1001L, "version", "v2.1")); + byte[] valueBytes = Tuple.from("constant_test_data").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the application-level path + KeySpacePath appPath = root.path("application"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(appPath, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path using assertNameAndValue + ResolvedKeySpacePath envLevel = assertNameAndValue(resolved, "data", "user_records"); + ResolvedKeySpacePath versionLevel = assertNameAndValue(envLevel, "environment", "production"); + ResolvedKeySpacePath applicationLevel = assertNameAndValue(versionLevel, "version", 1L); + assertNull(assertNameAndValue(applicationLevel, "application", appUuid)); + + // Verify the resolved path recreates the KeySpacePath portion + assertEquals(TupleHelpers.subTuple(Tuple.fromBytes(keyBytes), 0, 4), resolved.toTuple()); + + // Verify that the remainder contains the additional tuple elements + Tuple remainder = resolved.getRemainder(); + assertNotNull(remainder); + assertEquals(4, remainder.size()); + assertEquals("config_id", remainder.getString(0)); + assertEquals(1001L, remainder.getLong(1)); + assertEquals("version", remainder.getString(2)); + assertEquals("v2.1", remainder.getString(3)); + + context.commit(); + } + } + + @Test + void pathWithDirectoryLayer() { + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG) + .addSubdirectory(new DirectoryLayerDirectory("service")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath servicePath = root.path("tenant") + .add("user_id", 999L) + .add("service", "analytics"); + + Tuple keyTuple = servicePath.toTuple(context); + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = Tuple.from("directory_layer_data").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the tenant-level path + KeySpacePath tenantPath = root.path("tenant"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(tenantPath, keyValue, context); + + ResolvedKeySpacePath serviceLevel = dataInPath.getResolvedPath().join(); + + final ResolvedKeySpacePath userLevel = assertNameAndDirectoryScopedValue( + serviceLevel, "service", "analytics", servicePath, context); + ResolvedKeySpacePath tenantLevel = assertNameAndValue(userLevel, "user_id", 999L); + + assertNull(assertNameAndDirectoryScopedValue(tenantLevel, "tenant", tenantUuid, tenantPath, context)); + + context.commit(); + } + } + + @Test + void pathWithBinaryData() { + final String storeUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("binary_store", KeyType.STRING, storeUuid) + .addSubdirectory(new KeySpaceDirectory("blob_id", KeyType.BYTES))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + byte[] blobId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); + + Tuple keyTuple = blobPath.toTuple(context); + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = "binary_test_data".getBytes(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the binary_store-level path + KeySpacePath storePath = root.path("binary_store"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(storePath, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path using assertNameAndValue + assertEquals("blob_id", resolved.getDirectoryName()); + byte[] resolvedBytes = (byte[]) resolved.getResolvedValue(); + assertArrayEquals(blobId, resolvedBytes); + + ResolvedKeySpacePath storeLevel = assertNameAndValue(resolved.getParent(), "binary_store", storeUuid); + assertNull(storeLevel); + + // Verify the resolved path can recreate the original key + assertEquals(keyTuple, resolved.toTuple()); + + context.commit(); + } + } + + @Test + void keyValueAccessors() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + Tuple keyTuple = testPath.toTuple(context); + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = Tuple.from("accessor_test").pack(); + + KeyValue originalKeyValue = new KeyValue(keyBytes, valueBytes); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(testPath, originalKeyValue, context); + + // Verify accessor methods + KeyValue retrievedKeyValue = dataInPath.getRawKeyValue(); + assertNotNull(retrievedKeyValue); + assertEquals(originalKeyValue.getKey(), retrievedKeyValue.getKey()); + assertEquals(originalKeyValue.getValue(), retrievedKeyValue.getValue()); + + // Verify resolved path future is not null + CompletableFuture resolvedFuture = dataInPath.getResolvedPath(); + assertNotNull(resolvedFuture); + assertTrue(resolvedFuture.isDone() || !resolvedFuture.isCancelled()); + } + } + + @Test + void withWrapper() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Test 4: Export from specific data store level + final EnvironmentKeySpace.ApplicationPath appPath = keySpace.root().userid(100L).application("app1"); + EnvironmentKeySpace.DataPath dataStore = appPath.dataStore(); + + final byte[] key = dataStore.toTuple(context).add("record2").add(0).pack(); + final byte[] value = Tuple.from("data").pack(); + final DataInKeySpacePath dataInKeySpacePath = new DataInKeySpacePath(dataStore, new KeyValue(key, value), context); + + final ResolvedKeySpacePath resolvedPath = dataInKeySpacePath.getResolvedPath().join(); + assertEquals(dataStore.toResolvedPath(context), withoutRemainder(resolvedPath)); + assertEquals(Tuple.from("record2", 0), resolvedPath.getRemainder()); + + // Verify the path using assertNameAndValue + // Note: We expect the path to be: [environment] -> userid -> application -> data + ResolvedKeySpacePath appLevel; + appLevel = assertNameAndValue(resolvedPath, "data", EnvironmentKeySpace.DATA_VALUE); + ResolvedKeySpacePath userLevel = assertNameAndDirectoryScopedValue(appLevel, "application", "app1", + appPath, context); + ResolvedKeySpacePath envLevel = assertNameAndValue(userLevel, "userid", 100L); + assertNull(assertNameAndDirectoryScopedValue(envLevel, keySpace.root().getDirectoryName(), + keySpace.root().getValue(), keySpace.root(), context)); + } + } + + private static ResolvedKeySpacePath assertNameAndDirectoryScopedValue(ResolvedKeySpacePath resolved, + String name, Object logicalValue, + KeySpacePath path, FDBRecordContext context) { + assertNotNull(resolved); + assertEquals(name, resolved.getDirectoryName()); + assertEquals(path.toResolvedPath(context).getResolvedValue(), resolved.getResolvedValue()); + assertEquals(logicalValue, resolved.getLogicalValue()); + return resolved.getParent(); + } + + private static ResolvedKeySpacePath assertNameAndValue(ResolvedKeySpacePath resolved, String name, Object value) { + assertNotNull(resolved); + assertEquals(name, resolved.getDirectoryName()); + assertEquals(value, resolved.getResolvedValue()); + assertEquals(value, resolved.getLogicalValue()); + return resolved.getParent(); + } + + private ResolvedKeySpacePath withoutRemainder(final ResolvedKeySpacePath path) { + return new ResolvedKeySpacePath(path.getParent(), path.toPath(), path.getResolvedPathValue(), null); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java new file mode 100644 index 0000000000..c1751395cf --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java @@ -0,0 +1,195 @@ +/* + * EnvironmentKeySpace.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; +import com.apple.foundationdb.tuple.Tuple; + +import javax.annotation.Nonnull; +import java.util.UUID; + +/** + * This provides an example of a way in which you can define a KeySpace in a relatively clean and type-safe + * manner. It defines a keyspace that looks like: + * 
+ *    [environment]           - A string the identifies the logical environment (like prod, test, qa, etc.).
+ *      |                       This string is converted by the directory layer as a small integer value.
+ *      +- userid             - An integer ID for each user in the system
+ *         |
+ *         +- [application]   - Tne name of an application the user runs (again, converted by the directory
+ *            |                 layer into a small integer value)
+ *            +- data=1       - Constant value of "1", which is the location of a {@link FDBRecordStore}
+ *            |                 in which application data is to be stored
+ *            +- metadata=2   - Constant value of "2", which is the Location of another FDBRecordStore
+ *                              in which application metadata or configuration information can live.
+ *
+ * The main point of this class is to demonstrate how you can use the KeySpacePath wrapping facility to provide + * implementations of the path elements that are meaningful to your application environment and type safe. + */ +class EnvironmentKeySpace { + private final KeySpace root; + private final String rootName; + + static final String USER_KEY = "userid"; + static final String APPLICATION_KEY = "application"; + static final String DATA_KEY = "data"; + static final long DATA_VALUE = 1L; + static final String METADATA_KEY = "metadata"; + static final long METADATA_VALUE = 2L; + + /** + * The EnvironmentKeySpace scopes all of the data it stores underneath of a rootName, + * for example, you could define an instance for prod, test, qa, etc. + * + * @param rootName The root name underwhich all data is stored. + */ + public EnvironmentKeySpace(String rootName) { + this.rootName = rootName; + root = new KeySpace( + new DirectoryLayerDirectory(rootName, rootName, EnvironmentRoot::new) + .addSubdirectory(new KeySpaceDirectory(USER_KEY, KeySpaceDirectory.KeyType.LONG, UserPath::new) + .addSubdirectory(new DirectoryLayerDirectory(APPLICATION_KEY, ApplicationPath::new) + .addSubdirectory(new KeySpaceDirectory(DATA_KEY, KeySpaceDirectory.KeyType.LONG, DATA_VALUE, DataPath::new)) + .addSubdirectory(new KeySpaceDirectory(METADATA_KEY, KeySpaceDirectory.KeyType.LONG, METADATA_VALUE, MetadataPath::new))))); + } + + @Nonnull + static EnvironmentKeySpace setupSampleData(@Nonnull final FDBDatabase database) { + EnvironmentKeySpace keySpace = new EnvironmentKeySpace(UUID.randomUUID().toString()); + + // Store test data at different levels of the hierarchy + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create paths for different users and applications + ApplicationPath app1User1 = keySpace.root().userid(100L).application("app1"); + ApplicationPath app2User1 = keySpace.root().userid(100L).application("app2"); + ApplicationPath app1User2 = keySpace.root().userid(200L).application("app1"); + + DataPath dataUser1App1 = app1User1.dataStore(); + MetadataPath metaUser1App1 = app1User1.metadataStore(); + DataPath dataUser1App2 = app2User1.dataStore(); + DataPath dataUser2App1 = app1User2.dataStore(); + + // Store data records with additional tuple elements after the KeySpacePath + tr.set(dataUser1App1.toTuple(context).add("record1").pack(), Tuple.from("user100_app1_data1").pack()); + tr.set(dataUser1App1.toTuple(context).add("record2").add(0).pack(), Tuple.from("user100_app1_data2_0").pack()); + tr.set(dataUser1App1.toTuple(context).add("record2").add(1).pack(), Tuple.from("user100_app1_data2_1").pack()); + tr.set(metaUser1App1.toTuple(context).add("config1").pack(), Tuple.from("user100_app1_meta1").pack()); + tr.set(dataUser1App2.toTuple(context).add("record3").pack(), Tuple.from("user100_app2_data3").pack()); + tr.set(dataUser2App1.toTuple(context).add("record4").pack(), Tuple.from("user200_app1_data4").pack()); + + context.commit(); + } + return keySpace; + } + + public String getRootName() { + return rootName; + } + + /** + * Returns an implementation of a KeySpacePath that represents the start of the environment. + */ + public EnvironmentRoot root() { + return (EnvironmentRoot)root.path(rootName); + } + + /** + * Given a tuple that represents an FDB key that came from this KeySpace, returns the leaf-most path + * element in which the tuple resides. + */ + public ResolvedKeySpacePath fromKey(FDBRecordContext context, Tuple tuple) { + return root.resolveFromKey(context, tuple); + } + + /** + * A KeySpacePath that represents the logical root of the environment. + */ + static class EnvironmentRoot extends KeySpacePathWrapper { + public EnvironmentRoot(KeySpacePath path) { + super(path); + } + + public KeySpacePath parent() { + return null; + } + + public UserPath userid(long userid) { + return (UserPath) inner.add(USER_KEY, userid); + } + } + + static class UserPath extends KeySpacePathWrapper { + public UserPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath application(String applicationName) { + return (ApplicationPath) inner.add(APPLICATION_KEY, applicationName); + } + + public EnvironmentRoot parent() { + return (EnvironmentRoot) inner.getParent(); + } + } + + static class ApplicationPath extends KeySpacePathWrapper { + public ApplicationPath(KeySpacePath path) { + super(path); + } + + public DataPath dataStore() { + return (DataPath) inner.add(DATA_KEY); + } + + public MetadataPath metadataStore() { + return (MetadataPath) inner.add(METADATA_KEY); + } + + public UserPath parent() { + return (UserPath) inner.getParent(); + } + } + + static class DataPath extends KeySpacePathWrapper { + public DataPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath parent() { + return (ApplicationPath) inner.getParent(); + } + } + + static class MetadataPath extends KeySpacePathWrapper { + public MetadataPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath parent() { + return (ApplicationPath) inner.getParent(); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java index 0263ca9bf4..5c084cb3fa 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; import com.apple.foundationdb.record.provider.foundationdb.layers.interning.ScopedInterningLayer; @@ -1336,7 +1335,7 @@ public void testPathWrapperExample() throws Exception { final Tuple dataStoreTuple; final Tuple metadataStoreTuple; try (FDBRecordContext context = database.openContext()) { - ApplicationPath application = keySpace.root().userid(123).application("myApplication"); + EnvironmentKeySpace.ApplicationPath application = keySpace.root().userid(123).application("myApplication"); dataStoreTuple = application.dataStore().toTuple(context); metadataStoreTuple = application.metadataStore().toTuple(context); context.commit(); @@ -1350,9 +1349,9 @@ public void testPathWrapperExample() throws Exception { assertEquals(Tuple.from(entries.get(0), 123L, entries.get(1), EnvironmentKeySpace.METADATA_VALUE), metadataStoreTuple); ResolvedKeySpacePath path = keySpace.fromKey(context, dataStoreTuple); - assertThat(path.toPath(), instanceOf(DataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.DataPath.class)); - DataPath mainStorePath = (DataPath) path.toPath(); + EnvironmentKeySpace.DataPath mainStorePath = (EnvironmentKeySpace.DataPath) path.toPath(); assertEquals(EnvironmentKeySpace.DATA_VALUE, mainStorePath.getValue()); assertEquals(EnvironmentKeySpace.DATA_VALUE, mainStorePath.resolveAsync(context).get().getResolvedValue()); assertEquals(entries.get(1), mainStorePath.parent().resolveAsync(context).get().getResolvedValue()); @@ -1362,14 +1361,14 @@ public void testPathWrapperExample() throws Exception { assertEquals("production", mainStorePath.parent().parent().parent().getValue()); assertNull(mainStorePath.parent().parent().parent().parent()); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentRoot.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(UserPath.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(ApplicationPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentKeySpace.EnvironmentRoot.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(EnvironmentKeySpace.UserPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(EnvironmentKeySpace.ApplicationPath.class)); path = keySpace.fromKey(context, metadataStoreTuple); - assertThat(path.toPath(), instanceOf(MetadataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.MetadataPath.class)); - MetadataPath metadataPath = (MetadataPath) path.toPath(); + EnvironmentKeySpace.MetadataPath metadataPath = (EnvironmentKeySpace.MetadataPath) path.toPath(); assertEquals(EnvironmentKeySpace.METADATA_VALUE, metadataPath.getValue()); assertEquals(EnvironmentKeySpace.METADATA_VALUE, metadataPath.resolveAsync(context).get().getResolvedValue()); assertEquals(entries.get(1), metadataPath.parent().resolveAsync(context).get().getResolvedValue()); @@ -1379,14 +1378,14 @@ public void testPathWrapperExample() throws Exception { assertEquals("production", metadataPath.parent().parent().parent().getValue()); assertNull(metadataPath.parent().parent().parent().parent()); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentRoot.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(UserPath.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(ApplicationPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentKeySpace.EnvironmentRoot.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(EnvironmentKeySpace.UserPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(EnvironmentKeySpace.ApplicationPath.class)); // Create a fake main store "record" key to demonstrate that we can get the key as the remainder Tuple recordTuple = dataStoreTuple.add(1L).add("someStr").add(0L); // 1=record space, record id, 0=unsplit record path = keySpace.fromKey(context, recordTuple); - assertThat(path.toPath(), instanceOf(DataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.DataPath.class)); assertEquals(Tuple.from(1L, "someStr", 0L), path.getRemainder()); assertEquals(dataStoreTuple, path.toTuple()); } @@ -1538,137 +1537,4 @@ protected CompletableFuture toTupleValueAsyncImpl(@Nonnull FDBRecordC } } - /** - * This provides an example of a way in which you can define a KeySpace in a relatively clean and type-safe - * manner. It defines a keyspace that looks like: - * 
-     *    [environment]           - A string the identifies the logical environment (like prod, test, qa, etc.).
-     *      |                       This string is converted by the directory layer as a small integer value.
-     *      +- userid             - An integer ID for each user in the system
-     *         |
-     *         +- [application]   - Tne name of an application the user runs (again, converted by the directory
-     *            |                 layer into a small integer value)
-     *            +- data=1       - Constant value of "1", which is the location of a {@link FDBRecordStore}
-     *            |                 in which application data is to be stored
-     *            +- metadata=2   - Constant value of "2", which is the Location of another FDBRecordStore
-     *                              in which application metadata or configuration information can live.
-     *
- * The main point of this class is to demonstrate how you can use the KeySpacePath wrapping facility to provide - * implementations of the path elements that are meaningful to your application environment and type safe. - */ - private static class EnvironmentKeySpace { - private final KeySpace root; - private final String rootName; - - public static String USER_KEY = "userid"; - public static String APPLICATION_KEY = "application"; - public static String DATA_KEY = "data"; - public static long DATA_VALUE = 1L; - public static String METADATA_KEY = "metadata"; - public static long METADATA_VALUE = 2L; - - /** - * The EnvironmentKeySpace scopes all of the data it stores underneath of a rootName, - * for example, you could define an instance for prod, test, qa, etc. - * - * @param rootName The root name underwhich all data is stored. - */ - public EnvironmentKeySpace(String rootName) { - this.rootName = rootName; - root = new KeySpace( - new DirectoryLayerDirectory(rootName, rootName, EnvironmentRoot::new) - .addSubdirectory(new KeySpaceDirectory(USER_KEY, KeyType.LONG, UserPath::new) - .addSubdirectory(new DirectoryLayerDirectory(APPLICATION_KEY, ApplicationPath::new) - .addSubdirectory(new KeySpaceDirectory(DATA_KEY, KeyType.LONG, DATA_VALUE, DataPath::new)) - .addSubdirectory(new KeySpaceDirectory(METADATA_KEY, KeyType.LONG, METADATA_VALUE, MetadataPath::new))))); - } - - public String getRootName() { - return rootName; - } - - /** - * Returns an implementation of a KeySpacePath that represents the start of the environment. - */ - public EnvironmentRoot root() { - return (EnvironmentRoot) root.path(rootName); - } - - /** - * Given a tuple that represents an FDB key that came from this KeySpace, returns the leaf-most path - * element in which the tuple resides. - */ - public ResolvedKeySpacePath fromKey(FDBRecordContext context, Tuple tuple) { - return root.resolveFromKey(context, tuple); - } - } - - /** - * A KeySpacePath that represents the logical root of the environment. - */ - private static class EnvironmentRoot extends KeySpacePathWrapper { - public EnvironmentRoot(KeySpacePath path) { - super(path); - } - - public KeySpacePath parent() { - return null; - } - - public UserPath userid(long userid) { - return (UserPath) inner.add(EnvironmentKeySpace.USER_KEY, userid); - } - } - - private static class UserPath extends KeySpacePathWrapper { - public UserPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath application(String applicationName) { - return (ApplicationPath) inner.add(EnvironmentKeySpace.APPLICATION_KEY, applicationName); - } - - public EnvironmentRoot parent() { - return (EnvironmentRoot) inner.getParent(); - } - } - - private static class ApplicationPath extends KeySpacePathWrapper { - public ApplicationPath(KeySpacePath path) { - super(path); - } - - public DataPath dataStore() { - return (DataPath) inner.add(EnvironmentKeySpace.DATA_KEY); - } - - public MetadataPath metadataStore() { - return (MetadataPath) inner.add(EnvironmentKeySpace.METADATA_KEY); - } - - public UserPath parent() { - return (UserPath) inner.getParent(); - } - } - - private static class DataPath extends KeySpacePathWrapper { - public DataPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath parent() { - return (ApplicationPath) inner.getParent(); - } - } - - private static class MetadataPath extends KeySpacePathWrapper { - public MetadataPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath parent() { - return (ApplicationPath) inner.getParent(); - } - } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java new file mode 100644 index 0000000000..d835a8d2ef --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -0,0 +1,619 @@ +/* + * KeySpacePathDataExportTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorStartContinuation; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.Tags; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for the new KeySpacePath data export feature that fetches all data stored under a KeySpacePath + * and returns it in a {@code RecordCursor}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathDataExportTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + + @Test + void exportAllDataFromSimplePath() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath basePath = root.path("root"); + + // Add data at different levels + for (int i = 0; i < 5; i++) { + Tuple key = basePath.add("level1", (long) i).toTuple(context); + tr.set(key.pack(), Tuple.from("value" + i).pack()); + + // Add some sub-data under each key + for (int j = 0; j < 3; j++) { + Tuple subKey = key.add("sub" + j); + tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack()); + } + } + context.commit(); + } + + // Export all data from the root path + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = root.path("root"); + final List allData = exportAllData(rootPath, context); + + // Should have 5 main entries + 15 sub-entries = 20 total + assertEquals(20, allData.size()); + + // Verify the data is sorted by key + for (int i = 1; i < allData.size(); i++) { + assertTrue(Tuple.fromBytes(allData.get(i - 1).getKey()).compareTo( + Tuple.fromBytes(allData.get(i).getKey())) < 0); + } + } + } + + @Test + void exportAllDataFromSpecificSubPath() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("app", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("user", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL)))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data for multiple users + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + for (long userId = 1; userId <= 3; userId++) { + KeySpacePath userPath = root.path("app").add("user", userId); + KeySpacePath dataPath = userPath.add("data"); + + // Add data for each user + for (int i = 0; i < 4; i++) { + Tuple key = dataPath.toTuple(context).add("record" + i); + tr.set(key.pack(), Tuple.from("user" + userId + "_data" + i).pack()); + } + } + context.commit(); + } + + // Export data only for user 2 + try (FDBRecordContext context = database.openContext()) { + KeySpacePath user2Path = root.path("app").add("user", 2L); + final List userData = exportAllData(user2Path, context); + + // Should have 4 records for user 2 + assertEquals(4, userData.size()); + + // Verify all data belongs to user 2 + for (KeyValue kv : userData) { + String value = Tuple.fromBytes(kv.getValue()).getString(0); + assertTrue(value.startsWith("user2_")); + } + } + } + + @Test + void exportAllDataWithDirectoryLayer() { + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("env", UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("tenant", KeyType.LONG) + .addSubdirectory(new DirectoryLayerDirectory("service")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath basePath = root.path("env").add("tenant", 100L); + + // Add data for different services + String[] services = {"auth", "storage", "compute"}; + for (String service : services) { + KeySpacePath servicePath = basePath.add("service", service); + Tuple serviceKey = servicePath.toTuple(context); + + for (int i = 0; i < 2; i++) { + tr.set(serviceKey.add("config" + i).pack(), + Tuple.from(service + "_config_" + i).pack()); + } + } + context.commit(); + } + + // Export all data from tenant path + try (FDBRecordContext context = database.openContext()) { + KeySpacePath tenantPath = root.path("env").add("tenant", 100L); + final List allData = exportAllData(tenantPath, context); + + // Should have 6 records (3 services * 2 configs each) + assertEquals(6, allData.size()); + + // Verify we have data for all three services + Set serviceNames = new HashSet<>(); + for (KeyValue kv : allData) { + String value = Tuple.fromBytes(kv.getValue()).getString(0); + String serviceName = value.split("_")[0]; + serviceNames.add(serviceName); + } + assertEquals(3, serviceNames.size()); + assertTrue(serviceNames.containsAll(Arrays.asList("auth", "storage", "compute"))); + } + } + + @Test + void exportAllDataWithDifferentKeyTypes() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("mixed", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("strings", KeyType.STRING)) + .addSubdirectory(new KeySpaceDirectory("longs", KeyType.LONG)) + .addSubdirectory(new KeySpaceDirectory("bytes", KeyType.BYTES)) + .addSubdirectory(new KeySpaceDirectory("uuids", KeyType.UUID)) + .addSubdirectory(new KeySpaceDirectory("booleans", KeyType.BOOLEAN))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data with different key types + try (FDBRecordContext context = database.openContext()) { + KeySpacePath basePath = root.path("mixed"); + + // String keys (str0, str1, str2 -> string_value_0, string_value_1, string_value_2) + setData(List.of("str0", "str1", "str2"), context, basePath, "strings", "string_value_"); + + // Long keys (10, 11, 12 -> long_value_10, long_value_11, long_value_12) + setData(List.of(10L, 11L, 12L), context, basePath, "longs", "long_value_"); + + // Bytes keys (arrays -> bytes_value_[0, 1], bytes_value_[1, 2]) + setData(List.of(new byte[]{0, 1}, new byte[]{1, 2}), context, basePath, "bytes", "bytes_value_"); + + // UUID keys (UUIDs -> uuid_value_UUID) + setData(List.of(new UUID(0, 0), new UUID(1, 1)), context, basePath, "uuids", "uuid_value_"); + + // Boolean keys (true, false -> boolean_value_true, boolean_value_false) + setData(List.of(true, false), context, basePath, "booleans", "boolean_value_"); + + context.commit(); + } + + // Export all data and verify different key types + try (FDBRecordContext context = database.openContext()) { + KeySpacePath mixedPath = root.path("mixed"); + final List allData = exportAllData(mixedPath, context); + + // Should have 12 records total (3+3+2+2+2) + assertEquals(12, allData.size()); + + // Verify we have different value types + Set valueTypes = allData.stream() + .map(kv -> Tuple.fromBytes(kv.getValue()).getString(0).split("_")[0]) + .collect(Collectors.toSet()); + assertEquals(5, valueTypes.size()); + assertTrue(valueTypes.containsAll(Arrays.asList("string", "long", "bytes", "uuid", "boolean"))); + } + } + + @Test + void exportAllDataWithConstantValues() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("app", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING, "records")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data using constant values + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath dataPath = root.path("app").add("version").add("data"); + Tuple baseKey = dataPath.toTuple(context); + + // Add multiple records under the constant path + for (int i = 0; i < 4; i++) { + tr.set(baseKey.add("record" + i).pack(), + Tuple.from("constant_path_data_" + i).pack()); + } + context.commit(); + } + + // Export data from path with constant values + try (FDBRecordContext context = database.openContext()) { + KeySpacePath appPath = root.path("app"); + final List allData = exportAllData(appPath, context); + + // Should have 4 records + assertEquals(4, allData.size()); + + // Verify all data has expected prefix + for (KeyValue kv : allData) { + String value = Tuple.fromBytes(kv.getValue()).getString(0); + assertTrue(value.startsWith("constant_path_data_")); + } + } + } + + @Test + void exportAllDataEmpty() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("empty", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Don't store any data + try (FDBRecordContext context = database.openContext()) { + KeySpacePath emptyPath = root.path("empty"); + final List allData = exportAllData(emptyPath, context); + + // Should be empty + assertEquals(0, allData.size()); + } + } + + @Test + void exportAllDataWithDeepNestedStructure() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("org", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("dept", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("team", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL)))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Create deep nested structure + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + String[] departments = {"engineering", "sales"}; + for (String dept : departments) { + for (long team = 1; team <= 2; team++) { + for (int member = 0; member < 2; member++) { + UUID memberId = new UUID(dept.hashCode(), team * 100 + member); + KeySpacePath memberPath = root.path("org") + .add("dept", dept) + .add("team", team) + .add("member", memberId) + .add("data"); + + Tuple key = memberPath.toTuple(context); + tr.set(key.add("profile").pack(), + Tuple.from(dept + "_team" + team + "_member" + member).pack()); + tr.set(key.add("settings").pack(), + Tuple.from("settings_" + member).pack()); + } + } + } + context.commit(); + } + + // Export all data from organization root + try (FDBRecordContext context = database.openContext()) { + KeySpacePath orgPath = root.path("org"); + final List allData = exportAllData(orgPath, context); + + // Should have 16 records (2 departments * 2 teams * 2 members * 2 records each) + assertEquals(16, allData.size()); + } + + // Export data from specific department + try (FDBRecordContext context = database.openContext()) { + KeySpacePath engPath = root.path("org").add("dept", "engineering"); + final List allData = exportAllData(engPath, context); + + // Should have 8 records (1 dept * 2 teams * 2 members * 2 records each) + assertEquals(8, allData.size()); + + // Verify all belong to engineering + for (KeyValue kv : allData) { + String value = Tuple.fromBytes(kv.getValue()).getString(0); + if (value.startsWith("engineering_")) { + assertTrue(value.contains("engineering_")); + } + } + } + } + + @Test + void exportAllDataWithBinaryData() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("binary", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("blob", KeyType.BYTES))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store binary data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath basePath = root.path("binary"); + + // Store different types of binary data + byte[][] binaryKeys = { + {0x00, 0x01, 0x02}, + {(byte) 0xFF, (byte) 0xFE, (byte) 0xFD}, + {0x7F, 0x00, (byte) 0x80} + }; + + for (int i = 0; i < binaryKeys.length; i++) { + Tuple key = basePath.add("blob", binaryKeys[i]).toTuple(context); + byte[] value = ("binary_data_" + i).getBytes(); + tr.set(key.pack(), value); + } + context.commit(); + } + + // Export binary data + try (FDBRecordContext context = database.openContext()) { + KeySpacePath binaryPath = root.path("binary"); + final List allData = exportAllData(binaryPath, context); + + assertEquals(3, allData.size()); + + // Verify binary data integrity + for (KeyValue kv : allData) { + String valueStr = new String(kv.getValue()); + assertTrue(valueStr.startsWith("binary_data_")); + } + } + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 30}) + void exportAllDataWithContinuation(int limit) { + KeySpace root = new KeySpace( + new KeySpaceDirectory("continuation", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("item", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + final List> expectedBatches = new ArrayList<>(); + expectedBatches.add(new ArrayList<>()); + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath basePath = root.path("continuation"); + + IntStream.range(0, 20).forEach(i -> { + Tuple key = basePath.add("item", (long)i).toTuple(context); + final Tuple value = Tuple.from("continuation_item_" + i); + tr.set(key.pack(), value.pack()); + if (expectedBatches.get(expectedBatches.size() - 1).size() == limit) { + expectedBatches.add(new ArrayList<>()); + } + expectedBatches.get(expectedBatches.size() - 1).add(value); + }); + context.commit(); + } + if (20 % limit == 0) { + expectedBatches.add(List.of()); + } + + // Export with continuation support + try (FDBRecordContext context = database.openContext()) { + KeySpacePath continuationPath = root.path("continuation"); + + final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(limit)); + List> actual = new ArrayList<>(); + RecordCursorContinuation continuation = RecordCursorStartContinuation.START; + while (!continuation.isEnd()) { + final RecordCursor cursor = continuationPath.exportAllData(context, continuation.toBytes(), + scanProperties); + final AtomicReference> tupleResult = new AtomicReference<>(); + final List batch = cursor.map(dataInPath -> { + KeyValue kv = dataInPath.getRawKeyValue(); + return Tuple.fromBytes(kv.getValue()); + }).asList(tupleResult).join(); + actual.add(batch); + continuation = tupleResult.get().getContinuation(); + } + assertEquals(expectedBatches, actual); + } + } + + @Test + void exportAllDataThroughKeySpacePathWrapper() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Export from root level (should get all data) + EnvironmentKeySpace.EnvironmentRoot root = keySpace.root(); + List allData = exportAllData(root, context); + assertEquals(6, allData.size(), "Root level should export all data"); + + // Export from specific user level (should get data for user 100 only) + EnvironmentKeySpace.UserPath user100Path = keySpace.root().userid(100L); + verifyExtractedData(exportAllData(user100Path, context), + 5, "User 100 should have 4 records", + "user100", "All user 100 data should contain 'user100'"); + + // Export from specific application level (app1 for user 100) + EnvironmentKeySpace.ApplicationPath app1User100 = user100Path.application("app1"); + verifyExtractedData(exportAllData(app1User100, context), + 4, "App1 for user 100 should have 4 records (3 data + 1 metadata)", + "user100_app1", "All app1 user100 data should contain 'user100_app1'"); + + // Export from specific data store level + EnvironmentKeySpace.DataPath dataStore = app1User100.dataStore(); + List dataStoreData = exportAllData(dataStore, context); + verifyExtractedData(dataStoreData, + 3, "Data store should have exactly 3 records", + "user100_app1_data", "Data should be from user100 app1 data store"); + + // Export from metadata store level + EnvironmentKeySpace.MetadataPath metadataStore = app1User100.metadataStore(); + verifyExtractedData(exportAllData(metadataStore, context), + 1, "Metadata store should have exactly 1 record", + "user100_app1_meta1", "Metadata value should match"); + + // Verify empty export for user with no data + EnvironmentKeySpace.UserPath user300Path = keySpace.root().userid(300L); + assertEquals(0, exportAllData(user300Path, context).size(), "User 300 should have no data"); + } + } + + @Test + void exportAllDataThroughKeySpacePathWrapperResolvedPaths() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Test 4: Export from specific data store level + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + final List dataStoreData = dataStore.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .mapPipelined(DataInKeySpacePath::getResolvedPath, 1).asList().join(); + // Verify data store records have correct remainder + final ArrayList remainders = new ArrayList<>(); + for (ResolvedKeySpacePath kv : dataStoreData) { + // Path tuple should be the same + Tuple dataStoreTuple = dataStore.toTuple(context); + assertEquals(dataStoreTuple, kv.toTuple()); + remainders.add(kv.getRemainder()); + } + assertEquals(List.of( + Tuple.from("record1"), + Tuple.from("record2", 0), + Tuple.from("record2", 1) + ), remainders, "remainders should be the same"); + + } + } + + private void setData(List keys, FDBRecordContext context, KeySpacePath basePath, + String subdirectory, String valuePrefix) { + Transaction tr = context.ensureActive(); + for (int i = 0; i < keys.size(); i++) { + Tuple tuple = basePath.add(subdirectory, keys.get(i)).toTuple(context); + tr.set(tuple.pack(), Tuple.from(valuePrefix + i).pack()); + } + } + + /** + * Export all the data, and make some assertions that can always be done. + * This combines a lot of assertions, but most of the underlying behavior should be well covered by the objects + * that {@link KeySpacePath#exportAllData} is built on. + * @param pathToExport the path being exported + * @param context the context in which to export + * @return a list of the raw {@code KeyValue}s being exported + */ + private static List exportAllData(final KeySpacePath pathToExport, final FDBRecordContext context) { + final List asSingleExport = pathToExport.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .map(DataInKeySpacePath::getRawKeyValue).asList().join(); + + // assert that the resolved paths contain the right prefix + final List resolvedPaths = pathToExport.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .mapPipelined(DataInKeySpacePath::getResolvedPath, 1).asList().join(); + final ResolvedKeySpacePath rootResolvedPath = pathToExport.toResolvedPath(context); + for (ResolvedKeySpacePath resolvedPath : resolvedPaths) { + assertStartsWith(rootResolvedPath, resolvedPath); + } + + // assert that the reverse scan is the same as the forward scan, but in reverse + final List reversed = pathToExport.exportAllData(context, null, ScanProperties.REVERSE_SCAN) + .map(DataInKeySpacePath::getRawKeyValue).asList().join(); + Collections.reverse(reversed); + assertEquals(asSingleExport, reversed); + + // Assert continuations work correctly + final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(1)); + List asContinuations = new ArrayList<>(); + RecordCursorContinuation continuation = RecordCursorStartContinuation.START; + while (!continuation.isEnd()) { + final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), + scanProperties); + final AtomicReference> keyValueResult = new AtomicReference<>(); + final List batch = cursor.map(DataInKeySpacePath::getRawKeyValue).asList(keyValueResult).join(); + asContinuations.addAll(batch); + continuation = keyValueResult.get().getContinuation(); + if (keyValueResult.get().hasNext()) { + assertEquals(1, batch.size()); + } else { + assertThat(batch.size()).isLessThanOrEqualTo(1); + } + } + + assertEquals(asSingleExport, asContinuations); + return asSingleExport; + } + + private static void assertStartsWith(final ResolvedKeySpacePath rootResolvedPath, ResolvedKeySpacePath resolvedPath) { + do { + if (resolvedPath.equals(rootResolvedPath)) { + return; + } + resolvedPath = resolvedPath.getParent(); + } while (resolvedPath != null); + Assertions.fail("Expected <" + resolvedPath + "> to start with <" + rootResolvedPath + "> but it didn't"); + } + + private static void verifyExtractedData(final List app1User100Data, + int expectedCount, String expectedCountMessage, + String expectedValueContents, String contentMessage) { + assertEquals(expectedCount, app1User100Data.size(), expectedCountMessage); + + for (KeyValue kv : app1User100Data) { + String value = Tuple.fromBytes(kv.getValue()).getString(0); + assertTrue(value.contains(expectedValueContents), contentMessage); + } + } + +}