Merge branch 'main' into features/authz

bindings/c/CMakeLists.txt

@@ -194,11 +194,7 @@ if(NOT WIN32)
   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
 
   target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include)
-  if(USE_SANITIZER)
-    target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
-  else()
-    target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
-  endif()
+  target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
   target_link_libraries(fdb_c_api_tester_impl PRIVATE SimpleOpt)
 
   target_include_directories(fdb_c_unit_tests_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/)
@@ -211,11 +207,7 @@ if(NOT WIN32)
 
   # do not set RPATH for mako
   set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
-  if (USE_SANITIZER)
-    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_asan rapidjson)
-  else()
-    target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
-  endif()
+  target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson)
 
   if(NOT OPEN_FOR_IDE)
     # Make sure that fdb_c.h is compatible with c90
@@ -439,7 +431,7 @@ if (OPEN_FOR_IDE)
   add_library(fdb_c_shim OBJECT fdb_c_shim.cpp)
   target_link_libraries(fdb_c_shim PUBLIC dl)
 
-elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE) # Linux Only
+elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE AND NOT USE_UBSAN) # Linux, non-ubsan only
 
   set(SHIM_LIB_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
@@ -467,10 +459,12 @@ elseif(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE) # Linux Only
   add_test(NAME fdb_c_shim_library_tests
     COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
           --build-dir ${CMAKE_BINARY_DIR}
-          --source-dir ${CMAKE_SOURCE_DIR}
+          --unit-tests-bin $<TARGET_FILE:fdb_c_shim_unit_tests>
+          --api-tester-bin $<TARGET_FILE:fdb_c_shim_api_tester>
+          --api-test-dir ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
           )
 
-endif() # End Linux only
+endif() # End Linux, non-ubsan only
 
 # TODO: re-enable once the old vcxproj-based build system is removed.
 #generate_export_header(fdb_c EXPORT_MACRO_NAME "DLLEXPORT"

bindings/c/test/fdb_c_shim_tests.py

@@ -87,14 +87,12 @@ def __init__(
         self.build_dir = Path(args.build_dir).resolve()
         assert self.build_dir.exists(), "{} does not exist".format(args.build_dir)
         assert self.build_dir.is_dir(), "{} is not a directory".format(args.build_dir)
-        self.source_dir = Path(args.source_dir).resolve()
-        assert self.source_dir.exists(), "{} does not exist".format(args.source_dir)
-        assert self.source_dir.is_dir(), "{} is not a directory".format(args.source_dir)
-        self.api_tester_bin = self.build_dir.joinpath("bin", "fdb_c_shim_api_tester")
-        assert self.api_tester_bin.exists(), "{} does not exist".format(self.api_tester_bin)
-        self.unit_tests_bin = self.build_dir.joinpath("bin", "fdb_c_shim_unit_tests")
+        self.unit_tests_bin = Path(args.unit_tests_bin).resolve()
         assert self.unit_tests_bin.exists(), "{} does not exist".format(self.unit_tests_bin)
-        self.api_test_dir = self.source_dir.joinpath("bindings", "c", "test", "apitester", "tests")
+        self.api_tester_bin = Path(args.api_tester_bin).resolve()
+        assert self.api_tester_bin.exists(), "{} does not exist".format(self.api_tests_bin)
+        self.api_test_dir = Path(args.api_test_dir).resolve()
+        assert self.api_test_dir.exists(), "{} does not exist".format(self.api_test_dir)
         self.downloader = FdbBinaryDownloader(args.build_dir)
         # binary downloads are currently available only for x86_64
         self.platform = platform.machine()
@@ -196,13 +194,12 @@ def run_tests(self):
         help="FDB build directory",
         required=True,
     )
-    parser.add_argument(
-        "--source-dir",
-        "-s",
-        metavar="SOURCE_DIRECTORY",
-        help="FDB source directory",
-        required=True,
-    )
+    parser.add_argument('--unit-tests-bin', type=str,
+                        help='Path to the fdb_c_shim_unit_tests executable.')
+    parser.add_argument('--api-tester-bin', type=str,
+                        help='Path to the fdb_c_shim_api_tester executable.')
+    parser.add_argument('--api-test-dir', type=str,
+                        help='Path to a directory with api test definitions.')
     args = parser.parse_args()
     test = FdbCShimTests(args)
     test.run_tests()

bindings/c/test/unit/unit_tests.cpp

@@ -941,13 +941,13 @@ static Value dataOfRecord(const int i) {
 	return Value(format("data-of-record-%08d", i));
 }
 static std::string indexEntryKey(const int i) {
-	return Tuple().append(StringRef(prefix)).append(INDEX).append(indexKey(i)).append(primaryKey(i)).pack().toString();
+	return Tuple::makeTuple(prefix, INDEX, indexKey(i), primaryKey(i)).pack().toString();
 }
 static std::string recordKey(const int i, const int split) {
-	return Tuple().append(prefix).append(RECORD).append(primaryKey(i)).append(split).pack().toString();
+	return Tuple::makeTuple(prefix, RECORD, primaryKey(i), split).pack().toString();
 }
 static std::string recordValue(const int i, const int split) {
-	return Tuple().append(dataOfRecord(i)).append(split).pack().toString();
+	return Tuple::makeTuple(dataOfRecord(i), split).pack().toString();
 }
 
 const static int SPLIT_SIZE = 3;
@@ -993,13 +993,8 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
                                            fdb::Transaction& tr,
                                            int matchIndex,
                                            bool allMissing) {
-	std::string mapper = Tuple()
-	                         .append(prefix)
-	                         .append(RECORD)
-	                         .append(allMissing ? "{K[2]}"_sr : "{K[3]}"_sr)
-	                         .append("{...}"_sr)
-	                         .pack()
-	                         .toString();
+	std::string mapper =
+	    Tuple::makeTuple(prefix, RECORD, (allMissing ? "{K[2]}"_sr : "{K[3]}"_sr), "{...}"_sr).pack().toString();
 	return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
 }
 
@@ -1037,7 +1032,7 @@ TEST_CASE("tuple_support_versionstamp") {
 	// a random 12 bytes long StringRef as a versionstamp
 	StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12"_sr;
 	Versionstamp vs(str);
-	const Tuple t = Tuple().append(prefix).append(RECORD).appendVersionstamp(vs).append("{K[3]}"_sr).append("{...}"_sr);
+	const Tuple t = Tuple::makeTuple(prefix, RECORD, vs, "{K[3]}"_sr, "{...}"_sr);
 	ASSERT(t.getVersionstamp(2) == vs);
 
 	// verify the round-way pack-unpack path for a Tuple containing a versionstamp
@@ -1181,7 +1176,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_missing_all_secondary") {
 }
 
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
+	std::string mapper = Tuple::makeTuple(prefix, RECORD, "{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
 	auto result = get_mapped_range(
 	    tr,
@@ -1200,7 +1195,7 @@ TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_serializable") {
 }
 
 TEST_CASE("fdb_transaction_get_mapped_range_restricted_to_ryw_enable") {
-	std::string mapper = Tuple().append(prefix).append(RECORD).append("{K[3]}"_sr).pack().toString();
+	std::string mapper = Tuple::makeTuple(prefix, RECORD, "{K[3]}"_sr).pack().toString();
 	fdb::Transaction tr(db);
 	fdb_check(tr.set_option(FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE, nullptr, 0)); // Not disable RYW
 	auto result = get_mapped_range(

bindings/java/fdbJNI.cpp

@@ -765,6 +765,71 @@ JNIEXPORT jdouble JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getM
 	return (jdouble)fdb_database_get_main_thread_busyness(database);
 }
 
+JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1purgeBlobGranules(JNIEnv* jenv,
+                                                                                            jobject,
+                                                                                            jlong dbPtr,
+                                                                                            jbyteArray beginKeyBytes,
+                                                                                            jbyteArray endKeyBytes,
+                                                                                            jlong purgeVersion,
+                                                                                            jboolean force) {
+	if (!dbPtr || !beginKeyBytes || !endKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+
+	uint8_t* beginKeyArr = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL);
+	if (!beginKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	uint8_t* endKeyArr = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL);
+	if (!endKeyArr) {
+		jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+
+	FDBFuture* f = fdb_database_purge_blob_granules(database,
+	                                                beginKeyArr,
+	                                                jenv->GetArrayLength(beginKeyBytes),
+	                                                endKeyArr,
+	                                                jenv->GetArrayLength(endKeyBytes),
+	                                                purgeVersion,
+	                                                (fdb_bool_t)force);
+	jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)beginKeyArr, JNI_ABORT);
+	jenv->ReleaseByteArrayElements(endKeyBytes, (jbyte*)endKeyArr, JNI_ABORT);
+	return (jlong)f;
+}
+
+JNIEXPORT jlong JNICALL
+Java_com_apple_foundationdb_FDBDatabase_Database_1waitPurgeGranulesComplete(JNIEnv* jenv,
+                                                                            jobject,
+                                                                            jlong dbPtr,
+                                                                            jbyteArray purgeKeyBytes) {
+	if (!dbPtr || !purgeKeyBytes) {
+		throwParamNotNull(jenv);
+		return 0;
+	}
+	FDBDatabase* database = (FDBDatabase*)dbPtr;
+	uint8_t* purgeKeyArr = (uint8_t*)jenv->GetByteArrayElements(purgeKeyBytes, JNI_NULL);
+
+	if (!purgeKeyArr) {
+		if (!jenv->ExceptionOccurred())
+			throwRuntimeEx(jenv, "Error getting handle to native resources");
+		return 0;
+	}
+	FDBFuture* f =
+	    fdb_database_wait_purge_granules_complete(database, purgeKeyArr, jenv->GetArrayLength(purgeKeyBytes));
+	jenv->ReleaseByteArrayElements(purgeKeyBytes, (jbyte*)purgeKeyArr, JNI_ABORT);
+
+	return (jlong)f;
+}
+
 JNIEXPORT jboolean JNICALL Java_com_apple_foundationdb_FDB_Error_1predicate(JNIEnv* jenv,
                                                                             jobject,
                                                                             jint predicate,

bindings/java/src/main/com/apple/foundationdb/Database.java

@@ -161,6 +161,24 @@ default Transaction createTransaction() {
 	 */
 	double getMainThreadBusyness();
 
+	/**
+	 * Queues a purge of blob granules for the specified key range, at the specified version.
+     *
+	 * @param beginKey start of the key range
+	 * @param endKey end of the key range
+	 * @param purgeVersion version to purge at
+	 * @param force if true delete all data, if not keep data >= purgeVersion
+	 * @return the key to watch for purge complete
+	 */
+	CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, long purgeVersion, boolean force, Executor e);
+
+	/**
+	 * Wait for a previous call to purgeBlobGranules to complete
+	 *
+	 * @param purgeKey key to watch
+	 */
+	CompletableFuture<Void> waitPurgeGranulesComplete(byte[] purgeKey, Executor e);
+
 	/**
 	 * Runs a read-only transactional function against this {@code Database} with retry logic.
 	 *  {@link Function#apply(Object) apply(ReadTransaction)} will be called on the

bindings/java/src/main/com/apple/foundationdb/FDBDatabase.java

@@ -200,6 +200,26 @@ public double getMainThreadBusyness() {
 		}
 	}
 
+	@Override
+	public CompletableFuture<byte[]> purgeBlobGranules(byte[] beginKey, byte[] endKey, long purgeVersion, boolean force, Executor executor) {
+		pointerReadLock.lock();
+		try {
+			return new FutureKey(Database_purgeBlobGranules(getPtr(), beginKey, endKey, purgeVersion, force), executor, eventKeeper);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
+	@Override
+	public CompletableFuture<Void> waitPurgeGranulesComplete(byte[] purgeKey, Executor executor) {
+		pointerReadLock.lock();
+		try {
+			return new FutureVoid(Database_waitPurgeGranulesComplete(getPtr(), purgeKey), executor);
+		} finally {
+			pointerReadLock.unlock();
+		}
+	}
+
 	@Override
 	public Executor getExecutor() {
 		return executor;
@@ -215,4 +235,6 @@ protected void closeInternal(long cPtr) {
 	private native void Database_dispose(long cPtr);
 	private native void Database_setOption(long cPtr, int code, byte[] value) throws FDBException;
 	private native double Database_getMainThreadBusyness(long cPtr);
+	private native long Database_purgeBlobGranules(long cPtr, byte[] beginKey, byte[] endKey, long purgeVersion, boolean force);
+	private native long Database_waitPurgeGranulesComplete(long cPtr, byte[] purgeKey);
 }

bindings/python/tests/fdbcli_tests.py

@@ -561,6 +561,7 @@ def profile(logger):
     assert output2 == default_profile_client_get_output
     # set rate and size limit
     run_fdbcli_command('profile', 'client', 'set', '0.5', '1GB')
+    time.sleep(1) # global config can take some time to sync
     output3 = run_fdbcli_command('profile', 'client', 'get')
     logger.debug(output3)
     output3_list = output3.split(' ')
@@ -569,6 +570,7 @@ def profile(logger):
     assert output3_list[-1] == '1000000000.'
     # change back to default value and check
     run_fdbcli_command('profile', 'client', 'set', 'default', 'default')
+    time.sleep(1) # global config can take some time to sync
     assert run_fdbcli_command('profile', 'client', 'get') == default_profile_client_get_output
 
 
@@ -616,18 +618,23 @@ def tenants(logger):
 
     output = run_fdbcli_command('gettenant tenant')
     lines = output.split('\n')
-    assert len(lines) == 2
+    assert len(lines) == 3
     assert lines[0].strip().startswith('id: ')
     assert lines[1].strip().startswith('prefix: ')
+    assert lines[2].strip() == 'tenant state: ready'
 
     output = run_fdbcli_command('gettenant tenant JSON')
     json_output = json.loads(output, strict=False)
     assert(len(json_output) == 2)
     assert('tenant' in json_output)
     assert(json_output['type'] == 'success')
-    assert(len(json_output['tenant']) == 2)
+    assert(len(json_output['tenant']) == 3)
     assert('id' in json_output['tenant'])
     assert('prefix' in json_output['tenant'])
+    assert(len(json_output['tenant']['prefix']) == 2)
+    assert('base64' in json_output['tenant']['prefix'])
+    assert('printable' in json_output['tenant']['prefix'])
+    assert(json_output['tenant']['tenant_state'] == 'ready')
 
     output = run_fdbcli_command('usetenant')
     assert output == 'Using the default tenant'
@@ -699,6 +706,15 @@ def tenants(logger):
 
     run_fdbcli_command('writemode on; clear tenant_test')
 
+def integer_options():
+    process = subprocess.Popen(command_template[:-1], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=fdbcli_env)
+    cmd_sequence = ['option on TIMEOUT 1000', 'writemode on', 'clear foo']
+    output, error_output = process.communicate(input='\n'.join(cmd_sequence).encode())
+
+    lines = output.decode().strip().split('\n')[-2:]
+    assert lines[0] == 'Option enabled for all transactions'
+    assert lines[1].startswith('Committed')
+    assert error_output == b''
 
 if __name__ == '__main__':
     parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
@@ -745,6 +761,7 @@ def tenants(logger):
         triggerddteaminfolog()
         tenants()
         versionepoch()
+        integer_options()
     else:
         assert args.process_number > 1, "Process number should be positive"
         coordinators()

bindings/python/tests/tenant_tests.py

@@ -21,6 +21,7 @@
 import fdb
 import sys
 import json
+import base64
 from fdb.tuple import pack
 
 if __name__ == '__main__':
@@ -65,11 +66,11 @@ def test_tenant_operations(db):
 
     t1_entry = tenant_list[0].value
     t1_json = json.loads(t1_entry)
-    p1 = t1_json['prefix'].encode('utf8')
+    p1 = base64.b64decode(t1_json['prefix']['base64'])
 
     t2_entry = tenant_list[1].value
     t2_json = json.loads(t2_entry)
-    p2 = t2_json['prefix'].encode('utf8')
+    p2 = base64.b64decode(t2_json['prefix']['base64'])
 
     tenant1 = db.open_tenant(b'tenant1')
     tenant2 = db.open_tenant(b'tenant2')
@@ -80,12 +81,12 @@ def test_tenant_operations(db):
 
     tenant1_entry = db[b'\xff\xff/management/tenant/map/tenant1']
     tenant1_json = json.loads(tenant1_entry)
-    prefix1 = tenant1_json['prefix'].encode('utf8')
+    prefix1 = base64.b64decode(tenant1_json['prefix']['base64'])
     assert prefix1 == p1
 
     tenant2_entry = db[b'\xff\xff/management/tenant/map/tenant2']
     tenant2_json = json.loads(tenant2_entry)
-    prefix2 = tenant2_json['prefix'].encode('utf8')
+    prefix2 = base64.b64decode(tenant2_json['prefix']['base64'])
     assert prefix2 == p2
 
     assert tenant1[b'tenant_test_key'] == b'tenant1'