Merge release-5.2 into master

FDBLibTLS/FDBLibTLSSession.cpp

@@ -28,6 +28,7 @@
 
 #include <exception>
 
+#include <set>
 #include <string.h>
 #include <limits.h>
 
@@ -138,67 +139,70 @@ bool match_criteria(X509_NAME *name, int nid, const char *value, size_t len) {
 	return rc;
 }
 
-bool FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
+std::tuple<bool,std::string> FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs) {
 	X509_STORE_CTX *store_ctx = NULL;
 	X509_NAME *subject, *issuer;
 	BIO *bio = NULL;
 	bool rc = false;
+	// if returning false, give a reason string
+	std::string reason = "";
 
 	// If certificate verification is disabled, there's nothing more to do.
 	if (!verify->verify_cert)
-		return true;
+		return std::make_tuple(true, reason);
 
 	// Verify the certificate.
 	if ((store_ctx = X509_STORE_CTX_new()) == NULL) {
 		policy->logf("FDBLibTLSOutOfMemory", uid, true, NULL);
+		reason = "FDBLibTLSOutOfMemory";
 		goto err;
 	}
 	if (!X509_STORE_CTX_init(store_ctx, NULL, sk_X509_value(certs, 0), certs)) {
-		policy->logf("FDBLibTLSStoreCtxInit", uid, true, NULL);
+		reason = "FDBLibTLSStoreCtxInit";
 		goto err;
 	}
 	X509_STORE_CTX_trusted_stack(store_ctx, policy->roots);
-	X509_STORE_CTX_set_default(store_ctx, is_client ? "ssl_client" : "ssl_server");
+	X509_STORE_CTX_set_default(store_ctx, is_client ? "ssl_server" : "ssl_client");
 	if (!verify->verify_time)
 		X509_VERIFY_PARAM_set_flags(X509_STORE_CTX_get0_param(store_ctx), X509_V_FLAG_NO_CHECK_TIME);
 	if (X509_verify_cert(store_ctx) <= 0) {
 		const char *errstr = X509_verify_cert_error_string(X509_STORE_CTX_get_error(store_ctx));
-		policy->logf("FDBLibTLSVerifyCert", uid, true, "VerifyError", errstr, NULL);
+		reason = "FDBLibTLSVerifyCert VerifyError " + std::string(errstr);
 		goto err;
 	}
 
 	// Check subject criteria.
 	if ((subject = X509_get_subject_name(sk_X509_value(store_ctx->chain, 0))) == NULL) {
-		policy->logf("FDBLibTLSCertSubjectError", uid, true, NULL);
+		reason = "FDBLibTLSCertSubjectError";
 		goto err;
 	}
 	for (auto &pair: verify->subject_criteria) {
 		if (!match_criteria(subject, pair.first, pair.second.c_str(), pair.second.size())) {
-			policy->logf("FDBLibTLSCertSubjectMatchFailure", uid, true, NULL);
+			reason = "FDBLibTLSCertSubjectMatchFailure";
 			goto err;
 		}
 	}
 
 	// Check issuer criteria.
 	if ((issuer = X509_get_issuer_name(sk_X509_value(store_ctx->chain, 0))) == NULL) {
-		policy->logf("FDBLibTLSCertIssuerError", uid, true, NULL);
+		reason = "FDBLibTLSCertIssuerError";
 		goto err;
 	}
 	for (auto &pair: verify->issuer_criteria) {
 		if (!match_criteria(issuer, pair.first, pair.second.c_str(), pair.second.size())) {
-			policy->logf("FDBLibTLSCertIssuerMatchFailure", uid, true, NULL);
+			reason = "FDBLibTLSCertIssuerMatchFailure";
 			goto err;
 		}
 	}
 
 	// Check root criteria - this is the subject of the final certificate in the stack.
 	if ((subject = X509_get_subject_name(sk_X509_value(store_ctx->chain, sk_X509_num(store_ctx->chain) - 1))) == NULL) {
-		policy->logf("FDBLibTLSRootSubjectError", uid, true, NULL);
+		reason = "FDBLibTLSRootSubjectError";
 		goto err;
 	}
 	for (auto &pair: verify->root_criteria) {
 		if (!match_criteria(subject, pair.first, pair.second.c_str(), pair.second.size())) {
-			policy->logf("FDBLibTLSRootSubjectMatchFailure", uid, true, NULL);
+			reason = "FDBLibTLSRootSubjectMatchFailure";
 			goto err;
 		}
 	}
@@ -209,14 +213,17 @@ bool FDBLibTLSSession::check_verify(Reference<FDBLibTLSVerify> verify, struct st
  err:
 	X509_STORE_CTX_free(store_ctx);
 
-	return rc;
+	return std::make_tuple(rc, reason);
 }
 
 bool FDBLibTLSSession::verify_peer() {
 	struct stack_st_X509 *certs = NULL;
 	const uint8_t *cert_pem;
 	size_t cert_pem_len;
 	bool rc = false;
+	std::set<std::string> verify_failure_reasons;
+	bool verify_success;
+	std::string verify_failure_reason;
 
 	// If no verify peer rules have been set, we are relying on standard
 	// libtls verification.
@@ -232,9 +239,20 @@ bool FDBLibTLSSession::verify_peer() {
 
 	// Any matching rule is sufficient.
 	for (auto &verify_rule: policy->verify_rules) {
-		if (check_verify(verify_rule, certs)) {
+		std::tie(verify_success, verify_failure_reason) = check_verify(verify_rule, certs);
+		if (verify_success) {
 			rc = true;
 			break;
+		} else {
+			if (verify_failure_reason.length() > 0)
+				verify_failure_reasons.insert(verify_failure_reason);
+		}
+	}
+
+	if (!rc) {
+		// log the various failure reasons
+		for (std::string reason : verify_failure_reasons) {
+			policy->logf(reason.c_str(), uid, false, NULL);
 		}
 	}
 

FDBLibTLS/FDBLibTLSSession.h

@@ -39,7 +39,7 @@ struct FDBLibTLSSession : ITLSSession, ReferenceCounted<FDBLibTLSSession> {
 	virtual void delref() { ReferenceCounted<FDBLibTLSSession>::delref(); }
 
 	bool verify_peer();
-	bool check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs);
+	std::tuple<bool,std::string> check_verify(Reference<FDBLibTLSVerify> verify, struct stack_st_X509 *certs);
 
 	virtual int handshake();
 	virtual int read(uint8_t* data, int length);

bindings/flow/fdb_flow.actor.cpp

@@ -33,7 +33,7 @@ THREAD_FUNC networkThread(void* fdb) {
 }
 
 ACTOR Future<Void> _test() {
-	API *fdb = FDB::API::selectAPIVersion(510);
+	API *fdb = FDB::API::selectAPIVersion(520);
 	auto c = fdb->createCluster( std::string() );
 	auto db = c->createDatabase();
 	state Reference<Transaction> tr( new Transaction(db) );
@@ -77,7 +77,7 @@ ACTOR Future<Void> _test() {
 }
 
 void fdb_flow_test() {
-	API *fdb = FDB::API::selectAPIVersion(510);
+	API *fdb = FDB::API::selectAPIVersion(520);
 	fdb->setupNetwork();
 	startThread(networkThread, fdb);
 

bindings/flow/tester/Tester.actor.cpp

@@ -1739,7 +1739,7 @@ ACTOR void _test_versionstamp() {
 	try {
 		g_network = newNet2(NetworkAddress(), false);
 
-		API *fdb = FDB::API::selectAPIVersion(510);
+		API *fdb = FDB::API::selectAPIVersion(520);
 
 		fdb->setupNetwork();
 		startThread(networkThread, fdb);
@@ -1750,7 +1750,7 @@ ACTOR void _test_versionstamp() {
 
 		state Future<FDBStandalone<StringRef>> ftrVersion = tr->getVersionstamp();
 
-		tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
+		tr->atomicOp(LiteralStringRef("foo"), LiteralStringRef("blahblahbl\x00\x00\x00\x00"), FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE);
 
 		Void _ = wait(tr->commit()); // should use retry loop
 

bindings/go/src/fdb/generated.go

@@ -159,6 +159,27 @@ func (o NetworkOptions) SetBuggifySectionFiredProbability(param int64) error {
 	return o.setOpt(51, b)
 }
 
+// Set the ca bundle
+//
+// Parameter: ca bundle
+func (o NetworkOptions) SetTLSCaBytes(param []byte) error {
+	return o.setOpt(52, param)
+}
+
+// Set the file from which to load the certificate authority bundle
+//
+// Parameter: file path
+func (o NetworkOptions) SetTLSCaPath(param string) error {
+	return o.setOpt(53, []byte(param))
+}
+
+// Set the passphrase for encrypted private key. Password should be set before setting the key for the password to be used.
+//
+// Parameter: key passphrase
+func (o NetworkOptions) SetTLSPassword(param string) error {
+	return o.setOpt(54, []byte(param))
+}
+
 // Disables the multi-version client API and instead uses the local client directly. Must be set before setting up the network.
 func (o NetworkOptions) SetDisableMultiVersionClientApi() error {
 	return o.setOpt(60, nil)

bindings/java/local.mk

@@ -140,7 +140,7 @@ ifeq ($(PLATFORM),linux)
 	sed -e 's/-PRERELEASE/-SNAPSHOT/g' bindings/java/pom.xml > "$@"
 
   packages/fdb-java-$(JARVER).jar: fdb_java versions.target
-	@echo "Building      $@"
+	@echo "Building       $@"
 	@rm -f $@
 	@rm -rf packages/jar_regular
 	@mkdir -p packages/jar_regular

bindings/java/src/main/com/apple/foundationdb/tuple/Tuple.java

@@ -962,6 +962,14 @@ static void main(String[] args) {
 		System.out.println("t2.getNestedTuple(17): " + t2.getNestedTuple(17));
 		System.out.println("t2.getVersionstamp(20): " + t2.getVersionstamp(20));
 
+		int currOffset = 0;
+		for (Object item : t) {
+			int length = Tuple.from(item).pack().length;
+			Tuple t3 = Tuple.fromBytes(bytes, currOffset, length);
+			System.out.println("item = " + t3);
+			currOffset += length;
+		}
+
 		System.out.println("(2*(Long.MAX_VALUE+1),) = " + ByteArrayUtil.printable(Tuple.from(
 				BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).shiftLeft(1)
 		).pack()));

bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java

@@ -539,10 +539,10 @@ static int compareItems(Object item1, Object item2) {
 	}
 
 	static List<Object> unpack(byte[] bytes, int start, int length) {
-		List<Object> items = new LinkedList<Object>();
+		List<Object> items = new LinkedList<>();
 		int pos = start;
 		int end = start + length;
-		while(pos < bytes.length) {
+		while(pos < end) {
 			DecodeResult decoded = decode(bytes, pos, end);
 			items.add(decoded.o);
 			pos = decoded.end;

bindings/python/fdb/tuple.py

@@ -363,7 +363,7 @@ def _encode(value, nested=False):
     elif isinstance(value, tuple) or isinstance(value, list):
         child_bytes, version_pos = _reduce_children(map(lambda x: _encode(x, True), value))
         new_version_pos = -1 if version_pos < 0 else version_pos + 1
-        return b''.join([six.int2byte(NESTED_CODE)] + child_bytes + [six.int2byte(0x00)]), version_pos
+        return b''.join([six.int2byte(NESTED_CODE)] + child_bytes + [six.int2byte(0x00)]), new_version_pos
     else:
         raise ValueError("Unsupported data type: " + str(type(value)))