From 9b5ef3416ef0504c30e39295d449cb1dabf5fffc Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 4 Mar 2020 20:14:47 -0800 Subject: [PATCH 1/8] Refactor TLSParams into TLSConfig + LoadedTLSConfig The idea being that we keep around a TLSConfig that the configuration that the user has provided, and then when we want to intialize an SSL context, we ask the TLSConfig to load all certificates and return us a LoadedTLSConfig that is a concrete set of certificate bytes in memory. initTLS now just takes the in-memory bytes and applies them to the ssl context. This is a large refactor to lead up into certificate refeshing, where we will periodically check for changes to the certificates, and then re-load them and apply them to a new SSL context. --- bindings/flow/tester/Tester.actor.cpp | 5 +- fdbbackup/backup.actor.cpp | 13 +- fdbcli/fdbcli.actor.cpp | 13 +- fdbclient/NativeAPI.actor.cpp | 43 +-- fdbrpc/sim2.actor.cpp | 3 +- fdbserver/fdbserver.actor.cpp | 40 +-- flow/CMakeLists.txt | 10 +- flow/Net2.actor.cpp | 203 +++++++++----- flow/{TLSPolicy.cpp => TLSConfig.actor.cpp} | 186 +++++++++++- flow/TLSConfig.actor.h | 295 ++++++++++++++++++++ flow/TLSPolicy.h | 145 ---------- flow/flow.vcxproj | 6 +- flow/genericactors.actor.h | 30 ++ flow/network.h | 4 +- 14 files changed, 688 insertions(+), 308 deletions(-) rename flow/{TLSPolicy.cpp => TLSConfig.actor.cpp} (74%) create mode 100644 flow/TLSConfig.actor.h delete mode 100644 flow/TLSPolicy.h diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index 52d193320e9..10ca75d404b 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -28,6 +28,7 @@ #include "bindings/flow/FDBLoanerTypes.h" #include "fdbrpc/fdbrpc.h" #include "flow/DeterministicRandom.h" +#include "flow/TLSConfig.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. // Otherwise we have to type setupNetwork(), FDB::open(), etc. @@ -1748,7 +1749,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi populateOpsThatCreateDirectories(); // FIXME // This is "our" network - g_network = newNet2(false); + g_network = newNet2(TLSConfig()); ASSERT(!API::isAPIVersionSelected()); try { @@ -1791,7 +1792,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi ACTOR void _test_versionstamp() { try { - g_network = newNet2(false); + g_network = newNet2(TLSConfig()); API *fdb = FDB::API::selectAPIVersion(620); diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 2ea44f1a99b..022e12349c2 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -27,6 +27,7 @@ #include "flow/IRandom.h" #include "flow/genericactors.actor.h" #include "flow/SignalSafeUnwind.h" +#include "flow/TLSConfig.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/BackupAgent.actor.h" @@ -3071,22 +3072,22 @@ int main(int argc, char* argv[]) { blobCredentials.push_back(args->OptionArg()); break; #ifndef TLS_DISABLED - case TLSParams::OPT_TLS_PLUGIN: + case TLSConfig::OPT_TLS_PLUGIN: args->OptionArg(); break; - case TLSParams::OPT_TLS_CERTIFICATES: + case TLSConfig::OPT_TLS_CERTIFICATES: tlsCertPath = args->OptionArg(); break; - case TLSParams::OPT_TLS_PASSWORD: + case TLSConfig::OPT_TLS_PASSWORD: tlsPassword = args->OptionArg(); break; - case TLSParams::OPT_TLS_CA_FILE: + case TLSConfig::OPT_TLS_CA_FILE: tlsCAPath = args->OptionArg(); break; - case TLSParams::OPT_TLS_KEY: + case TLSConfig::OPT_TLS_KEY: tlsKeyPath = args->OptionArg(); break; - case TLSParams::OPT_TLS_VERIFY_PEERS: + case TLSConfig::OPT_TLS_VERIFY_PEERS: tlsVerifyPeers = args->OptionArg(); break; #endif diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index cf76fe7ee4a..ee16f9a6a31 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -35,6 +35,7 @@ #include "flow/SignalSafeUnwind.h" #include "fdbrpc/Platform.h" +#include "flow/TLSConfig.actor.h" #include "flow/SimpleOpt.h" #include "fdbcli/FlowLineNoise.h" @@ -2506,22 +2507,22 @@ struct CLIOptions { #ifndef TLS_DISABLED // TLS Options - case TLSParams::OPT_TLS_PLUGIN: + case TLSConfig::OPT_TLS_PLUGIN: args.OptionArg(); break; - case TLSParams::OPT_TLS_CERTIFICATES: + case TLSConfig::OPT_TLS_CERTIFICATES: tlsCertPath = args.OptionArg(); break; - case TLSParams::OPT_TLS_CA_FILE: + case TLSConfig::OPT_TLS_CA_FILE: tlsCAPath = args.OptionArg(); break; - case TLSParams::OPT_TLS_KEY: + case TLSConfig::OPT_TLS_KEY: tlsKeyPath = args.OptionArg(); break; - case TLSParams::OPT_TLS_PASSWORD: + case TLSConfig::OPT_TLS_PASSWORD: tlsPassword = args.OptionArg(); break; - case TLSParams::OPT_TLS_VERIFY_PEERS: + case TLSConfig::OPT_TLS_VERIFY_PEERS: tlsVerifyPeers = args.OptionArg(); break; #endif diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index e30390cf22b..4312cfee62e 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -43,7 +43,7 @@ #include "flow/Knobs.h" #include "flow/Platform.h" #include "flow/SystemMonitor.h" -#include "flow/TLSPolicy.h" +#include "flow/TLSConfig.actor.h" #include "flow/UnitTest.h" #if defined(CMAKE_BUILD) || !defined(WIN32) @@ -67,16 +67,7 @@ using std::min; using std::pair; NetworkOptions networkOptions; -TLSParams tlsParams; -static Reference tlsPolicy; - -static void initTLSPolicy() { -#ifndef TLS_DISABLED - if (!tlsPolicy) { - tlsPolicy = Reference(new TLSPolicy(TLSPolicy::Is::CLIENT)); - } -#endif -} +TLSConfig tlsConfig(TLSEndpointType::CLIENT); static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/"); static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/"); @@ -892,48 +883,40 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; case FDBNetworkOptions::TLS_CERT_PATH: validateOptionValue(value, true); - tlsParams.tlsCertBytes = ""; - tlsParams.tlsCertPath = value.get().toString(); + tlsConfig.setCertificatePath(value.get().toString()); break; case FDBNetworkOptions::TLS_CERT_BYTES: { validateOptionValue(value, true); - tlsParams.tlsCertPath = ""; - tlsParams.tlsCertBytes = value.get().toString(); + tlsConfig.setCertificateBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_PATH: { validateOptionValue(value, true); - tlsParams.tlsCABytes = ""; - tlsParams.tlsCAPath = value.get().toString(); + tlsConfig.setCAPath(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_BYTES: { validateOptionValue(value, true); - tlsParams.tlsCAPath = ""; - tlsParams.tlsCABytes = value.get().toString(); + tlsConfig.setCABytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_PASSWORD: validateOptionValue(value, true); - tlsParams.tlsPassword = value.get().toString(); + tlsConfig.setPassword(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_PATH: validateOptionValue(value, true); - tlsParams.tlsKeyBytes = ""; - tlsParams.tlsKeyPath = value.get().toString(); + tlsConfig.setKeyPath(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_BYTES: { validateOptionValue(value, true); - tlsParams.tlsKeyPath = ""; - tlsParams.tlsKeyBytes = value.get().toString(); + tlsConfig.setKeyBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_VERIFY_PEERS: validateOptionValue(value, true); - initTLSPolicy(); -#ifndef TLS_DISABLED - tlsPolicy->set_verify_peers({ value.get().toString() }); -#endif + tlsConfig.clearVerifyPeers(); + tlsConfig.addVerifyPeers( value.get().toString() ); break; case FDBNetworkOptions::CLIENT_BUGGIFY_ENABLE: enableBuggify(true, BuggifyType::Client); @@ -991,9 +974,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) { if (!networkOptions.logClientInfo.present()) networkOptions.logClientInfo = true; - initTLSPolicy(); - - g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present(), tlsPolicy, tlsParams); + g_network = newNet2(tlsConfig, false, useMetrics || networkOptions.traceDirectory.present()); FlowTransport::createInstance(true, transportId); Net2FileSystem::newFileSystem(); } diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 9d0e5168994..58f4b3fd5fa 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -30,6 +30,7 @@ #include "fdbrpc/TraceFileIO.h" #include "flow/FaultInjection.h" #include "flow/network.h" +#include "flow/TLSConfig.actor.h" #include "fdbrpc/Net2FileSystem.h" #include "fdbrpc/Replication.h" #include "fdbrpc/ReplicationUtils.h" @@ -1599,7 +1600,7 @@ class Sim2 : public ISimulator, public INetworkConnections { Sim2() : time(0.0), timerTime(0.0), taskCount(0), yielded(false), yield_limit(0), currentTaskID(TaskPriority::Zero) { // Not letting currentProcess be NULL eliminates some annoying special cases currentProcess = new ProcessInfo("NoMachine", LocalityData(Optional>(), StringRef(), StringRef(), StringRef()), ProcessClass(), {NetworkAddress()}, this, "", ""); - g_network = net2 = newNet2(false, true); + g_network = net2 = newNet2(TLSConfig(), false, true); Net2FileSystem::newFileSystem(); check_yield(TaskPriority::Zero); } diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 9f43cd2bded..56d06171ff1 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -57,7 +57,7 @@ #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbserver/CoroFlow.h" #include "flow/SignalSafeUnwind.h" -#include "flow/TLSPolicy.h" +#include "flow/TLSConfig.actor.h" #if defined(CMAKE_BUILD) || !defined(WIN32) #include "versions.h" #endif @@ -961,8 +961,7 @@ int main(int argc, char* argv[]) { int minTesterCount = 1; bool testOnServers = false; - Reference tlsPolicy = Reference(new TLSPolicy(TLSPolicy::Is::SERVER)); - TLSParams tlsParams; + TLSConfig tlsConfig(TLSEndpointType::SERVER); std::vector tlsVerifyPeers; double fileIoTimeout = 0.0; bool fileIoWarnOnly = false; @@ -1331,23 +1330,23 @@ int main(int argc, char* argv[]) { whitelistBinPaths = args.OptionArg(); break; #ifndef TLS_DISABLED - case TLSParams::OPT_TLS_PLUGIN: + case TLSConfig::OPT_TLS_PLUGIN: args.OptionArg(); break; - case TLSParams::OPT_TLS_CERTIFICATES: - tlsParams.tlsCertPath = args.OptionArg(); + case TLSConfig::OPT_TLS_CERTIFICATES: + tlsConfig.setCertificatePath(args.OptionArg()); break; - case TLSParams::OPT_TLS_PASSWORD: - tlsParams.tlsPassword = args.OptionArg(); + case TLSConfig::OPT_TLS_PASSWORD: + tlsConfig.setPassword(args.OptionArg()); break; - case TLSParams::OPT_TLS_CA_FILE: - tlsParams.tlsCAPath = args.OptionArg(); + case TLSConfig::OPT_TLS_CA_FILE: + tlsConfig.setCAPath(args.OptionArg()); break; - case TLSParams::OPT_TLS_KEY: - tlsParams.tlsKeyPath = args.OptionArg(); + case TLSConfig::OPT_TLS_KEY: + tlsConfig.setKeyPath(args.OptionArg()); break; - case TLSParams::OPT_TLS_VERIFY_PEERS: - tlsVerifyPeers.push_back(args.OptionArg()); + case TLSConfig::OPT_TLS_VERIFY_PEERS: + tlsConfig.addVerifyPeers(args.OptionArg()); break; #endif } @@ -1551,18 +1550,7 @@ int main(int argc, char* argv[]) { startNewSimulator(); openTraceFile(NetworkAddress(), rollsize, maxLogsSize, logFolder, "trace", logGroup); } else { -#ifndef TLS_DISABLED - if ( tlsVerifyPeers.size() ) { - try { - tlsPolicy->set_verify_peers( tlsVerifyPeers ); - } catch( Error &e ) { - fprintf(stderr, "ERROR: The format of the --tls_verify_peers option is incorrect.\n"); - printHelpTeaser(argv[0]); - flushAndExit(FDB_EXIT_ERROR); - } - } -#endif - g_network = newNet2(useThreadPool, true, tlsPolicy, tlsParams); + g_network = newNet2(tlsConfig, useThreadPool, true); FlowTransport::createInstance(false, 1); const bool expectsPublicAddress = (role == FDBD || role == NetworkTestServer || role == Restore); diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index f102ee3a714..adc6cc0406a 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -52,6 +52,8 @@ set(FLOW_SRCS SystemMonitor.h TDMetric.actor.h TDMetric.cpp + TLSConfig.actor.cpp + TLSConfig.actor.h ThreadHelper.actor.h ThreadHelper.cpp ThreadPrimitives.cpp @@ -59,24 +61,22 @@ set(FLOW_SRCS ThreadSafeQueue.h Trace.cpp Trace.h - TLSPolicy.h - TLSPolicy.cpp UnitTest.cpp UnitTest.h - XmlTraceLogFormatter.h XmlTraceLogFormatter.cpp + XmlTraceLogFormatter.h actorcompiler.h error_definitions.h - flat_buffers.h flat_buffers.cpp + flat_buffers.h flow.cpp flow.h genericactors.actor.cpp genericactors.actor.h network.cpp network.h - serialize.h serialize.cpp + serialize.h stacktrace.amalgamation.cpp stacktrace.h version.cpp) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index be6aecedd52..9209b5910cf 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -37,7 +37,7 @@ #include "flow/AsioReactor.h" #include "flow/Profiler.h" #include "flow/ProtocolVersion.h" -#include "flow/TLSPolicy.h" +#include "flow/TLSConfig.actor.h" #ifdef WIN32 #include @@ -111,7 +111,7 @@ thread_local INetwork* thread_network = 0; class Net2 sealed : public INetwork, public INetworkConnections { public: - Net2(bool useThreadPool, bool useMetrics, Reference tlsPolicy, const TLSParams& tlsParams); + Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics); void initTLS(); void run(); void initMetrics(); @@ -159,12 +159,12 @@ class Net2 sealed : public INetwork, public INetworkConnections { #ifndef TLS_DISABLED boost::asio::ssl::context sslContext; #endif - Reference tlsPolicy; - TLSParams tlsParams; + TLSConfig tlsConfig; + std::string tlsPassword; bool tlsInitialized; std::string get_password() const { - return tlsParams.tlsPassword; + return tlsPassword; } INetworkConnections *network; // initially this, but can be changed @@ -847,7 +847,7 @@ bool insecurely_always_accept(bool _1, boost::asio::ssl::verify_context& _2) { } #endif -Net2::Net2(bool useThreadPool, bool useMetrics, Reference tlsPolicy, const TLSParams& tlsParams) +Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) : useThreadPool(useThreadPool), network(this), reactor(this), @@ -858,8 +858,7 @@ Net2::Net2(bool useThreadPool, bool useMetrics, Reference tlsPolicy, lastMinTaskID(TaskPriority::Zero), numYields(0), tlsInitialized(false), - tlsPolicy(tlsPolicy), - tlsParams(tlsParams) + tlsConfig(tlsConfig) #ifndef TLS_DISABLED ,sslContext(boost::asio::ssl::context(boost::asio::ssl::context::tlsv12)) #endif @@ -888,92 +887,142 @@ Net2::Net2(bool useThreadPool, bool useMetrics, Reference tlsPolicy, } +/* +ACTOR static Future watchFileForChanges( std::string filename, AsyncVar> *contents_var ) { + state std::time_t lastModTime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); + loop { + wait(delay(FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS)); + std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); + if (lastModTime != modtime) { + lastModTime = modtime; + ErrorOr> contents = wait(readEntireFile(filename)); + if (contents.present()) { + contents_var->set(contents.get()); + } + } + } +} + +ACTOR static Future reloadConfigurationOnChange( TLSOptions::PolicyInfo *pci, Reference plugin, AsyncVar> *realVerifyPeersPolicy, AsyncVar> *realNoVerifyPeersPolicy ) { + if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) { + return Void(); + return Void(); + } + loop { + // Early in bootup, the filesystem might not be initialized yet. Wait until it is. + if (IAsyncFileSystem::filesystem() != nullptr) { + break; + } + wait(delay(1.0)); + } + state int mismatches = 0; + state AsyncVar> ca_var; + state AsyncVar> key_var; + state AsyncVar> cert_var; + state std::vector> lifetimes; + if (!pci->ca_path.empty()) lifetimes.push_back(watchFileForChanges(pci->ca_path, &ca_var)); + if (!pci->key_path.empty()) lifetimes.push_back(watchFileForChanges(pci->key_path, &key_var)); + if (!pci->cert_path.empty()) lifetimes.push_back(watchFileForChanges(pci->cert_path, &cert_var)); + loop { + state Future ca_changed = ca_var.onChange(); + state Future key_changed = key_var.onChange(); + state Future cert_changed = cert_var.onChange(); + wait( ca_changed || key_changed || cert_changed ); + if (ca_changed.isReady()) { + TraceEvent(SevInfo, "TLSRefreshCAChanged").detail("path", pci->ca_path).detail("length", ca_var.get().size()); + pci->ca_contents = ca_var.get(); + } + if (key_changed.isReady()) { + TraceEvent(SevInfo, "TLSRefreshKeyChanged").detail("path", pci->key_path).detail("length", key_var.get().size()); + pci->key_contents = key_var.get(); + } + if (cert_changed.isReady()) { + TraceEvent(SevInfo, "TLSRefreshCertChanged").detail("path", pci->cert_path).detail("length", cert_var.get().size()); + pci->cert_contents = cert_var.get(); + } + bool rc = true; + Reference verifypeers = Reference(plugin->create_policy()); + Reference noverifypeers = Reference(plugin->create_policy()); + loop { + // Don't actually loop. We're just using loop/break as a `goto err`. + // This loop always ends with an unconditional break. + rc = verifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size()); + if (!rc) break; + rc = verifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str()); + if (!rc) break; + rc = verifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size()); + if (!rc) break; + { + std::unique_ptr verify_peers_arr(new const uint8_t*[pci->verify_peers.size()]); + std::unique_ptr verify_peers_len(new int[pci->verify_peers.size()]); + for (int i = 0; i < pci->verify_peers.size(); i++) { + verify_peers_arr[i] = (const uint8_t *)&pci->verify_peers[i][0]; + verify_peers_len[i] = pci->verify_peers[i].size(); + } + rc = verifypeers->set_verify_peers(pci->verify_peers.size(), verify_peers_arr.get(), verify_peers_len.get()); + if (!rc) break; + } + rc = noverifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size()); + if (!rc) break; + rc = noverifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str()); + if (!rc) break; + rc = noverifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size()); + if (!rc) break; + break; + } + + if (rc) { + TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded"); + realVerifyPeersPolicy->set(verifypeers); + realNoVerifyPeersPolicy->set(noverifypeers); + mismatches = 0; + } else { + // Some files didn't match up, they should in the future, and we'll retry then. + mismatches++; + TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").detail("mismatches", mismatches); + } + } +} +*/ + void Net2::initTLS() { if(tlsInitialized) { return; } #ifndef TLS_DISABLED try { - const char *defaultCertFileName = "fdb.pem"; - - if( tlsPolicy && !tlsPolicy->rules.size() ) { - std::string verify_peers; - if (platform::getEnvironmentVar("FDB_TLS_VERIFY_PEERS", verify_peers)) { - tlsPolicy->set_verify_peers({ verify_peers }); - } else { - tlsPolicy->set_verify_peers({ std::string("Check.Valid=1")}); - } - } + LoadedTLSConfig loaded = tlsConfig.loadSync(); sslContext.set_options(boost::asio::ssl::context::default_workarounds); sslContext.set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); - if (tlsPolicy) { - Reference policy = tlsPolicy; - sslContext.set_verify_callback([policy](bool preverified, boost::asio::ssl::verify_context& ctx) { - return policy->verify_peer(preverified, ctx.native_handle()); - }); + + if (loaded.isTLSEnabled()) { + Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); + tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); + + sslContext.set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { + return policy->verify_peer(preverified, ctx.native_handle()); + }); } else { sslContext.set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); } - if ( !tlsParams.tlsPassword.size() ) { - platform::getEnvironmentVar( "FDB_TLS_PASSWORD", tlsParams.tlsPassword ); - } + tlsPassword = loaded.getPassword(); sslContext.set_password_callback(std::bind(&Net2::get_password, this)); - if ( tlsParams.tlsCertBytes.size() ) { - sslContext.use_certificate_chain(boost::asio::buffer(tlsParams.tlsCertBytes.data(), tlsParams.tlsCertBytes.size())); - } - else { - if ( !tlsParams.tlsCertPath.size() ) { - if ( !platform::getEnvironmentVar( "FDB_TLS_CERTIFICATE_FILE", tlsParams.tlsCertPath ) ) { - if( fileExists(defaultCertFileName) ) { - tlsParams.tlsCertPath = defaultCertFileName; - } else if( fileExists( joinPath(platform::getDefaultConfigPath(), defaultCertFileName) ) ) { - tlsParams.tlsCertPath = joinPath(platform::getDefaultConfigPath(), defaultCertFileName); - } - } - } - if ( tlsParams.tlsCertPath.size() ) { - sslContext.use_certificate_chain_file(tlsParams.tlsCertPath); - } + const std::string& certBytes = loaded.getCertificateBytes(); + if ( certBytes.size() ) { + sslContext.use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size())); } - if ( tlsParams.tlsCABytes.size() ) { - sslContext.add_certificate_authority(boost::asio::buffer(tlsParams.tlsCABytes.data(), tlsParams.tlsCABytes.size())); - } - else { - if ( !tlsParams.tlsCAPath.size() ) { - platform::getEnvironmentVar("FDB_TLS_CA_FILE", tlsParams.tlsCAPath); - } - if ( tlsParams.tlsCAPath.size() ) { - try { - std::string cert = readFileBytes(tlsParams.tlsCAPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE); - sslContext.add_certificate_authority(boost::asio::buffer(cert.data(), cert.size())); - } - catch (Error& e) { - fprintf(stderr, "Error reading CA file %s: %s\n", tlsParams.tlsCAPath.c_str(), e.what()); - TraceEvent("Net2TLSReadCAError").error(e); - throw tls_error(); - } - } + const std::string& CABytes = loaded.getCABytes(); + if ( CABytes.size() ) { + sslContext.add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size())); } - if (tlsParams.tlsKeyBytes.size()) { - sslContext.use_private_key(boost::asio::buffer(tlsParams.tlsKeyBytes.data(), tlsParams.tlsKeyBytes.size()), boost::asio::ssl::context::pem); - } else { - if (!tlsParams.tlsKeyPath.size()) { - if(!platform::getEnvironmentVar( "FDB_TLS_KEY_FILE", tlsParams.tlsKeyPath)) { - if( fileExists(defaultCertFileName) ) { - tlsParams.tlsKeyPath = defaultCertFileName; - } else if( fileExists( joinPath(platform::getDefaultConfigPath(), defaultCertFileName) ) ) { - tlsParams.tlsKeyPath = joinPath(platform::getDefaultConfigPath(), defaultCertFileName); - } - } - } - if (tlsParams.tlsKeyPath.size()) { - sslContext.use_private_key_file(tlsParams.tlsKeyPath, boost::asio::ssl::context::pem); - } + const std::string& keyBytes = loaded.getKeyBytes(); + if (keyBytes.size()) { + sslContext.use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()), boost::asio::ssl::context::pem); } } catch(boost::system::system_error e) { fprintf(stderr, "Error initializing TLS: %s\n", e.what()); @@ -1522,9 +1571,9 @@ void ASIOReactor::wake() { } // namespace net2 -INetwork* newNet2(bool useThreadPool, bool useMetrics, Reference policy, const TLSParams& tlsParams) { +INetwork* newNet2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) { try { - N2::g_net2 = new N2::Net2(useThreadPool, useMetrics, policy, tlsParams); + N2::g_net2 = new N2::Net2(tlsConfig, useThreadPool, useMetrics); } catch(boost::system::system_error e) { TraceEvent("Net2InitError").detail("Message", e.what()); diff --git a/flow/TLSPolicy.cpp b/flow/TLSConfig.actor.cpp similarity index 74% rename from flow/TLSPolicy.cpp rename to flow/TLSConfig.actor.cpp index cc83a246293..28685e88ad7 100644 --- a/flow/TLSPolicy.cpp +++ b/flow/TLSConfig.actor.cpp @@ -1,5 +1,5 @@ /* - * TLSPolicy.cpp + * TLSConfig.actor.cpp * * This source file is part of the FoundationDB open source project * @@ -18,8 +18,11 @@ * limitations under the License. */ -#include "flow/TLSPolicy.h" +#define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP public +#include "flow/TLSConfig.actor.h" +#undef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP +// To force typeinfo to only be emitted once. TLSPolicy::~TLSPolicy() {} #ifndef TLS_DISABLED @@ -39,18 +42,191 @@ TLSPolicy::~TLSPolicy() {} #include #include #include +#include + +// This include breaks module dependencies, but we need to do async file reads. +// So either we include fdbrpc here, or this file is moved to fdbrpc/, and then +// Net2, which depends on us, includes fdbrpc/. +// +// Either way, the only way to break this dependency cycle is to move all of +// AsyncFile to flow/ +#include "fdbrpc/IAsyncFile.h" +#include "flow/Platform.h" #include "flow/FastRef.h" #include "flow/Trace.h" +#include "flow/genericactors.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + + +std::vector LoadedTLSConfig::getVerifyPeers() const { + if (tlsVerifyPeers.size()) { + return tlsVerifyPeers; + } + + std::string envVerifyPeers; + if (platform::getEnvironmentVar("FDB_TLS_VERIFY_PEERS", envVerifyPeers)) { + return {envVerifyPeers}; + } + + return {"Check.Valid=1"}; +} + +std::string LoadedTLSConfig::getPassword() const { + if (tlsPassword.size()) { + return tlsPassword; + } + + std::string envPassword; + platform::getEnvironmentVar("FDB_TLS_PASSWORD", envPassword); + return envPassword; +} + +std::string TLSConfig::getCertificatePathSync() const { + if (tlsCertPath.size()) { + return tlsCertPath; + } + + std::string envCertPath; + if (platform::getEnvironmentVar("FDB_TLS_CERTIFICATE_FILE", envCertPath)) { + return envCertPath; + } + + const char *defaultCertFileName = "fdb.pem"; + if( fileExists(defaultCertFileName) ) { + return defaultCertFileName; + } + + if( fileExists( joinPath(platform::getDefaultConfigPath(), defaultCertFileName) ) ) { + return joinPath(platform::getDefaultConfigPath(), defaultCertFileName); + } + + return std::string(); +} + +std::string TLSConfig::getKeyPathSync() const { + if (tlsKeyPath.size()) { + return tlsKeyPath; + } + + std::string envKeyPath; + if (platform::getEnvironmentVar("FDB_TLS_KEY_FILE", envKeyPath)) { + return envKeyPath; + } + + const char *defaultCertFileName = "fdb.pem"; + if( fileExists(defaultCertFileName) ) { + return defaultCertFileName; + } + + if( fileExists( joinPath(platform::getDefaultConfigPath(), defaultCertFileName) ) ) { + return joinPath(platform::getDefaultConfigPath(), defaultCertFileName); + } + + return std::string(); +} + +std::string TLSConfig::getCAPathSync() const { + if (tlsCAPath.size()) { + return tlsCAPath; + } + + std::string envCAPath; + platform::getEnvironmentVar("FDB_TLS_CA_FILE", envCAPath); + return envCAPath; +} + +LoadedTLSConfig TLSConfig::loadSync() const { + LoadedTLSConfig loaded; + + const std::string certPath = getCertificatePathSync(); + if (certPath.size()) { + loaded.tlsCertBytes = readFileBytes( certPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE ); + } else { + loaded.tlsCertBytes = tlsCertBytes; + } + + const std::string keyPath = getKeyPathSync(); + if (keyPath.size()) { + loaded.tlsKeyBytes = readFileBytes( keyPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE ); + } else { + loaded.tlsKeyBytes = tlsKeyBytes; + } + + const std::string CAPath = getCAPathSync(); + if (CAPath.size()) { + loaded.tlsCABytes = readFileBytes( CAPath, FLOW_KNOBS->CERT_FILE_MAX_SIZE ); + } else { + loaded.tlsCABytes = tlsCABytes; + } + + loaded.tlsPassword = tlsPassword; + loaded.tlsVerifyPeers = tlsVerifyPeers; + loaded.endpointType = endpointType; + + return loaded; +} + +// And now do the same thing, but async... + +ACTOR static Future readEntireFile( std::string filename, std::string* destination ) { + state Reference file = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0)); + state int64_t filesize = wait(file->size()); + if (filesize > FLOW_KNOBS->CERT_FILE_MAX_SIZE) { + throw tls_error(); + } + destination->resize(filesize); + int rc = wait(file->read(const_cast(destination->c_str()), filesize, 0)); + if (rc != filesize) { + // File modified during read, probably. The mtime should change, and thus we'll be called again. + throw tls_error(); + } + return Void(); +} + +ACTOR Future TLSConfig::loadAsync(const TLSConfig* self) { + state LoadedTLSConfig loaded; + state std::vector> reads; + + const std::string& certPath = self->getCertificatePathSync(); + if (certPath.size()) { + reads.push_back( readEntireFile( certPath, &loaded.tlsCertBytes ) ); + } else { + loaded.tlsCertBytes = self->tlsCertBytes; + } + + const std::string& keyPath = self->getKeyPathSync(); + if (keyPath.size()) { + reads.push_back( readEntireFile( keyPath, &loaded.tlsKeyBytes ) ); + } else { + loaded.tlsKeyBytes = self->tlsKeyBytes; + } + + const std::string& CAPath = self->getCAPathSync(); + if (CAPath.size()) { + reads.push_back( readEntireFile( CAPath, &loaded.tlsCABytes ) ); + } else { + loaded.tlsCABytes = self->tlsKeyBytes; + } + + wait(waitForAll(reads)); + + loaded.tlsPassword = self->tlsPassword; + loaded.tlsVerifyPeers = self->tlsVerifyPeers; + loaded.endpointType = self->endpointType; + + return loaded; +} + +void ConfigureSSLContext( boost::asio::ssl::context *context, const LoadedTLSConfig& config ) { + +} std::string TLSPolicy::ErrorString(boost::system::error_code e) { char* str = ERR_error_string(e.value(), NULL); return std::string(str); } -// To force typeinfo to only be emitted once. - - std::string TLSPolicy::toString() const { std::stringstream ss; ss << "TLSPolicy{ Rules=["; diff --git a/flow/TLSConfig.actor.h b/flow/TLSConfig.actor.h new file mode 100644 index 00000000000..e750c3de938 --- /dev/null +++ b/flow/TLSConfig.actor.h @@ -0,0 +1,295 @@ +/* + * TLSConfig.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source version. +#if defined(NO_INTELLISENSE) && !defined(FLOW_TLS_CONFIG_ACTOR_G_H) + #define FLOW_TLS_CONFIG_ACTOR_G_H + #include "flow/TLSConfig.actor.g.h" +#elif !defined(FLOW_TLS_CONFIG_ACTOR_H) + #define FLOW_TLS_CONFIG_ACTOR_H + +#pragma once + +#include +#include +#include +#include +#include "flow/FastRef.h" +#include "flow/Knobs.h" +#include "flow/flow.h" + +#ifndef TLS_DISABLED + +#include +typedef int NID; + +enum class MatchType { + EXACT, + PREFIX, + SUFFIX, +}; + +enum class X509Location { + // This NID is located within a X509_NAME + NAME, + // This NID is an X509 extension, and should be parsed accordingly + EXTENSION, +}; + +struct Criteria { + Criteria( const std::string& s ) + : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} + Criteria( const std::string& s, MatchType mt ) + : criteria(s), match_type(mt), location(X509Location::NAME) {} + Criteria( const std::string& s, X509Location loc) + : criteria(s), match_type(MatchType::EXACT), location(loc) {} + Criteria( const std::string& s, MatchType mt, X509Location loc) + : criteria(s), match_type(mt), location(loc) {} + + std::string criteria; + MatchType match_type; + X509Location location; + + bool operator==(const Criteria& c) const { + return criteria == c.criteria && match_type == c.match_type && location == c.location; + } +}; +#endif + +#include "flow/actorcompiler.h" // This must be the last #include. + +enum class TLSEndpointType { + UNSET = 0, + CLIENT, + SERVER +}; + +class TLSConfig; +template class LoadAsyncActorState; +// TODO: Remove this once this code is merged with master/to-be 7.0 and actors can access private variables. +#ifndef PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP +#define PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP private +#endif + +class LoadedTLSConfig { +public: + std::string getCertificateBytes() const { + return tlsCertBytes; + } + + std::string getKeyBytes() const { + return tlsKeyBytes; + } + + std::string getCABytes() const { + return tlsCABytes; + } + + // Return the explicitly set verify peers string. + // If no verify peers string was set, return the environment setting + // If no environment setting exists, return "Check.Valid=1" + std::vector getVerifyPeers() const; + + // Return the explicitly set password. + // If no password was set, return the environment setting + // If no environment setting exists, return an empty string + std::string getPassword() const; + + TLSEndpointType getEndpointType() const { + return endpointType; + } + + bool isTLSEnabled() const { + return endpointType != TLSEndpointType::UNSET; + } + +PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP: + std::string tlsCertBytes, tlsKeyBytes, tlsCABytes; + std::string tlsPassword; + std::vector tlsVerifyPeers; + TLSEndpointType endpointType = TLSEndpointType::UNSET; + + friend class TLSConfig; + template + friend class LoadAsyncActorState; +}; + +class TLSConfig { +public: + enum { OPT_TLS = 100000, OPT_TLS_PLUGIN, OPT_TLS_CERTIFICATES, OPT_TLS_KEY, OPT_TLS_VERIFY_PEERS, OPT_TLS_CA_FILE, OPT_TLS_PASSWORD }; + + TLSConfig() = default; + explicit TLSConfig( TLSEndpointType endpointType ) + : endpointType( endpointType ) { + } + + void setCertificatePath( const std::string& path ) { + tlsCertPath = path; + tlsCertBytes = ""; + } + + void setCertificateBytes( const std::string& bytes ) { + tlsCertBytes = bytes; + tlsCertPath = ""; + } + + void setKeyPath( const std::string& path ) { + tlsKeyPath = path; + tlsKeyBytes = ""; + } + + void setKeyBytes( const std::string& bytes ) { + tlsKeyBytes = bytes; + tlsKeyPath = ""; + } + + void setCAPath( const std::string& path ) { + tlsCAPath = path; + tlsCABytes = ""; + } + + void setCABytes( const std::string& bytes ) { + tlsCABytes = bytes; + tlsCAPath = ""; + } + + void setPassword( const std::string& password ) { + tlsPassword = password; + } + + void clearVerifyPeers() { + tlsVerifyPeers.clear(); + } + + void addVerifyPeers( const std::string& verifyPeers ) { + tlsVerifyPeers.push_back( verifyPeers ); + } + + // Load all specified certificates into memory, and return an object that + // allows access to them. + // If self has any certificates by path, they will be *synchronously* loaded from disk. + LoadedTLSConfig loadSync() const; + + // Load all specified certificates into memory, and return an object that + // allows access to them. + // If self has any certificates by path, they will be *asynchronously* loaded from disk. + Future loadAsync() const { + return loadAsync(this); + } + +PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP: + // Return the explicitly set path. + // If one was not set, return the path from the environment. + // (Cert and Key only) If neither exist, check for fdb.pem in cwd + // (Cert and Key only) If fdb.pem doesn't exist, check for it in default config dir + // Otherwise return the empty string. + // Theoretically, fileExists() can block, so these functions are labelled as synchronous + // TODO: make an easy to use Future fileExists, and port lots of code over to it. + std::string getCertificatePathSync() const; + std::string getKeyPathSync() const; + std::string getCAPathSync() const; + + ACTOR static Future loadAsync(const TLSConfig* self); + template + friend class LoadAsyncActorState; + + std::string tlsCertPath, tlsKeyPath, tlsCAPath; + std::string tlsCertBytes, tlsKeyBytes, tlsCABytes; + std::string tlsPassword; + std::vector tlsVerifyPeers; + TLSEndpointType endpointType = TLSEndpointType::UNSET; +}; + +namespace boost { + namespace asio { + namespace ssl { + struct context; + } + } +} +void ConfigureSSLContext( boost::asio::ssl::context *context, const LoadedTLSConfig& config ); + +class TLSPolicy : ReferenceCounted { +public: + + TLSPolicy(TLSEndpointType client) : is_client(client == TLSEndpointType::CLIENT) {} + virtual ~TLSPolicy(); + + virtual void addref() { ReferenceCounted::addref(); } + virtual void delref() { ReferenceCounted::delref(); } + +#ifndef TLS_DISABLED + static std::string ErrorString(boost::system::error_code e); + + void set_verify_peers(std::vector verify_peers); + bool verify_peer(bool preverified, X509_STORE_CTX* store_ctx); + + std::string toString() const; + + struct Rule { + explicit Rule(std::string input); + + std::string toString() const; + + std::map< NID, Criteria > subject_criteria; + std::map< NID, Criteria > issuer_criteria; + std::map< NID, Criteria > root_criteria; + + bool verify_cert = true; + bool verify_time = true; + }; + + std::vector rules; +#endif + bool is_client; +}; + +#define TLS_PLUGIN_FLAG "--tls_plugin" +#define TLS_CERTIFICATE_FILE_FLAG "--tls_certificate_file" +#define TLS_KEY_FILE_FLAG "--tls_key_file" +#define TLS_VERIFY_PEERS_FLAG "--tls_verify_peers" +#define TLS_CA_FILE_FLAG "--tls_ca_file" +#define TLS_PASSWORD_FLAG "--tls_password" + +#define TLS_OPTION_FLAGS \ + { TLSConfig::OPT_TLS_PLUGIN, TLS_PLUGIN_FLAG, SO_REQ_SEP }, \ + { TLSConfig::OPT_TLS_CERTIFICATES, TLS_CERTIFICATE_FILE_FLAG, SO_REQ_SEP }, \ + { TLSConfig::OPT_TLS_KEY, TLS_KEY_FILE_FLAG, SO_REQ_SEP }, \ + { TLSConfig::OPT_TLS_VERIFY_PEERS, TLS_VERIFY_PEERS_FLAG, SO_REQ_SEP }, \ + { TLSConfig::OPT_TLS_PASSWORD, TLS_PASSWORD_FLAG, SO_REQ_SEP }, \ + { TLSConfig::OPT_TLS_CA_FILE, TLS_CA_FILE_FLAG, SO_REQ_SEP }, + +#define TLS_HELP \ + " " TLS_CERTIFICATE_FILE_FLAG " CERTFILE\n" \ + " The path of a file containing the TLS certificate and CA\n" \ + " chain.\n" \ + " " TLS_CA_FILE_FLAG " CERTAUTHFILE\n" \ + " The path of a file containing the CA certificates chain.\n" \ + " " TLS_KEY_FILE_FLAG " KEYFILE\n" \ + " The path of a file containing the private key corresponding\n" \ + " to the TLS certificate.\n" \ + " " TLS_PASSWORD_FLAG " PASSCODE\n" \ + " The passphrase of encrypted private key\n" \ + " " TLS_VERIFY_PEERS_FLAG " CONSTRAINTS\n" \ + " The constraints by which to validate TLS peers. The contents\n" \ + " and format of CONSTRAINTS are plugin-specific.\n" + +#include "flow/unactorcompiler.h" +#endif diff --git a/flow/TLSPolicy.h b/flow/TLSPolicy.h deleted file mode 100644 index 9a0ddfcfa9c..00000000000 --- a/flow/TLSPolicy.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * TLSPolicy.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _FLOW_TLSPOLICY_H_ -#define _FLOW_TLSPOLICY_H_ -#pragma once - -#include -#include -#include -#include -#include "flow/FastRef.h" - -#ifndef TLS_DISABLED - -#include -typedef int NID; - -enum class MatchType { - EXACT, - PREFIX, - SUFFIX, -}; - -enum class X509Location { - // This NID is located within a X509_NAME - NAME, - // This NID is an X509 extension, and should be parsed accordingly - EXTENSION, -}; - -struct Criteria { - Criteria( const std::string& s ) - : criteria(s), match_type(MatchType::EXACT), location(X509Location::NAME) {} - Criteria( const std::string& s, MatchType mt ) - : criteria(s), match_type(mt), location(X509Location::NAME) {} - Criteria( const std::string& s, X509Location loc) - : criteria(s), match_type(MatchType::EXACT), location(loc) {} - Criteria( const std::string& s, MatchType mt, X509Location loc) - : criteria(s), match_type(mt), location(loc) {} - - std::string criteria; - MatchType match_type; - X509Location location; - - bool operator==(const Criteria& c) const { - return criteria == c.criteria && match_type == c.match_type && location == c.location; - } -}; -#endif - -struct TLSParams { - enum { OPT_TLS = 100000, OPT_TLS_PLUGIN, OPT_TLS_CERTIFICATES, OPT_TLS_KEY, OPT_TLS_VERIFY_PEERS, OPT_TLS_CA_FILE, OPT_TLS_PASSWORD }; - - std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword; - std::string tlsCertBytes, tlsKeyBytes, tlsCABytes; -}; - -class TLSPolicy : ReferenceCounted { -public: - enum class Is { - CLIENT, - SERVER - }; - - TLSPolicy(Is client) : is_client(client == Is::CLIENT) {} - virtual ~TLSPolicy(); - - virtual void addref() { ReferenceCounted::addref(); } - virtual void delref() { ReferenceCounted::delref(); } - -#ifndef TLS_DISABLED - static std::string ErrorString(boost::system::error_code e); - - void set_verify_peers(std::vector verify_peers); - bool verify_peer(bool preverified, X509_STORE_CTX* store_ctx); - - std::string toString() const; - - struct Rule { - explicit Rule(std::string input); - - std::string toString() const; - - std::map< NID, Criteria > subject_criteria; - std::map< NID, Criteria > issuer_criteria; - std::map< NID, Criteria > root_criteria; - - bool verify_cert = true; - bool verify_time = true; - }; - - std::vector rules; -#endif - bool is_client; -}; - -#define TLS_PLUGIN_FLAG "--tls_plugin" -#define TLS_CERTIFICATE_FILE_FLAG "--tls_certificate_file" -#define TLS_KEY_FILE_FLAG "--tls_key_file" -#define TLS_VERIFY_PEERS_FLAG "--tls_verify_peers" -#define TLS_CA_FILE_FLAG "--tls_ca_file" -#define TLS_PASSWORD_FLAG "--tls_password" - -#define TLS_OPTION_FLAGS \ - { TLSParams::OPT_TLS_PLUGIN, TLS_PLUGIN_FLAG, SO_REQ_SEP }, \ - { TLSParams::OPT_TLS_CERTIFICATES, TLS_CERTIFICATE_FILE_FLAG, SO_REQ_SEP }, \ - { TLSParams::OPT_TLS_KEY, TLS_KEY_FILE_FLAG, SO_REQ_SEP }, \ - { TLSParams::OPT_TLS_VERIFY_PEERS, TLS_VERIFY_PEERS_FLAG, SO_REQ_SEP }, \ - { TLSParams::OPT_TLS_PASSWORD, TLS_PASSWORD_FLAG, SO_REQ_SEP }, \ - { TLSParams::OPT_TLS_CA_FILE, TLS_CA_FILE_FLAG, SO_REQ_SEP }, - -#define TLS_HELP \ - " " TLS_CERTIFICATE_FILE_FLAG " CERTFILE\n" \ - " The path of a file containing the TLS certificate and CA\n" \ - " chain.\n" \ - " " TLS_CA_FILE_FLAG " CERTAUTHFILE\n" \ - " The path of a file containing the CA certificates chain.\n" \ - " " TLS_KEY_FILE_FLAG " KEYFILE\n" \ - " The path of a file containing the private key corresponding\n" \ - " to the TLS certificate.\n" \ - " " TLS_PASSWORD_FLAG " PASSCODE\n" \ - " The passphrase of encrypted private key\n" \ - " " TLS_VERIFY_PEERS_FLAG " CONSTRAINTS\n" \ - " The constraints by which to validate TLS peers. The contents\n" \ - " and format of CONSTRAINTS are plugin-specific.\n" - -#endif diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj index 3fb6b0a517a..1002079cc6f 100644 --- a/flow/flow.vcxproj +++ b/flow/flow.vcxproj @@ -50,7 +50,7 @@ - + @@ -94,7 +94,9 @@ - + + false + diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h index 7365220e97a..e8b03db4123 100644 --- a/flow/genericactors.actor.h +++ b/flow/genericactors.actor.h @@ -647,6 +647,36 @@ class AsyncMap : NonCopyable { } }; +template +class ReferencedObject : NonCopyable, public ReferenceCounted> { + public: + ReferencedObject() : value() {} + ReferencedObject(V const& v) : value(v) {} + ReferencedObject(ReferencedObject&& r) : value(std::move(r.value)) {} + void operator=(ReferencedObject&& r) { + value = std::move(r.value); + } + + V const& get() const { + return value; + } + + V& mutate() const { + return value; + } + + void set(V const& v) { + value = v; + } + + static Reference> from(V const& v) { + return Reference>(new ReferencedObject(v)); + } + + private: + V value; +}; + template class AsyncVar : NonCopyable, public ReferenceCounted> { public: diff --git a/flow/network.h b/flow/network.h index 127d765bba2..c15c0de6b08 100644 --- a/flow/network.h +++ b/flow/network.h @@ -32,7 +32,6 @@ #endif #include "flow/serialize.h" #include "flow/IRandom.h" -#include "flow/TLSPolicy.h" enum class TaskPriority { Max = 1000000, @@ -406,9 +405,10 @@ typedef void* flowGlobalType; typedef NetworkAddress (*NetworkAddressFuncPtr)(); typedef NetworkAddressList (*NetworkAddressesFuncPtr)(); +class TLSConfig; class INetwork; extern INetwork* g_network; -extern INetwork* newNet2(bool useThreadPool = false, bool useMetrics = false, Reference policy = Reference(), const TLSParams& tlsParams = TLSParams()); +extern INetwork* newNet2(const TLSConfig& tlsConfig, bool useThreadPool = false, bool useMetrics = false); class INetwork { public: From f657ca069e872450c72d5424eaa1036f5618b284 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 4 Mar 2020 23:51:21 -0800 Subject: [PATCH 2/8] Fix bindings build breakage, because I hadn't built bindings. --- bindings/flow/fdb_flow.actor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index dc37e28b233..a6ddadba8ab 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -25,6 +25,7 @@ #include "flow/DeterministicRandom.h" #include "flow/SystemMonitor.h" +#include "flow/TLSConfig.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. using namespace FDB; @@ -82,7 +83,7 @@ void fdb_flow_test() { fdb->setupNetwork(); startThread(networkThread, fdb); - g_network = newNet2(false); + g_network = newNet2(TLSConfig()); openTraceFile(NetworkAddress(), 1000000, 1000000, "."); systemMonitor(); From 2d95a1e64dd0d6b086f895f35f76d7c42d55db9f Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 5 Mar 2020 17:25:33 -0800 Subject: [PATCH 3/8] Implement certificate refreshing --- flow/Net2.actor.cpp | 233 +++++++++++++++---------------------- flow/TLSConfig.actor.h | 11 +- flow/genericactors.actor.h | 11 +- 3 files changed, 108 insertions(+), 147 deletions(-) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 1780279d3f2..ed0b10c118d 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -38,6 +38,10 @@ #include "flow/Profiler.h" #include "flow/ProtocolVersion.h" #include "flow/TLSConfig.actor.h" +#include "flow/genericactors.actor.h" + +// See the comment in TLSConfig.actor.h for the explanation of why this module breaking include was done. +#include "fdbrpc/IAsyncFile.h" #ifdef WIN32 #include @@ -157,16 +161,12 @@ class Net2 sealed : public INetwork, public INetworkConnections { ASIOReactor reactor; #ifndef TLS_DISABLED - boost::asio::ssl::context sslContext; + AsyncVar>> sslContextVar; #endif TLSConfig tlsConfig; - std::string tlsPassword; + Future backgroundCertRefresh; bool tlsInitialized; - std::string get_password() const { - return tlsPassword; - } - INetworkConnections *network; // initially this, but can be changed int64_t tsc_begin, tsc_end; @@ -505,13 +505,13 @@ class SSLConnection : public IConnection, ReferenceCounted { closeSocket(); } - explicit SSLConnection( boost::asio::io_service& io_service, boost::asio::ssl::context& context ) - : id(nondeterministicRandom()->randomUniqueID()), socket(io_service), ssl_sock(socket, context) + explicit SSLConnection( boost::asio::io_service& io_service, Reference> context ) + : id(nondeterministicRandom()->randomUniqueID()), socket(io_service), ssl_sock(socket, context->mutate()), sslContext(context) { } // This is not part of the IConnection interface, because it is wrapped by INetwork::connect() - ACTOR static Future> connect( boost::asio::io_service* ios, boost::asio::ssl::context* context, NetworkAddress addr ) { + ACTOR static Future> connect( boost::asio::io_service* ios, Reference> context, NetworkAddress addr ) { std::pair peerIP = std::make_pair(addr.ip, addr.port); auto iter(g_network->networkInfo.serverTLSConnectionThrottler.find(peerIP)); if(iter != g_network->networkInfo.serverTLSConnectionThrottler.end()) { @@ -526,7 +526,7 @@ class SSLConnection : public IConnection, ReferenceCounted { } } - state Reference self( new SSLConnection(*ios, *context) ); + state Reference self( new SSLConnection(*ios, context) ); self->peer_address = addr; try { @@ -729,6 +729,7 @@ class SSLConnection : public IConnection, ReferenceCounted { tcp::socket socket; ssl_socket ssl_sock; NetworkAddress peer_address; + Reference> sslContext; struct SendBufferIterator { typedef boost::asio::const_buffer value_type; @@ -789,11 +790,11 @@ class SSLListener : public IListener, ReferenceCounted { boost::asio::io_context& io_service; NetworkAddress listenAddress; tcp::acceptor acceptor; - boost::asio::ssl::context* context; + AsyncVar>> *contextVar; public: - SSLListener( boost::asio::io_context& io_service, boost::asio::ssl::context* context, NetworkAddress listenAddress ) - : io_service(io_service), listenAddress(listenAddress), acceptor( io_service, tcpEndpoint( listenAddress ) ), context(context) + SSLListener( boost::asio::io_context& io_service, AsyncVar>>* contextVar, NetworkAddress listenAddress ) + : io_service(io_service), listenAddress(listenAddress), acceptor( io_service, tcpEndpoint( listenAddress ) ), contextVar(contextVar) { platform::setCloseOnExec(acceptor.native_handle()); } @@ -810,7 +811,7 @@ class SSLListener : public IListener, ReferenceCounted { private: ACTOR static Future> doAccept( SSLListener* self ) { - state Reference conn( new SSLConnection( self->io_service, *self->context) ); + state Reference conn( new SSLConnection( self->io_service, self->contextVar->get() ) ); state tcp::acceptor::endpoint_type peer_endpoint; try { BindPromise p("N2_AcceptError", UID()); @@ -862,7 +863,7 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) tlsInitialized(false), tlsConfig(tlsConfig) #ifndef TLS_DISABLED - ,sslContext(boost::asio::ssl::context(boost::asio::ssl::context::tls)) + ,sslContextVar({ReferencedObject::from(boost::asio::ssl::context(boost::asio::ssl::context::tls))}) #endif { @@ -889,103 +890,92 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) } -/* -ACTOR static Future watchFileForChanges( std::string filename, AsyncVar> *contents_var ) { +void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::context* context ) { + context->set_options(boost::asio::ssl::context::default_workarounds); + context->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); + + if (loaded.isTLSEnabled()) { + Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); + tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); + + context->set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { + return policy->verify_peer(preverified, ctx.native_handle()); + }); + } else { + context->set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); + } + + context->set_password_callback( + [password=loaded.getPassword()](size_t, boost::asio::ssl::context::password_purpose) { + return password; + }); + + const std::string& certBytes = loaded.getCertificateBytes(); + if ( certBytes.size() ) { + context->use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size())); + } + + const std::string& CABytes = loaded.getCABytes(); + if ( CABytes.size() ) { + context->add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size())); + } + + const std::string& keyBytes = loaded.getKeyBytes(); + if (keyBytes.size()) { + context->use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()), boost::asio::ssl::context::pem); + } +} + +ACTOR static Future watchFileForChanges( std::string filename, AsyncTrigger* fileChanged ) { + if (filename == "") { + return Never(); + } state std::time_t lastModTime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); loop { wait(delay(FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS)); std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); if (lastModTime != modtime) { lastModTime = modtime; - ErrorOr> contents = wait(readEntireFile(filename)); - if (contents.present()) { - contents_var->set(contents.get()); - } + fileChanged->trigger(); } } } -ACTOR static Future reloadConfigurationOnChange( TLSOptions::PolicyInfo *pci, Reference plugin, AsyncVar> *realVerifyPeersPolicy, AsyncVar> *realNoVerifyPeersPolicy ) { - if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) { - return Void(); - return Void(); - } - loop { - // Early in bootup, the filesystem might not be initialized yet. Wait until it is. - if (IAsyncFileSystem::filesystem() != nullptr) { - break; - } - wait(delay(1.0)); - } - state int mismatches = 0; - state AsyncVar> ca_var; - state AsyncVar> key_var; - state AsyncVar> cert_var; - state std::vector> lifetimes; - if (!pci->ca_path.empty()) lifetimes.push_back(watchFileForChanges(pci->ca_path, &ca_var)); - if (!pci->key_path.empty()) lifetimes.push_back(watchFileForChanges(pci->key_path, &key_var)); - if (!pci->cert_path.empty()) lifetimes.push_back(watchFileForChanges(pci->cert_path, &cert_var)); - loop { - state Future ca_changed = ca_var.onChange(); - state Future key_changed = key_var.onChange(); - state Future cert_changed = cert_var.onChange(); - wait( ca_changed || key_changed || cert_changed ); - if (ca_changed.isReady()) { - TraceEvent(SevInfo, "TLSRefreshCAChanged").detail("path", pci->ca_path).detail("length", ca_var.get().size()); - pci->ca_contents = ca_var.get(); - } - if (key_changed.isReady()) { - TraceEvent(SevInfo, "TLSRefreshKeyChanged").detail("path", pci->key_path).detail("length", key_var.get().size()); - pci->key_contents = key_var.get(); - } - if (cert_changed.isReady()) { - TraceEvent(SevInfo, "TLSRefreshCertChanged").detail("path", pci->cert_path).detail("length", cert_var.get().size()); - pci->cert_contents = cert_var.get(); - } - bool rc = true; - Reference verifypeers = Reference(plugin->create_policy()); - Reference noverifypeers = Reference(plugin->create_policy()); - loop { - // Don't actually loop. We're just using loop/break as a `goto err`. - // This loop always ends with an unconditional break. - rc = verifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size()); - if (!rc) break; - rc = verifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str()); - if (!rc) break; - rc = verifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size()); - if (!rc) break; - { - std::unique_ptr verify_peers_arr(new const uint8_t*[pci->verify_peers.size()]); - std::unique_ptr verify_peers_len(new int[pci->verify_peers.size()]); - for (int i = 0; i < pci->verify_peers.size(); i++) { - verify_peers_arr[i] = (const uint8_t *)&pci->verify_peers[i][0]; - verify_peers_len[i] = pci->verify_peers[i].size(); - } - rc = verifypeers->set_verify_peers(pci->verify_peers.size(), verify_peers_arr.get(), verify_peers_len.get()); - if (!rc) break; - } - rc = noverifypeers->set_ca_data(pci->ca_contents.begin(), pci->ca_contents.size()); - if (!rc) break; - rc = noverifypeers->set_key_data(pci->key_contents.begin(), pci->key_contents.size(), pci->keyPassword.c_str()); - if (!rc) break; - rc = noverifypeers->set_cert_data(pci->cert_contents.begin(), pci->cert_contents.size()); - if (!rc) break; - break; - } - - if (rc) { - TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded"); - realVerifyPeersPolicy->set(verifypeers); - realNoVerifyPeersPolicy->set(noverifypeers); - mismatches = 0; - } else { - // Some files didn't match up, they should in the future, and we'll retry then. - mismatches++; - TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").detail("mismatches", mismatches); - } - } +ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar>>* contextVar ) { + if (FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS <= 0) { + return Void(); + } + loop { + // Early in bootup, the filesystem might not be initialized yet. Wait until it is. + if (IAsyncFileSystem::filesystem() != nullptr) { + break; + } + wait(delay(1.0)); + } + state int mismatches = 0; + state AsyncTrigger fileChanged; + state std::vector> lifetimes; + lifetimes.push_back(watchFileForChanges(config.getCertificatePathSync(), &fileChanged)); + lifetimes.push_back(watchFileForChanges(config.getKeyPathSync(), &fileChanged)); + lifetimes.push_back(watchFileForChanges(config.getCAPathSync(), &fileChanged)); + loop { + wait( fileChanged.onTrigger() ); + TraceEvent("TLSCertificateRefreshBegin"); + + try { + LoadedTLSConfig loaded = wait( config.loadAsync() ); + boost::asio::ssl::context context(boost::asio::ssl::context::tls); + ConfigureSSLContext(loaded, &context); + TraceEvent(SevInfo, "TLSCertificateRefreshSucceeded"); + mismatches = 0; + contextVar->set(ReferencedObject::from(std::move(context))); + } catch (Error &e) { + // Some files didn't match up, they should in the future, and we'll retry then. + mismatches++; + TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").detail("mismatches", mismatches); + } + } } -*/ void Net2::initTLS() { if(tlsInitialized) { @@ -993,39 +983,10 @@ void Net2::initTLS() { } #ifndef TLS_DISABLED try { - LoadedTLSConfig loaded = tlsConfig.loadSync(); - - sslContext.set_options(boost::asio::ssl::context::default_workarounds); - sslContext.set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); - - if (loaded.isTLSEnabled()) { - Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); - tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); - - sslContext.set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { - return policy->verify_peer(preverified, ctx.native_handle()); - }); - } else { - sslContext.set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); - } - - tlsPassword = loaded.getPassword(); - sslContext.set_password_callback(std::bind(&Net2::get_password, this)); - - const std::string& certBytes = loaded.getCertificateBytes(); - if ( certBytes.size() ) { - sslContext.use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size())); - } - - const std::string& CABytes = loaded.getCABytes(); - if ( CABytes.size() ) { - sslContext.add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size())); - } - - const std::string& keyBytes = loaded.getKeyBytes(); - if (keyBytes.size()) { - sslContext.use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()), boost::asio::ssl::context::pem); - } + boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); + ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); + sslContextVar.set(ReferencedObject::from(std::move(newContext))); + backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); } catch(boost::system::system_error e) { TraceEvent("Net2TLSInitError").detail("Message", e.what()); throw tls_error(); @@ -1392,7 +1353,7 @@ Future< Reference > Net2::connect( NetworkAddress toAddr, std::stri #ifndef TLS_DISABLED initTLS(); if ( toAddr.isTLS() ) { - return SSLConnection::connect(&this->reactor.ios, &this->sslContext, toAddr); + return SSLConnection::connect(&this->reactor.ios, this->sslContextVar.get(), toAddr); } #endif @@ -1472,7 +1433,7 @@ Reference Net2::listen( NetworkAddress localAddr ) { #ifndef TLS_DISABLED initTLS(); if ( localAddr.isTLS() ) { - return Reference(new SSLListener( reactor.ios, &this->sslContext, localAddr )); + return Reference(new SSLListener( reactor.ios, &this->sslContextVar, localAddr )); } #endif return Reference( new Listener( reactor.ios, localAddr ) ); diff --git a/flow/TLSConfig.actor.h b/flow/TLSConfig.actor.h index e750c3de938..667a6f28225 100644 --- a/flow/TLSConfig.actor.h +++ b/flow/TLSConfig.actor.h @@ -194,7 +194,6 @@ class TLSConfig { return loadAsync(this); } -PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP: // Return the explicitly set path. // If one was not set, return the path from the environment. // (Cert and Key only) If neither exist, check for fdb.pem in cwd @@ -206,6 +205,7 @@ class TLSConfig { std::string getKeyPathSync() const; std::string getCAPathSync() const; +PRIVATE_EXCEPT_FOR_TLSCONFIG_CPP: ACTOR static Future loadAsync(const TLSConfig* self); template friend class LoadAsyncActorState; @@ -217,15 +217,6 @@ class TLSConfig { TLSEndpointType endpointType = TLSEndpointType::UNSET; }; -namespace boost { - namespace asio { - namespace ssl { - struct context; - } - } -} -void ConfigureSSLContext( boost::asio::ssl::context *context, const LoadedTLSConfig& config ); - class TLSPolicy : ReferenceCounted { public: diff --git a/flow/genericactors.actor.h b/flow/genericactors.actor.h index ae022b9ef7d..14db6428abd 100644 --- a/flow/genericactors.actor.h +++ b/flow/genericactors.actor.h @@ -652,6 +652,7 @@ class ReferencedObject : NonCopyable, public ReferenceCounted> from(V const& v) { return Reference>(new ReferencedObject(v)); } + static Reference> from(V&& v) { + return Reference>(new ReferencedObject(std::move(v))); + } + private: V value; }; From ccef3f7d05cb9c250652b8f6c75f98d18ed87e02 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 5 Mar 2020 17:32:10 -0800 Subject: [PATCH 4/8] Attempt to fix TLS_DISABLED compiles. --- flow/Net2.actor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index ed0b10c118d..2080b7a336a 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -890,6 +890,7 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) } +#ifndef TLS_DISABLED void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::context* context ) { context->set_options(boost::asio::ssl::context::default_workarounds); context->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); @@ -976,6 +977,7 @@ ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar } } } +#endif void Net2::initTLS() { if(tlsInitialized) { From ac52b6b4741ffb1a800e46258356cb8830ed2911 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 6 Mar 2020 02:33:16 -0800 Subject: [PATCH 5/8] Rework a bit of error and exception handling. I went back and dug through all of the "what functions can throw what types", and made sane decisions about them. boost errors are aggressively translated into FDB ones, whcih might result in multiple lines of logging about errors, but this is in infrequently run code, so it should be fine. --- flow/Net2.actor.cpp | 92 +++++++++++++++++++++++----------------- flow/TLSConfig.actor.cpp | 12 +----- 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 2080b7a336a..89f68e3e50e 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -892,38 +892,43 @@ Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) #ifndef TLS_DISABLED void ConfigureSSLContext( const LoadedTLSConfig& loaded, boost::asio::ssl::context* context ) { - context->set_options(boost::asio::ssl::context::default_workarounds); - context->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); + try { + context->set_options(boost::asio::ssl::context::default_workarounds); + context->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); - if (loaded.isTLSEnabled()) { - Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); - tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); + if (loaded.isTLSEnabled()) { + Reference tlsPolicy = Reference(new TLSPolicy(loaded.getEndpointType())); + tlsPolicy->set_verify_peers({ loaded.getVerifyPeers() }); - context->set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { - return policy->verify_peer(preverified, ctx.native_handle()); - }); - } else { - context->set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); - } + context->set_verify_callback([policy=tlsPolicy](bool preverified, boost::asio::ssl::verify_context& ctx) { + return policy->verify_peer(preverified, ctx.native_handle()); + }); + } else { + context->set_verify_callback(boost::bind(&insecurely_always_accept, _1, _2)); + } - context->set_password_callback( - [password=loaded.getPassword()](size_t, boost::asio::ssl::context::password_purpose) { - return password; - }); + context->set_password_callback( + [password=loaded.getPassword()](size_t, boost::asio::ssl::context::password_purpose) { + return password; + }); - const std::string& certBytes = loaded.getCertificateBytes(); - if ( certBytes.size() ) { - context->use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size())); - } + const std::string& certBytes = loaded.getCertificateBytes(); + if ( certBytes.size() ) { + context->use_certificate_chain(boost::asio::buffer(certBytes.data(), certBytes.size())); + } - const std::string& CABytes = loaded.getCABytes(); - if ( CABytes.size() ) { - context->add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size())); - } + const std::string& CABytes = loaded.getCABytes(); + if ( CABytes.size() ) { + context->add_certificate_authority(boost::asio::buffer(CABytes.data(), CABytes.size())); + } - const std::string& keyBytes = loaded.getKeyBytes(); - if (keyBytes.size()) { - context->use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()), boost::asio::ssl::context::pem); + const std::string& keyBytes = loaded.getKeyBytes(); + if (keyBytes.size()) { + context->use_private_key(boost::asio::buffer(keyBytes.data(), keyBytes.size()), boost::asio::ssl::context::pem); + } + } catch (boost::system::system_error& e) { + TraceEvent("TLSConfigureError").detail("What", e.what()).detail("Value", e.code().value()).detail("WhichMeans", TLSPolicy::ErrorString(e.code())); + throw tls_error(); } } @@ -934,10 +939,22 @@ ACTOR static Future watchFileForChanges( std::string filename, AsyncTrigge state std::time_t lastModTime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); loop { wait(delay(FLOW_KNOBS->TLS_CERT_REFRESH_DELAY_SECONDS)); - std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); - if (lastModTime != modtime) { - lastModTime = modtime; - fileChanged->trigger(); + try { + std::time_t modtime = wait(IAsyncFileSystem::filesystem()->lastWriteTime(filename)); + if (lastModTime != modtime) { + lastModTime = modtime; + fileChanged->trigger(); + } + } catch (Error& e) { + if (e.code() == error_code_io_error) { + // EACCES, ELOOP, ENOENT all come out as io_error(), but are more of a system + // configuration issue than an FDB problem. If we managed to load valid + // certificates, then there's no point in crashing, but we should complain + // loudly. IAsyncFile will log the error, but not necessarily as a warning. + TraceEvent(SevWarnAlways, "TLSCertificateRefreshStatError").detail("File", filename); + } else { + throw; + } } } } @@ -973,7 +990,7 @@ ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar } catch (Error &e) { // Some files didn't match up, they should in the future, and we'll retry then. mismatches++; - TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").detail("mismatches", mismatches); + TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").error(e).detail("mismatches", mismatches); } } } @@ -984,15 +1001,10 @@ void Net2::initTLS() { return; } #ifndef TLS_DISABLED - try { - boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); - ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); - sslContextVar.set(ReferencedObject::from(std::move(newContext))); - backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); - } catch(boost::system::system_error e) { - TraceEvent("Net2TLSInitError").detail("Message", e.what()); - throw tls_error(); - } + boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); + ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); + sslContextVar.set(ReferencedObject::from(std::move(newContext))); + backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); #endif tlsInitialized = true; } diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.actor.cpp index 28685e88ad7..ea61317b80f 100644 --- a/flow/TLSConfig.actor.cpp +++ b/flow/TLSConfig.actor.cpp @@ -173,14 +173,10 @@ ACTOR static Future readEntireFile( std::string filename, std::string* des state Reference file = wait(IAsyncFileSystem::filesystem()->open(filename, IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED, 0)); state int64_t filesize = wait(file->size()); if (filesize > FLOW_KNOBS->CERT_FILE_MAX_SIZE) { - throw tls_error(); + throw file_too_large(); } destination->resize(filesize); - int rc = wait(file->read(const_cast(destination->c_str()), filesize, 0)); - if (rc != filesize) { - // File modified during read, probably. The mtime should change, and thus we'll be called again. - throw tls_error(); - } + wait(file->read(const_cast(destination->c_str()), filesize, 0)); return Void(); } @@ -218,10 +214,6 @@ ACTOR Future TLSConfig::loadAsync(const TLSConfig* self) { return loaded; } -void ConfigureSSLContext( boost::asio::ssl::context *context, const LoadedTLSConfig& config ) { - -} - std::string TLSPolicy::ErrorString(boost::system::error_code e) { char* str = ERR_error_string(e.value(), NULL); return std::string(str); From 1f56bf89332478d09a13608817ebd8ade0f265b8 Mon Sep 17 00:00:00 2001 From: Alex Miller <35046903+alexmiller-apple@users.noreply.github.com> Date: Fri, 6 Mar 2020 10:15:04 -0800 Subject: [PATCH 6/8] Fix the build with success() Co-Authored-By: A.J. Beamon --- flow/TLSConfig.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.actor.cpp index ea61317b80f..51e2ac9c931 100644 --- a/flow/TLSConfig.actor.cpp +++ b/flow/TLSConfig.actor.cpp @@ -176,7 +176,7 @@ ACTOR static Future readEntireFile( std::string filename, std::string* des throw file_too_large(); } destination->resize(filesize); - wait(file->read(const_cast(destination->c_str()), filesize, 0)); + wait(success(file->read(const_cast(destination->c_str()), filesize, 0))); return Void(); } From 9b760fae2db4329dc1bfad3ca5da2aecd28acadf Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 6 Mar 2020 11:06:19 -0800 Subject: [PATCH 7/8] Rewrite all Errors into tls_errors if they happen as part of initializing TLS. --- flow/Net2.actor.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 89f68e3e50e..f3613e1b4f2 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -1001,10 +1001,15 @@ void Net2::initTLS() { return; } #ifndef TLS_DISABLED - boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); - ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); - sslContextVar.set(ReferencedObject::from(std::move(newContext))); - backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); + try { + boost::asio::ssl::context newContext(boost::asio::ssl::context::tls); + ConfigureSSLContext( tlsConfig.loadSync(), &newContext ); + sslContextVar.set(ReferencedObject::from(std::move(newContext))); + backgroundCertRefresh = reloadCertificatesOnChange( tlsConfig, &sslContextVar ); + } catch (Error& e) { + TraceEvent("Net2TLSInitError").error(e); + throw tls_error(); + } #endif tlsInitialized = true; } From 188d9b8239b2e7f91a3c84ee6ed8de42d70cd290 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 6 Mar 2020 11:09:17 -0800 Subject: [PATCH 8/8] Don't swallow actor cancellation in certificate refreshing. --- flow/Net2.actor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index f3613e1b4f2..011f335a4cf 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -988,6 +988,9 @@ ACTOR static Future reloadCertificatesOnChange( TLSConfig config, AsyncVar mismatches = 0; contextVar->set(ReferencedObject::from(std::move(context))); } catch (Error &e) { + if (e.code() == error_code_actor_cancelled) { + throw; + } // Some files didn't match up, they should in the future, and we'll retry then. mismatches++; TraceEvent(SevWarn, "TLSCertificateRefreshMismatch").error(e).detail("mismatches", mismatches);