From f8ce1de60132f89a9f8e849b0e61517e89eb4e2d Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Wed, 20 Jun 2018 09:21:23 -0700 Subject: [PATCH 1/9] Added support for compiling TLS into binaries --- FDBLibTLS/FDBLibTLS.vcxproj | 4 +- FDBLibTLS/Makefile | 109 --------------------------------- FDBLibTLS/local.mk | 35 ++++++++--- bindings/c/local.mk | 4 ++ fdbbackup/local.mk | 4 ++ fdbcli/fdbcli.actor.cpp | 17 ++--- fdbcli/local.mk | 4 ++ fdbclient/NativeAPI.actor.cpp | 27 ++++---- fdbrpc/LoadPlugin.h | 15 ++--- fdbrpc/TLSConnection.actor.cpp | 26 ++------ fdbrpc/TLSConnection.h | 16 ++--- fdbrpc/local.mk | 4 ++ fdbrpc/md5/md5.h | 6 +- fdbserver/fdbserver.actor.cpp | 21 +++---- fdbserver/local.mk | 4 ++ 15 files changed, 105 insertions(+), 191 deletions(-) delete mode 100644 FDBLibTLS/Makefile diff --git a/FDBLibTLS/FDBLibTLS.vcxproj b/FDBLibTLS/FDBLibTLS.vcxproj index be6ed76f6f2..579e03297fc 100644 --- a/FDBLibTLS/FDBLibTLS.vcxproj +++ b/FDBLibTLS/FDBLibTLS.vcxproj @@ -21,12 +21,12 @@ - DynamicLibrary + StaticLibrary MultiByte v140_xp - DynamicLibrary + StaticLibrary MultiByte v140_xp diff --git a/FDBLibTLS/Makefile b/FDBLibTLS/Makefile deleted file mode 100644 index c0d05116226..00000000000 --- a/FDBLibTLS/Makefile +++ /dev/null @@ -1,109 +0,0 @@ -PROJECTPATH = $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) -PLUGINPATH = $(PROJECTPATH)/$(PLUGIN) - -CFLAGS ?= -O2 -g - -CXXFLAGS ?= -std=c++0x - -CFLAGS += -I/usr/local/include -LDFLAGS += -L/usr/local/lib - -LIBS += -ltls -lssl -lcrypto - -PLATFORM := $(shell uname) -ifneq ($(PLATFORM),Darwin) - PLATFORM := $(shell uname -o) -endif - -ifeq ($(PLATFORM),Cygwin) - HOST := x86_64-w64-mingw32 - CC := $(HOST)-gcc - CXX := $(HOST)-g++ - STRIP := $(HOST)-strip --strip-all - - DYEXT = dll - PLUGINPATH = $(PLUGIN) - - LIBS += -static-libstdc++ -static-libgcc - LIBS += -lws2_32 - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-soname,$(PLUGIN) - LINK_LDFLAGS += -Wl,--version-script=FDBLibTLS.map - LINK_LDFLAGS += -Wl,-Bstatic $(LIBS) -Wl,-Bdynamic - -else ifeq ($(PLATFORM),Darwin) - CC := clang - CXX := clang++ - STRIP := strip -S -x - - CFLAGS += -fPIC - - DYEXT = dylib - - vpath %.a /usr/local/lib - .LIBPATTERNS = lib%.a lib%.dylib lib%.so - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-exported_symbols_list,FDBLibTLS.symbols - LINK_LDFLAGS += -Wl,-dylib_install_name,$(PLUGIN) - LINK_LDFLAGS += $(LIBS) - -else ifeq ($(PLATFORM),GNU/Linux) - CC := clang - CXX := clang++ - STRIP := strip --strip-all - - CFLAGS += -fPIC - DYEXT = so - - LIBS += -static-libstdc++ -static-libgcc -lrt - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-soname,$(PLUGIN) - LINK_LDFLAGS += -Wl,--version-script=FDBLibTLS.map - LINK_LDFLAGS += -Wl,-Bstatic $(LIBS) -Wl,-Bdynamic - -else -$(error Unknown platform $(PLATFORM)) -endif - -PLUGIN := FDBLibTLS.$(DYEXT) -OBJECTS := FDBLibTLSPlugin.o FDBLibTLSPolicy.o FDBLibTLSSession.o FDBLibTLSVerify.o -LINKLINE := $(CXXFLAGS) $(CFLAGS) $(LDFLAGS) $(OBJECTS) $(LINK_LDFLAGS) -o $(PLUGIN) - -all: $(PLUGIN) - -build-depends-linux: - apt install clang make libboost-dev - -clean: - @rm -f *.o *.d $(PLUGIN) plugin-test verify-test - @rm -rf *.dSYM - -DEPS := $(patsubst %.o,%.d,$(OBJECTS)) --include $(DEPS) - -$(OBJECTS): %.o: %.cpp Makefile - @echo "Compiling $<" - @$(CXX) $(CXXFLAGS) $(CFLAGS) $(INCLUDES) -c $< -o $@ -MD -MP - -$(PLUGIN): $(OBJECTS) Makefile - @echo "Linking $@" - @$(CXX) $(LINKLINE) - @echo "Stripping $@" - @$(STRIP) $@ - -test: test-plugin test-verify - -test-plugin: plugin-test.cpp $(PLUGIN) Makefile - @echo "Compiling plugin-test" - @$(CXX) $(CXXFLAGS) $(CFLAGS) plugin-test.cpp -ldl -o plugin-test - @echo "Running plugin-test..." - @$(PROJECTPATH)/plugin-test $(PLUGINPATH) - -test-verify: verify-test.cpp $(OBJECTS) Makefile - @echo "Compiling verify-test" - @$(CXX) $(CXXFLAGS) $(CFLAGS) $(LDFLAGS) $(OBJECTS) verify-test.cpp $(LIBS) -o verify-test - @echo "Running verify-test..." - @$(PROJECTPATH)/verify-test diff --git a/FDBLibTLS/local.mk b/FDBLibTLS/local.mk index 5e6b9cfb405..e3e1e82eb1a 100644 --- a/FDBLibTLS/local.mk +++ b/FDBLibTLS/local.mk @@ -1,11 +1,28 @@ -FDBLibTLS_CFLAGS := -fPIC -I/usr/local/include -I$(BOOSTDIR) -Ifdbrpc -FDBLibTLS_STATIC_LIBS := -ltls -lssl -lcrypto -FDBLibTLS_LDFLAGS := -L/usr/local/lib -static-libstdc++ -static-libgcc -lrt -FDBLibTLS_LDFLAGS += -Wl,-soname,FDBLibTLS.so -Wl,--version-script=FDBLibTLS/FDBLibTLS.map +# +# local.mk +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -# The plugin isn't a typical library, so it feels more sensible to have a copy -# of it in bin/. -bin/fdb-libressl-plugin.$(DLEXT): lib/libFDBLibTLS.$(DLEXT) - @cp $< $@ +# -*- mode: makefile; -*- -TARGETS += bin/fdb-libressl-plugin.$(DLEXT) +FDBLibTLS_BUILD_SOURCES += + + +FDBLibTLS_CFLAGS := -fPIC -I/usr/local/include -I$(BOOSTDIR) -Ifdbrpc -DUSE_UCONTEXT + +lib/libFDBLibTLS.a: bin/coverage.FDBLibTLS.xml diff --git a/bindings/c/local.mk b/bindings/c/local.mk index 44f0c31b422..a87aad98821 100644 --- a/bindings/c/local.mk +++ b/bindings/c/local.mk @@ -26,6 +26,10 @@ fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a fdb_c_tests_LIBS := -Llib -lfdb_c fdb_c_tests_HEADERS := -Ibindings/c +ifndef __TLS_DISABLED__ +fdb_c_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a +endif + CLEAN_TARGETS += fdb_c_tests_clean ifeq ($(PLATFORM),linux) diff --git a/fdbbackup/local.mk b/fdbbackup/local.mk index 033fe7d4046..c4e234162cd 100644 --- a/fdbbackup/local.mk +++ b/fdbbackup/local.mk @@ -24,6 +24,10 @@ fdbbackup_CFLAGS := $(fdbclient_CFLAGS) fdbbackup_LDFLAGS := $(fdbrpc_LDFLAGS) fdbbackup_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a +ifndef __TLS_DISABLED__ +fdbbackup_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a +endif + ifeq ($(PLATFORM),linux) fdbbackup_LIBS += -ldl -lpthread -lrt fdbbackup_LDFLAGS += -static-libstdc++ -static-libgcc diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 13757c07fc8..db0e6eb5152 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -72,7 +72,9 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_VERSION, "--version", SO_NONE }, { OPT_VERSION, "-v", SO_NONE }, +#ifndef __TLS_DISABLED__ TLS_OPTION_FLAGS +#endif SO_END_OF_OPTIONS }; @@ -400,7 +402,9 @@ static void printProgramUsage(const char* name) { " and then exits.\n" " --no-status Disables the initial status check done when starting\n" " the CLI.\n" +#ifndef __TLS_DISABLED__ TLS_HELP +#endif " -v, --version Print FoundationDB CLI version information and exit.\n" " -h, --help Display this help and exit.\n"); } @@ -1218,7 +1222,7 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, } if(drSecondaryTags.size() > 0) { outputString += format("%d as secondary", drSecondaryTags.size()); - } + } } // status details @@ -2141,14 +2145,9 @@ struct CLIOptions { initialStatusCheck = false; break; +#ifndef __TLS_DISABLED__ // TLS Options case TLSOptions::OPT_TLS_PLUGIN: - try { - setNetworkOption(FDBNetworkOptions::TLS_PLUGIN, std::string(args.OptionArg())); - } catch( Error& e ) { - fprintf(stderr, "ERROR: cannot load TLS plugin `%s' (%s)\n", args.OptionArg(), e.what()); - return 1; - } break; case TLSOptions::OPT_TLS_CERTIFICATES: tlsCertPath = args.OptionArg(); @@ -2165,6 +2164,7 @@ struct CLIOptions { case TLSOptions::OPT_TLS_VERIFY_PEERS: tlsVerifyPeers = args.OptionArg(); break; +#endif case OPT_HELP: printProgramUsage(program_name.c_str()); return 0; @@ -2224,7 +2224,9 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now. + printf("Setting network thread\n"); TraceEvent::setNetworkThread(); + printf("Set network thread\n"); try { cluster = Cluster::createCluster(ccf->getFilename().c_str(), -1); @@ -3185,6 +3187,7 @@ int main(int argc, char **argv) { return 1; } } + if (opt.tlsCAPath.size()) { try { setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, opt.tlsCAPath); diff --git a/fdbcli/local.mk b/fdbcli/local.mk index 81a4a42e086..79cce6f2e2e 100644 --- a/fdbcli/local.mk +++ b/fdbcli/local.mk @@ -25,6 +25,10 @@ fdbcli_LDFLAGS := $(fdbrpc_LDFLAGS) fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ldl fdbcli_STATIC_LIBS := +ifndef __TLS_DISABLED__ +fdb_cli_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a +endif + fdbcli_GENERATED_SOURCES += versions.h ifeq ($(PLATFORM),linux) diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 373d0cf7f8f..e6da4adab9b 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -454,7 +454,7 @@ ACTOR static Future clientStatusUpdateActor(DatabaseContext *cx) { ACTOR static Future monitorMasterProxiesChange(Reference> clientDBInfo, AsyncTrigger *triggerVar) { state vector< MasterProxyInterface > curProxies; curProxies = clientDBInfo->get().proxies; - + loop{ Void _ = wait(clientDBInfo->onChange()); if (clientDBInfo->get().proxies != curProxies) { @@ -470,10 +470,10 @@ DatabaseContext::DatabaseContext( Standalone dbName, Standalone dbId, int taskID, LocalityData clientLocality, bool enableLocalityLoadBalance, bool lockAware ) : clientInfo(clientInfo), masterProxiesChangeTrigger(), cluster(cluster), clientInfoMonitor(clientInfoMonitor), dbName(dbName), dbId(dbId), - transactionReadVersions(0), transactionLogicalReads(0), transactionPhysicalReads(0), transactionCommittedMutations(0), transactionCommittedMutationBytes(0), transactionsCommitStarted(0), + transactionReadVersions(0), transactionLogicalReads(0), transactionPhysicalReads(0), transactionCommittedMutations(0), transactionCommittedMutationBytes(0), transactionsCommitStarted(0), transactionsCommitCompleted(0), transactionsTooOld(0), transactionsFutureVersions(0), transactionsNotCommitted(0), transactionsMaybeCommitted(0), transactionsResourceConstrained(0), taskID(taskID), outstandingWatches(0), maxOutstandingWatches(CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), - latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000) + latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000) { logger = databaseLogger( this ); locationCacheSize = g_network->isSimulated() ? @@ -488,7 +488,7 @@ DatabaseContext::DatabaseContext( } ACTOR static Future monitorClientInfo( Reference>> clusterInterface, Standalone dbName, - Reference ccf, Reference> outInfo ) + Reference ccf, Reference> outInfo ) { try { loop { @@ -782,7 +782,6 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } case FDBNetworkOptions::TLS_PLUGIN: validateOptionValue(value, true); - tlsOptions->set_plugin_name_or_path( value.get().toString() ); break; case FDBNetworkOptions::TLS_CERT_PATH: validateOptionValue(value, true); @@ -871,7 +870,9 @@ void setupNetwork(uint64_t transportId, bool useMetrics) { FlowTransport::createInstance(transportId); Net2FileSystem::newFileSystem(); +#ifndef __TLS_DISABLED__ tlsOptions->register_network(); +#endif } void runNetwork() { @@ -1033,7 +1034,7 @@ ACTOR Future< pair> > getKeyLocation_internal( if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before"); - + loop { choose { when ( Void _ = wait( cx->onMasterProxiesChanged() ) ) {} @@ -1247,7 +1248,7 @@ ACTOR Future getKey( Database cx, KeySelector k, Future version, T Key locationKey(k.getKey(), k.arena()); state pair> ssi = wait( getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, k.isBackward()) ); - + try { if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.Before"); //.detail("StartKey", printable(k.getKey())).detail("Offset",k.offset).detail("OrEqual",k.orEqual); @@ -1554,8 +1555,8 @@ ACTOR Future> getRangeFallback( Database cx, Version return r; } -void getRangeFinished(Reference trLogInfo, double startTime, KeySelector begin, KeySelector end, bool snapshot, - Promise> conflictRange, bool reverse, Standalone result) +void getRangeFinished(Reference trLogInfo, double startTime, KeySelector begin, KeySelector end, bool snapshot, + Promise> conflictRange, bool reverse, Standalone result) { if( trLogInfo ) { int rangeSize = 0; @@ -1601,7 +1602,7 @@ void getRangeFinished(Reference trLogInfo, double startTime, } ACTOR Future> getRange( Database cx, Reference trLogInfo, Future fVersion, - KeySelector begin, KeySelector end, GetRangeLimits limits, Promise> conflictRange, bool snapshot, bool reverse, + KeySelector begin, KeySelector end, GetRangeLimits limits, Promise> conflictRange, bool snapshot, bool reverse, TransactionInfo info ) { state GetRangeLimits originalLimits( limits ); @@ -1801,8 +1802,8 @@ ACTOR Future> getRange( Database cx, Reference> getRange( Database const& cx, Future const& fVersion, KeySelector const& begin, KeySelector const& end, - GetRangeLimits const& limits, bool const& reverse, TransactionInfo const& info ) +Future> getRange( Database const& cx, Future const& fVersion, KeySelector const& begin, KeySelector const& end, + GetRangeLimits const& limits, bool const& reverse, TransactionInfo const& info ) { return getRange(cx, Reference(), fVersion, begin, end, limits, Promise>(), true, reverse, info); } @@ -2903,7 +2904,7 @@ ACTOR Future< StorageMetrics > waitStorageMetrics( state int tooManyShardsCount = 0; loop { state vector< pair> > locations = wait( getKeyRangeLocations( cx, keys, shardLimit, false, &StorageServerInterface::waitMetrics, TransactionInfo(TaskDataDistribution) ) ); - + if( locations.size() == shardLimit ) { TraceEvent(!g_network->isSimulated() && ++tooManyShardsCount >= 15 ? SevWarnAlways : SevWarn, "WaitStorageMetricsPenalty") .detail("Keys", printable(keys)) diff --git a/fdbrpc/LoadPlugin.h b/fdbrpc/LoadPlugin.h index 183fa9c948c..d64dbc4ca10 100644 --- a/fdbrpc/LoadPlugin.h +++ b/fdbrpc/LoadPlugin.h @@ -20,13 +20,14 @@ #pragma once +// Will need to be defined at link time +extern "C" void *get_plugin(const char *plugin_type_name_and_version); + template Reference loadPlugin( std::string const& plugin_name ) { - void* plugin = loadLibrary( plugin_name.c_str() ); - void *(*get_plugin)(const char*) = (void*(*)(const char*))loadFunction( plugin, "get_plugin" ); - - if ( get_plugin ) - return Reference( (T*)get_plugin( T::get_plugin_type_name_and_version() ) ); - else - return Reference( NULL ); +#ifdef __TLS_DISABLED__ + return Reference( NULL ); +#else + return (plugin_name.empty()) ? Reference( NULL ) : Reference( (T*)get_plugin( T::get_plugin_type_name_and_version() ) ); +#endif } diff --git a/fdbrpc/TLSConnection.actor.cpp b/fdbrpc/TLSConnection.actor.cpp index 3c091281015..adbe4249906 100644 --- a/fdbrpc/TLSConnection.actor.cpp +++ b/fdbrpc/TLSConnection.actor.cpp @@ -194,13 +194,6 @@ Reference TLSNetworkConnections::listen( NetworkAddress localAddr ) { // 5MB for loading files into memory #define CERT_FILE_MAX_SIZE (5 * 1024 * 1024) -void TLSOptions::set_plugin_name_or_path( std::string const& plugin_name_or_path ) { - if ( plugin ) - throw invalid_option(); - - init_plugin( plugin_name_or_path ); -} - void TLSOptions::set_cert_file( std::string const& cert_file ) { try { TraceEvent("TLSConnectionSettingCertFile").detail("CertFilePath", cert_file); @@ -374,22 +367,15 @@ static void TLSConnectionLogFunc( const char* event, void* uid_ptr, bool is_erro va_end( ap ); } -void TLSOptions::init_plugin( std::string const& plugin_path ) { - std::string path; +void TLSOptions::init_plugin() { + std::string plugin_path = platform::getDefaultPluginPath("fdb-libressl-plugin"); - if ( plugin_path.length() ) { - path = plugin_path; - } else { - if ( !platform::getEnvironmentVar( "FDB_TLS_PLUGIN", path ) ) - // FIXME: should there be other fallbacks? - path = platform::getDefaultPluginPath("fdb-libressl-plugin"); - } + TraceEvent("TLSConnectionLoadingPlugin").detail("PluginPath", plugin_path); + + plugin = loadPlugin( plugin_path.c_str() ); - TraceEvent("TLSConnectionLoadingPlugin").detail("PluginPath", path); - plugin = loadPlugin( path.c_str() ); if ( !plugin ) { - // FIXME: allow? - TraceEvent(SevError, "TLSConnectionPluginInitError").detail("Plugin", path).GetLastError(); + TraceEvent(SevError, "TLSConnectionPluginInitError").detail("Plugin", plugin_path).GetLastError(); throw tls_error(); } diff --git a/fdbrpc/TLSConnection.h b/fdbrpc/TLSConnection.h index 18da10843a4..0f575f8b9a3 100644 --- a/fdbrpc/TLSConnection.h +++ b/fdbrpc/TLSConnection.h @@ -82,9 +82,12 @@ struct TLSListener : IListener, ReferenceCounted { struct TLSOptions : ReferenceCounted { enum { OPT_TLS = 100000, OPT_TLS_PLUGIN, OPT_TLS_CERTIFICATES, OPT_TLS_KEY, OPT_TLS_VERIFY_PEERS, OPT_TLS_CA_FILE, OPT_TLS_PASSWORD }; enum PolicyType { POLICY_VERIFY_PEERS = 1, POLICY_NO_VERIFY_PEERS }; - TLSOptions() : certs_set(false), key_set(false), verify_peers_set(false), ca_set(false) {} + TLSOptions() : certs_set(false), key_set(false), verify_peers_set(false), ca_set(false) { +#ifndef __TLS_DISABLED__ + init_plugin( ); +#endif + } - void set_plugin_name_or_path( std::string const& plugin_name_or_path ); void set_cert_file( std::string const& cert_file ); void set_cert_data( std::string const& cert_data ); void set_ca_file(std::string const& ca_file); @@ -101,7 +104,7 @@ struct TLSOptions : ReferenceCounted { bool enabled(); private: - void init_plugin( std::string const& plugin_path = "" ); + void init_plugin( ); Reference plugin; Reference policyVerifyPeersSet; @@ -124,7 +127,6 @@ struct TLSNetworkConnections : INetworkConnections { Reference options; }; -#define TLS_PLUGIN_FLAG "--tls_plugin" #define TLS_CERTIFICATE_FILE_FLAG "--tls_certificate_file" #define TLS_KEY_FILE_FLAG "--tls_key_file" #define TLS_VERIFY_PEERS_FLAG "--tls_verify_peers" @@ -132,7 +134,6 @@ struct TLSNetworkConnections : INetworkConnections { #define TLS_PASSWORD_FLAG "--tls_password" #define TLS_OPTION_FLAGS \ - { TLSOptions::OPT_TLS_PLUGIN, TLS_PLUGIN_FLAG, SO_OPT }, \ { TLSOptions::OPT_TLS_CERTIFICATES, TLS_CERTIFICATE_FILE_FLAG, SO_REQ_SEP }, \ { TLSOptions::OPT_TLS_KEY, TLS_KEY_FILE_FLAG, SO_REQ_SEP }, \ { TLSOptions::OPT_TLS_VERIFY_PEERS, TLS_VERIFY_PEERS_FLAG, SO_REQ_SEP }, \ @@ -140,11 +141,6 @@ struct TLSNetworkConnections : INetworkConnections { { TLSOptions::OPT_TLS_CA_FILE, TLS_CA_FILE_FLAG, SO_REQ_SEP }, #define TLS_HELP \ - " " TLS_PLUGIN_FLAG " PLUGIN\n" \ - " The name/path of a FoundationDB TLS plugin to be loaded.\n" \ - " PLUGIN will be opened using dlopen (or LoadLibrary on\n" \ - " Windows) and will be located using the search order\n" \ - " of dlopen or LoadLibrary on your platform.\n" \ " " TLS_CERTIFICATE_FILE_FLAG " CERTFILE\n" \ " The path of a file containing the TLS certificate and CA\n" \ " chain.\n" \ diff --git a/fdbrpc/local.mk b/fdbrpc/local.mk index 2e0bd42933f..eb26c0874aa 100644 --- a/fdbrpc/local.mk +++ b/fdbrpc/local.mk @@ -25,6 +25,10 @@ fdbrpc_BUILD_SOURCES += fdbrpc/libeio/eio.c fdbrpc_CFLAGS := -I$(BOOSTDIR) -I. -Ifdbrpc -Ifdbrpc/libeio -DUSE_UCONTEXT fdbrpc_LDFLAGS := +ifdef __TLS_DISABLED__ +fdbrpc_CFLAGS += -D__TLS_DISABLED__ +endif + ifeq ($(PLATFORM),osx) fdbrpc_CFLAGS += -fasynchronous-unwind-tables -fno-omit-frame-pointer fdbrpc_BUILD_SOURCES += fdbrpc/libcoroutine/asm.S fdbrpc/libcoroutine/context.c diff --git a/fdbrpc/md5/md5.h b/fdbrpc/md5/md5.h index a8e90b8b7a6..882f021a480 100644 --- a/fdbrpc/md5/md5.h +++ b/fdbrpc/md5/md5.h @@ -41,9 +41,9 @@ typedef struct { #ifdef __cplusplus extern "C" { #endif - void MD5_Init(MD5_CTX *ctx); - void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); - void MD5_Final(unsigned char *result, MD5_CTX *ctx); + void __attribute__((weak)) MD5_Init(MD5_CTX *ctx); + void __attribute__((weak)) MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); + void __attribute__((weak)) MD5_Final(unsigned char *result, MD5_CTX *ctx); #ifdef __cplusplus } #endif diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 38e244c2404..005f957d951 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -151,7 +151,9 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_IO_TRUST_SECONDS, "--io_trust_seconds", SO_REQ_SEP }, { OPT_IO_TRUST_WARN_ONLY, "--io_trust_warn_only", SO_NONE }, +#ifndef __TLS_DISABLED__ TLS_OPTION_FLAGS +#endif SO_END_OF_OPTIONS }; @@ -627,7 +629,9 @@ static void printUsage( const char *name, bool devhelp ) { " Machine class (valid options are storage, transaction,\n" " resolution, proxy, master, test, unset, stateless, log, router,\n" " and cluster_controller).\n"); +#ifndef __TLS_DISABLED__ printf(TLS_HELP); +#endif printf(" -v, --version Print version information and exit.\n"); printf(" -h, -?, --help Display this help and exit.\n"); if( devhelp ) { @@ -813,7 +817,7 @@ int main(int argc, char* argv[]) { //Enables profiling on this thread (but does not start it) registerThreadForProfiling(); - + std::string commandLine; for (int a = 0; aset_plugin_name_or_path( plugin_path ? plugin_path : "" ); - } catch (Error& e) { - fprintf(stderr, "ERROR: cannot load TLS plugin `%s' (%s)\n", args.OptionArg(), e.what()); - printHelpTeaser(argv[0]); - flushAndExit(FDB_EXIT_ERROR); - } break; case TLSOptions::OPT_TLS_CERTIFICATES: tlsCertPath = args.OptionArg(); @@ -1216,6 +1213,7 @@ int main(int argc, char* argv[]) { case TLSOptions::OPT_TLS_VERIFY_PEERS: tlsVerifyPeers.push_back(args.OptionArg()); break; +#endif } } @@ -1463,7 +1461,7 @@ int main(int argc, char* argv[]) { // Initialize the thread pool CoroThreadPool::init(); // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now. - TraceEvent::setNetworkThread(); + TraceEvent::setNetworkThread(); if (role == Simulation || role == CreateTemplateDatabase) { //startOldSimulator(); @@ -1475,6 +1473,7 @@ int main(int argc, char* argv[]) { openTraceFile(publicAddress, rollsize, maxLogsSize, logFolder, "trace", logGroup); +#ifndef __TLS_DISABLED__ if ( tlsCertPath.size() ) tlsOptions->set_cert_file( tlsCertPath ); if (tlsCAPath.size()) @@ -1489,7 +1488,7 @@ int main(int argc, char* argv[]) { tlsOptions->set_verify_peers( tlsVerifyPeers ); tlsOptions->register_network(); - +#endif if (role == FDBD || role == NetworkTestServer) { try { listenError = FlowTransport::transport().bind(publicAddress, listenAddress); diff --git a/fdbserver/local.mk b/fdbserver/local.mk index 78cad1bf874..45105be9ce7 100644 --- a/fdbserver/local.mk +++ b/fdbserver/local.mk @@ -24,6 +24,10 @@ fdbserver_CFLAGS := $(fdbclient_CFLAGS) fdbserver_LDFLAGS := $(fdbrpc_LDFLAGS) fdbserver_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a +ifndef __TLS_DISABLED__ +fdbserver_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a +endif + ifeq ($(PLATFORM),linux) fdbserver_LIBS += -ldl -lpthread -lrt fdbserver_LDFLAGS += -static-libstdc++ -static-libgcc From df1e3770b1d4f779d3de6393ce2a2085d51a2e06 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Mon, 25 Jun 2018 14:11:51 -0700 Subject: [PATCH 2/9] Fixed problem with adding xls libraries to linker --- fdbcli/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbcli/local.mk b/fdbcli/local.mk index 79cce6f2e2e..2da2d4899bb 100644 --- a/fdbcli/local.mk +++ b/fdbcli/local.mk @@ -26,7 +26,7 @@ fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ldl fdbcli_STATIC_LIBS := ifndef __TLS_DISABLED__ -fdb_cli_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a +fdbcli_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a endif fdbcli_GENERATED_SOURCES += versions.h From ef8de426d31a5841d088c7981cd51631c4dd1896 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 26 Jun 2018 12:08:32 -0700 Subject: [PATCH 3/9] Changed the TLS_DISABLED macro Disable TLS within Windows until working --- bindings/c/fdb_c.vcxproj | 6 +++--- bindings/c/local.mk | 2 +- bindings/flow/fdb_flow.vcxproj | 6 +++--- bindings/flow/tester/fdb_flow_tester.vcxproj | 6 +++--- bindings/java/fdb_java.vcxproj | 6 +++--- build/link-wrapper.sh | 2 ++ build/scver.mk | 6 ++++++ fdbbackup/fdbbackup.vcxproj | 4 ++-- fdbbackup/local.mk | 2 +- fdbcli/fdbcli.actor.cpp | 6 +++--- fdbcli/fdbcli.vcxproj | 6 +++--- fdbcli/local.mk | 2 +- fdbclient/NativeAPI.actor.cpp | 2 +- fdbclient/fdbclient.vcxproj | 4 ++-- fdbrpc/LoadPlugin.h | 2 +- fdbrpc/TLSConnection.h | 2 +- fdbrpc/fdbrpc.vcxproj | 4 ++-- fdbrpc/local.mk | 4 ++-- fdbrpc/md5/md5.h | 7 ++++--- fdbserver/fdbserver.actor.cpp | 8 ++++---- fdbserver/fdbserver.vcxproj | 6 +++--- fdbserver/local.mk | 2 +- flow/Platform.h | 11 +++++++++++ flow/flow.vcxproj | 4 ++-- 24 files changed, 65 insertions(+), 45 deletions(-) diff --git a/bindings/c/fdb_c.vcxproj b/bindings/c/fdb_c.vcxproj index 14af02f63d3..76d305988dc 100644 --- a/bindings/c/fdb_c.vcxproj +++ b/bindings/c/fdb_c.vcxproj @@ -80,7 +80,7 @@ FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath) Level3 Disabled - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDebug /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) @@ -95,7 +95,7 @@ FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath) MaxSpeed true true - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) @@ -121,4 +121,4 @@ FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath) - \ No newline at end of file + diff --git a/bindings/c/local.mk b/bindings/c/local.mk index a87aad98821..ef40e487ff4 100644 --- a/bindings/c/local.mk +++ b/bindings/c/local.mk @@ -26,7 +26,7 @@ fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a fdb_c_tests_LIBS := -Llib -lfdb_c fdb_c_tests_HEADERS := -Ibindings/c -ifndef __TLS_DISABLED__ +ifndef TLS_DISABLED fdb_c_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a endif diff --git a/bindings/flow/fdb_flow.vcxproj b/bindings/flow/fdb_flow.vcxproj index 2fce553f590..ed5a85216c5 100755 --- a/bindings/flow/fdb_flow.vcxproj +++ b/bindings/flow/fdb_flow.vcxproj @@ -95,7 +95,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories);..\c true /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) @@ -118,7 +118,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories);..\c NotSet false @@ -147,4 +147,4 @@ - \ No newline at end of file + diff --git a/bindings/flow/tester/fdb_flow_tester.vcxproj b/bindings/flow/tester/fdb_flow_tester.vcxproj index ce8fe46ea8e..7eed24fdbdc 100644 --- a/bindings/flow/tester/fdb_flow_tester.vcxproj +++ b/bindings/flow/tester/fdb_flow_tester.vcxproj @@ -77,7 +77,7 @@ Level3 Disabled - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true false MultiThreadedDebug @@ -100,7 +100,7 @@ Full true - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true Speed MultiThreaded @@ -125,4 +125,4 @@ - \ No newline at end of file + diff --git a/bindings/java/fdb_java.vcxproj b/bindings/java/fdb_java.vcxproj index 839dd166108..4ffa3959717 100644 --- a/bindings/java/fdb_java.vcxproj +++ b/bindings/java/fdb_java.vcxproj @@ -60,7 +60,7 @@ Level3 Disabled %(AdditionalIncludeDirectories);$(SolutionDir)bindings\c - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDebug /bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions) @@ -75,7 +75,7 @@ true true %(AdditionalIncludeDirectories);$(SolutionDir)bindings\c - WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded /bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions) @@ -99,4 +99,4 @@ - \ No newline at end of file + diff --git a/build/link-wrapper.sh b/build/link-wrapper.sh index c34aac99672..1cc6bc2d38d 100755 --- a/build/link-wrapper.sh +++ b/build/link-wrapper.sh @@ -19,6 +19,7 @@ case $1 in fi OPTIONS=$( eval echo "$OPTIONS $LDFLAGS \$$2_LDFLAGS \$$2_OBJECTS \$$2_LIBS \$$2_STATIC_LIBS_REAL -o $3" ) + echo "OPTIONS: $OPTIONS" if echo $OPTIONS | grep -q -- -static-libstdc\+\+ ; then OPTIONS=$( echo $OPTIONS | sed -e s,-static-libstdc\+\+,, -e s,\$,\ `$CC -print-file-name=libstdc++.a`\ -lm, ) @@ -33,6 +34,7 @@ case $1 in fi ;; *) + echo "Command: $CC $OPTIONS" $CC $OPTIONS ;; esac diff --git a/build/scver.mk b/build/scver.mk index c20d7d59218..c35143423e9 100644 --- a/build/scver.mk +++ b/build/scver.mk @@ -159,4 +159,10 @@ info: @echo "User: ($(USERID)) $(USER)" @echo "Java Version: ($(JAVAVERMAJOR).$(JAVAVERMINOR)) $(JAVAVER)" @echo "Platform: $(PLATFORM)" +ifdef TLS_DISABLED + @echo "TLS: Disabled" +else + @echo "TLS: Enabled" +endif + @echo "fdbcli CFLAGS: $(fdbcli_CFLAGS)" @echo "" diff --git a/fdbbackup/fdbbackup.vcxproj b/fdbbackup/fdbbackup.vcxproj index c215b453224..d21279bc67b 100644 --- a/fdbbackup/fdbbackup.vcxproj +++ b/fdbbackup/fdbbackup.vcxproj @@ -78,7 +78,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) true /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) @@ -98,7 +98,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) NotSet false diff --git a/fdbbackup/local.mk b/fdbbackup/local.mk index c4e234162cd..0896bf6c051 100644 --- a/fdbbackup/local.mk +++ b/fdbbackup/local.mk @@ -24,7 +24,7 @@ fdbbackup_CFLAGS := $(fdbclient_CFLAGS) fdbbackup_LDFLAGS := $(fdbrpc_LDFLAGS) fdbbackup_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ifndef __TLS_DISABLED__ +ifndef TLS_DISABLED fdbbackup_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a endif diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index db0e6eb5152..60ed64a0d79 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -72,7 +72,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_VERSION, "--version", SO_NONE }, { OPT_VERSION, "-v", SO_NONE }, -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -402,7 +402,7 @@ static void printProgramUsage(const char* name) { " and then exits.\n" " --no-status Disables the initial status check done when starting\n" " the CLI.\n" -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED TLS_HELP #endif " -v, --version Print FoundationDB CLI version information and exit.\n" @@ -2145,7 +2145,7 @@ struct CLIOptions { initialStatusCheck = false; break; -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED // TLS Options case TLSOptions::OPT_TLS_PLUGIN: break; diff --git a/fdbcli/fdbcli.vcxproj b/fdbcli/fdbcli.vcxproj index fb3302cfdd7..a76ce05ac52 100644 --- a/fdbcli/fdbcli.vcxproj +++ b/fdbcli/fdbcli.vcxproj @@ -81,7 +81,7 @@ Level3 Disabled - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\zookeeper\win32;..\zookeeper\generated;..\zookeeper\include;%(AdditionalIncludeDirectories) true false @@ -105,7 +105,7 @@ Full true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\zookeeper\win32;..\zookeeper\generated;..\zookeeper\include;%(AdditionalIncludeDirectories) true Speed @@ -131,4 +131,4 @@ - \ No newline at end of file + diff --git a/fdbcli/local.mk b/fdbcli/local.mk index 2da2d4899bb..4bbfe35941e 100644 --- a/fdbcli/local.mk +++ b/fdbcli/local.mk @@ -25,7 +25,7 @@ fdbcli_LDFLAGS := $(fdbrpc_LDFLAGS) fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ldl fdbcli_STATIC_LIBS := -ifndef __TLS_DISABLED__ +ifndef TLS_DISABLED fdbcli_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a endif diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index e6da4adab9b..44733fae03c 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -870,7 +870,7 @@ void setupNetwork(uint64_t transportId, bool useMetrics) { FlowTransport::createInstance(transportId); Net2FileSystem::newFileSystem(); -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED tlsOptions->register_network(); #endif } diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj index 769c4f46ac8..ceae4f38c3f 100755 --- a/fdbclient/fdbclient.vcxproj +++ b/fdbclient/fdbclient.vcxproj @@ -155,7 +155,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) true /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) @@ -178,7 +178,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) NotSet false diff --git a/fdbrpc/LoadPlugin.h b/fdbrpc/LoadPlugin.h index d64dbc4ca10..bd06e30f8ba 100644 --- a/fdbrpc/LoadPlugin.h +++ b/fdbrpc/LoadPlugin.h @@ -25,7 +25,7 @@ extern "C" void *get_plugin(const char *plugin_type_name_and_version); template Reference loadPlugin( std::string const& plugin_name ) { -#ifdef __TLS_DISABLED__ +#ifdef TLS_DISABLED return Reference( NULL ); #else return (plugin_name.empty()) ? Reference( NULL ) : Reference( (T*)get_plugin( T::get_plugin_type_name_and_version() ) ); diff --git a/fdbrpc/TLSConnection.h b/fdbrpc/TLSConnection.h index 0f575f8b9a3..3aadb6865f6 100644 --- a/fdbrpc/TLSConnection.h +++ b/fdbrpc/TLSConnection.h @@ -83,7 +83,7 @@ struct TLSOptions : ReferenceCounted { enum { OPT_TLS = 100000, OPT_TLS_PLUGIN, OPT_TLS_CERTIFICATES, OPT_TLS_KEY, OPT_TLS_VERIFY_PEERS, OPT_TLS_CA_FILE, OPT_TLS_PASSWORD }; enum PolicyType { POLICY_VERIFY_PEERS = 1, POLICY_NO_VERIFY_PEERS }; TLSOptions() : certs_set(false), key_set(false), verify_peers_set(false), ca_set(false) { -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED init_plugin( ); #endif } diff --git a/fdbrpc/fdbrpc.vcxproj b/fdbrpc/fdbrpc.vcxproj index 8e392121211..5baed2e14af 100644 --- a/fdbrpc/fdbrpc.vcxproj +++ b/fdbrpc/fdbrpc.vcxproj @@ -198,7 +198,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) true /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) @@ -222,7 +222,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;FDB_CLEAN_BUILD;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;FDB_CLEAN_BUILD;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) NotSet false diff --git a/fdbrpc/local.mk b/fdbrpc/local.mk index eb26c0874aa..11f35aa436a 100644 --- a/fdbrpc/local.mk +++ b/fdbrpc/local.mk @@ -25,8 +25,8 @@ fdbrpc_BUILD_SOURCES += fdbrpc/libeio/eio.c fdbrpc_CFLAGS := -I$(BOOSTDIR) -I. -Ifdbrpc -Ifdbrpc/libeio -DUSE_UCONTEXT fdbrpc_LDFLAGS := -ifdef __TLS_DISABLED__ -fdbrpc_CFLAGS += -D__TLS_DISABLED__ +ifdef TLS_DISABLED +fdbrpc_CFLAGS += -DTLS_DISABLED endif ifeq ($(PLATFORM),osx) diff --git a/fdbrpc/md5/md5.h b/fdbrpc/md5/md5.h index 882f021a480..e73fb29c35c 100644 --- a/fdbrpc/md5/md5.h +++ b/fdbrpc/md5/md5.h @@ -27,6 +27,7 @@ #include #elif !defined(_MD5_H) #define _MD5_H +#include "flow/Platform.h" /* Any 32-bit or wider unsigned integer data type will do */ typedef unsigned int MD5_u32plus; @@ -41,9 +42,9 @@ typedef struct { #ifdef __cplusplus extern "C" { #endif - void __attribute__((weak)) MD5_Init(MD5_CTX *ctx); - void __attribute__((weak)) MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); - void __attribute__((weak)) MD5_Final(unsigned char *result, MD5_CTX *ctx); + extern void MULTIPLY_DEFINED_SYMBOL MD5_Init(MD5_CTX *ctx); + extern void MULTIPLY_DEFINED_SYMBOL MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); + extern void MULTIPLY_DEFINED_SYMBOL MD5_Final(unsigned char *result, MD5_CTX *ctx); #ifdef __cplusplus } #endif diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 005f957d951..85367bd5c5e 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -151,7 +151,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_IO_TRUST_SECONDS, "--io_trust_seconds", SO_REQ_SEP }, { OPT_IO_TRUST_WARN_ONLY, "--io_trust_warn_only", SO_NONE }, -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -629,7 +629,7 @@ static void printUsage( const char *name, bool devhelp ) { " Machine class (valid options are storage, transaction,\n" " resolution, proxy, master, test, unset, stateless, log, router,\n" " and cluster_controller).\n"); -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED printf(TLS_HELP); #endif printf(" -v, --version Print version information and exit.\n"); @@ -1195,7 +1195,7 @@ int main(int argc, char* argv[]) { case OPT_IO_TRUST_WARN_ONLY: fileIoWarnOnly = true; break; -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED case TLSOptions::OPT_TLS_PLUGIN: break; case TLSOptions::OPT_TLS_CERTIFICATES: @@ -1473,7 +1473,7 @@ int main(int argc, char* argv[]) { openTraceFile(publicAddress, rollsize, maxLogsSize, logFolder, "trace", logGroup); -#ifndef __TLS_DISABLED__ +#ifndef TLS_DISABLED if ( tlsCertPath.size() ) tlsOptions->set_cert_file( tlsCertPath ); if (tlsCAPath.size()) diff --git a/fdbserver/fdbserver.vcxproj b/fdbserver/fdbserver.vcxproj index 376ab3cf99c..6160eef3653 100644 --- a/fdbserver/fdbserver.vcxproj +++ b/fdbserver/fdbserver.vcxproj @@ -252,7 +252,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) true /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) @@ -273,7 +273,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) NotSet false @@ -299,4 +299,4 @@ - \ No newline at end of file + diff --git a/fdbserver/local.mk b/fdbserver/local.mk index 45105be9ce7..d4235895585 100644 --- a/fdbserver/local.mk +++ b/fdbserver/local.mk @@ -24,7 +24,7 @@ fdbserver_CFLAGS := $(fdbclient_CFLAGS) fdbserver_LDFLAGS := $(fdbrpc_LDFLAGS) fdbserver_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ifndef __TLS_DISABLED__ +ifndef TLS_DISABLED fdbserver_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a endif diff --git a/flow/Platform.h b/flow/Platform.h index 938e3602adc..19ea128484e 100644 --- a/flow/Platform.h +++ b/flow/Platform.h @@ -519,6 +519,17 @@ void* loadFunction(void* lib, const char* func_name); #define EXTERNC #endif // __cplusplus +/* + * Multiply Defined Symbol (support for weak function declaration). + */ +#ifndef MULTIPLY_DEFINED_SYMBOL +#if defined(_MSC_VER) +#define MULTIPLY_DEFINED_SYMBOL +#else +#define MULTIPLY_DEFINED_SYMBOL __attribute__((weak)) +#endif +#endif + // Logs a critical error message and exits the program EXTERNC void criticalError(int exitCode, const char *type, const char *message); EXTERNC void flushAndExit(int exitCode); diff --git a/flow/flow.vcxproj b/flow/flow.vcxproj index 5d1b0d88814..00bceb0e25b 100644 --- a/flow/flow.vcxproj +++ b/flow/flow.vcxproj @@ -148,7 +148,7 @@ Disabled EnableFastChecks MultiThreadedDebug - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) true /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) @@ -171,7 +171,7 @@ Full MultiThreaded true - WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;FDB_CLEAN_BUILD;%(PreprocessorDefinitions) + TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;FDB_CLEAN_BUILD;%(PreprocessorDefinitions) %(AdditionalIncludeDirectories) NotSet false From dd967bd9e2418471e2bb4bff58f72b5dfb400bf4 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 26 Jun 2018 14:44:21 -0700 Subject: [PATCH 4/9] Removed developmental debug messages --- build/link-wrapper.sh | 4 +--- fdbcli/fdbcli.actor.cpp | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/build/link-wrapper.sh b/build/link-wrapper.sh index 1cc6bc2d38d..6d8b5058c60 100755 --- a/build/link-wrapper.sh +++ b/build/link-wrapper.sh @@ -19,7 +19,6 @@ case $1 in fi OPTIONS=$( eval echo "$OPTIONS $LDFLAGS \$$2_LDFLAGS \$$2_OBJECTS \$$2_LIBS \$$2_STATIC_LIBS_REAL -o $3" ) - echo "OPTIONS: $OPTIONS" if echo $OPTIONS | grep -q -- -static-libstdc\+\+ ; then OPTIONS=$( echo $OPTIONS | sed -e s,-static-libstdc\+\+,, -e s,\$,\ `$CC -print-file-name=libstdc++.a`\ -lm, ) @@ -34,8 +33,7 @@ case $1 in fi ;; *) - echo "Command: $CC $OPTIONS" - $CC $OPTIONS +\ $CC $OPTIONS ;; esac diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 60ed64a0d79..94174f10822 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -2224,9 +2224,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { } // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now. - printf("Setting network thread\n"); TraceEvent::setNetworkThread(); - printf("Set network thread\n"); try { cluster = Cluster::createCluster(ccf->getFilename().c_str(), -1); From 65d8b38ae9d24eb3949fcda09f1bcb3a1dff3df6 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Tue, 26 Jun 2018 16:01:25 -0700 Subject: [PATCH 5/9] Changed generic plugin code to work as expected plugin code except for TLS use case Defined TLS plugin name constant Changed TLS plugin name to get_tls_plugin Fixed link script Removed compilation flags from info make target --- FDBLibTLS/FDBLibTLSPlugin.cpp | 2 +- build/link-wrapper.sh | 2 +- build/scver.mk | 1 - fdbrpc/LoadPlugin.h | 23 +++++++++++++++++------ fdbrpc/TLSConnection.actor.cpp | 10 ++++++---- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/FDBLibTLS/FDBLibTLSPlugin.cpp b/FDBLibTLS/FDBLibTLSPlugin.cpp index 712fd4a66f3..41ed49f523c 100644 --- a/FDBLibTLS/FDBLibTLSPlugin.cpp +++ b/FDBLibTLS/FDBLibTLSPlugin.cpp @@ -42,7 +42,7 @@ ITLSPolicy *FDBLibTLSPlugin::create_policy(ITLSLogFunc logf) { return new FDBLibTLSPolicy(Reference::addRef(this), logf); } -extern "C" BOOST_SYMBOL_EXPORT void *get_plugin(const char *plugin_type_name_and_version) { +extern "C" BOOST_SYMBOL_EXPORT void *get_tls_plugin(const char *plugin_type_name_and_version) { if (strcmp(plugin_type_name_and_version, FDBLibTLSPlugin::get_plugin_type_name_and_version()) == 0) { return new FDBLibTLSPlugin; } diff --git a/build/link-wrapper.sh b/build/link-wrapper.sh index 6d8b5058c60..c34aac99672 100755 --- a/build/link-wrapper.sh +++ b/build/link-wrapper.sh @@ -33,7 +33,7 @@ case $1 in fi ;; *) -\ $CC $OPTIONS + $CC $OPTIONS ;; esac diff --git a/build/scver.mk b/build/scver.mk index c35143423e9..ce8cc7305d2 100644 --- a/build/scver.mk +++ b/build/scver.mk @@ -164,5 +164,4 @@ ifdef TLS_DISABLED else @echo "TLS: Enabled" endif - @echo "fdbcli CFLAGS: $(fdbcli_CFLAGS)" @echo "" diff --git a/fdbrpc/LoadPlugin.h b/fdbrpc/LoadPlugin.h index bd06e30f8ba..a74cc4f9b70 100644 --- a/fdbrpc/LoadPlugin.h +++ b/fdbrpc/LoadPlugin.h @@ -20,14 +20,25 @@ #pragma once -// Will need to be defined at link time -extern "C" void *get_plugin(const char *plugin_type_name_and_version); +// Specialized TLS plugin library +extern "C" void *get_tls_plugin(const char *plugin_type_name_and_version); + +// Name of specialized TLS Plugin +extern const char* tlsPluginName; template Reference loadPlugin( std::string const& plugin_name ) { -#ifdef TLS_DISABLED - return Reference( NULL ); -#else - return (plugin_name.empty()) ? Reference( NULL ) : Reference( (T*)get_plugin( T::get_plugin_type_name_and_version() ) ); + void *(*get_plugin)(const char*) = NULL; +#ifndef TLS_DISABLED + if (!plugin_name.compare(tlsPluginName)) { + get_plugin = (void*(*)(const char*)) get_tls_plugin; + } + else #endif + { + void* plugin = loadLibrary( plugin_name.c_str() ); + if (plugin) + get_plugin = (void*(*)(const char*))loadFunction( plugin, "get_plugin" ); + } + return (get_plugin) ? Reference( (T*)get_plugin( T::get_plugin_type_name_and_version() ) ) : Reference( NULL ); } diff --git a/fdbrpc/TLSConnection.actor.cpp b/fdbrpc/TLSConnection.actor.cpp index adbe4249906..514a166429d 100644 --- a/fdbrpc/TLSConnection.actor.cpp +++ b/fdbrpc/TLSConnection.actor.cpp @@ -28,6 +28,9 @@ #include "Platform.h" #include +// Name of specialized TLS Plugin +const char* tlsPluginName = "fdb-libressl-plugin"; + // Must not throw an exception from this function! static int send_func(void* ctx, const uint8_t* buf, int len) { TLSConnection* conn = (TLSConnection*)ctx; @@ -368,14 +371,13 @@ static void TLSConnectionLogFunc( const char* event, void* uid_ptr, bool is_erro } void TLSOptions::init_plugin() { - std::string plugin_path = platform::getDefaultPluginPath("fdb-libressl-plugin"); - TraceEvent("TLSConnectionLoadingPlugin").detail("PluginPath", plugin_path); + TraceEvent("TLSConnectionLoadingPlugin").detail("Plugin", tlsPluginName); - plugin = loadPlugin( plugin_path.c_str() ); + plugin = loadPlugin( tlsPluginName ); if ( !plugin ) { - TraceEvent(SevError, "TLSConnectionPluginInitError").detail("Plugin", plugin_path).GetLastError(); + TraceEvent(SevError, "TLSConnectionPluginInitError").detail("Plugin", tlsPluginName).GetLastError(); throw tls_error(); } From c6e18f5b2b4f04ffd2246252a5198e033316a905 Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Wed, 27 Jun 2018 07:46:04 -0700 Subject: [PATCH 6/9] Removed TLS target Changed TLS documentation to not specify it as a plugin but compiled in --- build/packages.mk | 4 +-- documentation/sphinx/source/tls.rst | 42 ++++++++++------------------- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/build/packages.mk b/build/packages.mk index 425fd095267..780a5ef5434 100644 --- a/build/packages.mk +++ b/build/packages.mk @@ -120,9 +120,7 @@ ifeq ($(PLATFORM),linux) @mkdir -p packages @rm -f packages/foundationdb-server-*.rpm packages/foundationdb-clients-*.rpm @packaging/rpm/buildrpms.sh $(VERSION) $(PKGRELEASE) - - FDBTLS: bin/fdb-libressl-plugin.$(DLEXT) - + packages: DEB RPM JAVA_RELEASE FDBTESTS FDBHEADERS FDBTLS packages_clean: DEB_clean RPM_clean JAVA_RELEASE_clean FDBHEADERS_clean diff --git a/documentation/sphinx/source/tls.rst b/documentation/sphinx/source/tls.rst index 1c07bc95653..786458996a1 100644 --- a/documentation/sphinx/source/tls.rst +++ b/documentation/sphinx/source/tls.rst @@ -9,7 +9,7 @@ Introduction Transport Layer Security (TLS) and its predecessor, Secure Sockets Layer (SSL), are protocols designed to provide communication security over public networks. Users exchange a symmetric session key that is used to encrypt data exchanged between the parties. -By default, a FoundationDB cluster uses *unencrypted* connections among client and server processes. This document describes the `Transport Layer Security `_ (TLS) capabilities of FoundationDB, which enable security and authentication through a public/private key infrastructure. TLS is provided in FoundationDB via a plugin-based architecture. This document will describe the basic TLS capabilities of FoundationDB and document the default plugin, which is based on `LibreSSL `_. TLS-enabled servers will only communicate with other TLS-enabled servers and TLS-enabled clients. Therefore, a cluster's machines must all enable TLS in order for TLS to be used. +By default, a FoundationDB cluster uses *unencrypted* connections among client and server processes. This document describes the `Transport Layer Security `_ (TLS) capabilities of FoundationDB, which enable security and authentication through a public/private key infrastructure. TLS is available within each FoundationDB binary. This document will describe the basic TLS capabilities of FoundationDB and document its implementation, which is based on `LibreSSL `_. TLS-enabled servers will only communicate with other TLS-enabled servers and TLS-enabled clients. Therefore, a cluster's machines must all enable TLS in order for TLS to be used. Setting Up FoundationDB to use TLS @@ -42,23 +42,22 @@ Enabling TLS on an existing (non-TLS) cluster cannot be accomplished without dow 3) Restart the cluster and the clients. -.. _configuring-tls-plugin: +.. _configuring-tls: -Configuring the TLS Plugin +Configuring TLS ========================== -The location and operation of the TLS plugin are configured through four settings. These settings can be provided as command-line options, client options, or environment variables, and are named as follows: +The operation of TLS is configured through five settings. These settings can be provided as command-line options, client options, or environment variables, and are named as follows: ======================== ==================== ============================ ================================================== Command-line Option Client Option Environment Variable Purpose ======================== ==================== ============================ ================================================== -``tls_plugin`` ``TLS_plugin`` ``FDB_TLS_PLUGIN`` Path to the file to be loaded as the TLS plugin ``tls_certificate_file`` ``TLS_cert_path`` ``FDB_TLS_CERTIFICATE_FILE`` Path to the file from which the local certificates - can be loaded, used by the plugin + can be loaded ``tls_key_file`` ``TLS_key_path`` ``FDB_TLS_KEY_FILE`` Path to the file from which to load the private - key, used by the plugin + key ``tls_verify_peers`` ``TLS_verify_peers`` ``FDB_TLS_VERIFY_PEERS`` The byte-string for the verification of peer - certificates and sessions, used by the plugin + certificates and sessions ``tls_password`` ``TLS_password`` ``FDB_TLS_PASSWORD`` The byte-string representing the passcode for unencrypting the private key ``tls_ca_file`` ``TLS_ca_path`` ``FDB_TLS_CA_FILE`` Path to the file containing the CA certificates @@ -73,22 +72,11 @@ The value for each setting can be specified in more than one way. The actual va As with all other command-line options to ``fdbserver``, the TLS settings can be specified in the :ref:`[fdbserver] section of the configuration file `. -The settings for certificate file, key file, peer verification, password and CA file are interpreted by the loaded plugin. +The settings for certificate file, key file, peer verification, password and CA file are interpreted by the TLS. Default Values -------------- -Plugin default location -^^^^^^^^^^^^^^^^^^^^^^^ - -Similarly, if a value is not specified for the parameter ``tls_plugin``, the file will be specified by the environment variable ``FDB_TLS_PLUGIN`` or, if this variable is not set, the system-dependent location: - - * Linux: ``/usr/lib/foundationdb/plugins/fdb-libressl-plugin.so`` - * macOS: ``/usr/local/foundationdb/plugins/fdb-libressl-plugin.dylib`` - * Windows: ``C:\Program Files\foundationdb\plugins\fdb-libressl-plugin.dll`` - -On Windows, this location will be relative to the chosen installation location. The environment variable ``FOUNDATIONDB_INSTALL_PATH`` will be used in place of ``C:\Program Files\foundationdb\`` to determine this location. - Certificate file default location ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,31 +94,29 @@ The default peer verification is ``Check.Valid=0``. Default Password ^^^^^^^^^^^^^^^^^^^^^^^^^ -There is no default password. If no password is specified, the plugin assumes that private key is unencrypted. +There is no default password. If no password is specified, TLS assumes that private key is unencrypted. CA file default location ^^^^^^^^^^^^^^^^^^^^^^^^^ -If a value is not specified, the plugin searches for certs in the default openssl certs location. +If a value is not specified, TLS searches for certs in the default openssl certs location. Parameters and client bindings ------------------------------ -When loading a TLS plugin from a non-default location when using a client binding, the ``TLS_PLUGIN`` network option must be specified before any other TLS option. Because a loaded TLS plugin is allowed to reject the values specified in the other options, the plugin load operation will be forced by specifying one of the other options, if it not already specified. - -The default LibreSSL-based plugin +The default LibreSSL-based implementation ================================= -FoundationDB offers a TLS plugin based on the LibreSSL library. By default, it will be loaded automatically when participating in a TLS-enabled cluster. +FoundationDB offers TLS based on the LibreSSL library. By default, it will be loaded automatically when participating in a TLS-enabled cluster. -For the plugin to operate, each process (both server and client) must have an X509 certificate, its corresponding private key, and potentially the certificates with which is was signed. When a process begins to communicate with a FoundationDB server process, the peer's certificate is checked to see if it is trusted and the fields of the peer certificate are verified. Peers must share the same root trusted certificate, and they must both present certificates whose signing chain includes this root certificate. +For TLS to operate, each process (both server and client) must have an X509 certificate, its corresponding private key, and potentially the certificates with which is was signed. When a process begins to communicate with a FoundationDB server process, the peer's certificate is checked to see if it is trusted and the fields of the peer certificate are verified. Peers must share the same root trusted certificate, and they must both present certificates whose signing chain includes this root certificate. If the local certificate and chain is invalid, a FoundationDB server process bound to a TLS address will not start. In the case of invalid certificates on a client, the client will be able to start but will be unable to connect any TLS-enabled cluster. Formats ------- -The LibreSSL plugin can read certificates and their private keys in base64-encoded DER-formatted X.509 format (which is known as PEM). A PEM file can contain both certificates and a private key or the two can be stored in separate files. +The LibreSSL TLS can read certificates and their private keys in base64-encoded DER-formatted X.509 format (which is known as PEM). A PEM file can contain both certificates and a private key or the two can be stored in separate files. Required files -------------- From f4937b261c1c271b5cfb4f691b4f00bb0ad045ee Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Wed, 27 Jun 2018 07:50:26 -0700 Subject: [PATCH 7/9] Removed TLS target from packages --- build/packages.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/packages.mk b/build/packages.mk index 780a5ef5434..36f10dcc9d2 100644 --- a/build/packages.mk +++ b/build/packages.mk @@ -120,8 +120,8 @@ ifeq ($(PLATFORM),linux) @mkdir -p packages @rm -f packages/foundationdb-server-*.rpm packages/foundationdb-clients-*.rpm @packaging/rpm/buildrpms.sh $(VERSION) $(PKGRELEASE) - - packages: DEB RPM JAVA_RELEASE FDBTESTS FDBHEADERS FDBTLS + + packages: DEB RPM JAVA_RELEASE FDBTESTS FDBHEADERS packages_clean: DEB_clean RPM_clean JAVA_RELEASE_clean FDBHEADERS_clean From 45849d1f9523440fa2dbb3088efd6af44854a3fd Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Wed, 27 Jun 2018 09:25:05 -0700 Subject: [PATCH 8/9] Added support for no-op legacy TLS options --- fdbcli/fdbcli.actor.cpp | 1 + fdbrpc/TLSConnection.h | 2 ++ fdbserver/fdbserver.actor.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 94174f10822..00f6785f6bb 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -2148,6 +2148,7 @@ struct CLIOptions { #ifndef TLS_DISABLED // TLS Options case TLSOptions::OPT_TLS_PLUGIN: + args.OptionArg(); break; case TLSOptions::OPT_TLS_CERTIFICATES: tlsCertPath = args.OptionArg(); diff --git a/fdbrpc/TLSConnection.h b/fdbrpc/TLSConnection.h index 3aadb6865f6..08ac201d689 100644 --- a/fdbrpc/TLSConnection.h +++ b/fdbrpc/TLSConnection.h @@ -127,6 +127,7 @@ struct TLSNetworkConnections : INetworkConnections { Reference options; }; +#define TLS_PLUGIN_FLAG "--tls_plugin" #define TLS_CERTIFICATE_FILE_FLAG "--tls_certificate_file" #define TLS_KEY_FILE_FLAG "--tls_key_file" #define TLS_VERIFY_PEERS_FLAG "--tls_verify_peers" @@ -134,6 +135,7 @@ struct TLSNetworkConnections : INetworkConnections { #define TLS_PASSWORD_FLAG "--tls_password" #define TLS_OPTION_FLAGS \ + { TLSOptions::OPT_TLS_PLUGIN, TLS_PLUGIN_FLAG, SO_REQ_SEP }, \ { TLSOptions::OPT_TLS_CERTIFICATES, TLS_CERTIFICATE_FILE_FLAG, SO_REQ_SEP }, \ { TLSOptions::OPT_TLS_KEY, TLS_KEY_FILE_FLAG, SO_REQ_SEP }, \ { TLSOptions::OPT_TLS_VERIFY_PEERS, TLS_VERIFY_PEERS_FLAG, SO_REQ_SEP }, \ diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 85367bd5c5e..45a7221b7e7 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -1197,6 +1197,7 @@ int main(int argc, char* argv[]) { break; #ifndef TLS_DISABLED case TLSOptions::OPT_TLS_PLUGIN: + args.OptionArg(); break; case TLSOptions::OPT_TLS_CERTIFICATES: tlsCertPath = args.OptionArg(); From 132e2d92677378800dd9db48f2a0c7a9b25eb7cf Mon Sep 17 00:00:00 2001 From: Alvin Moore Date: Sun, 1 Jul 2018 22:49:39 -0700 Subject: [PATCH 9/9] Defined TLS build flags for projects Updated TLS documentation --- Makefile | 7 +++++++ bindings/c/local.mk | 6 +----- build/Dockerfile | 5 ++++- documentation/sphinx/source/tls.rst | 12 ++++++------ fdbbackup/local.mk | 6 +----- fdbcli/local.mk | 6 +----- fdbrpc/local.mk | 4 ---- fdbserver/local.mk | 6 +----- 8 files changed, 21 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index ab1ee722bd6..5b75368bed7 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,13 @@ CFLAGS += -g # valgrind-compatibile builds are enabled by uncommenting lines in valgind.mk +ifdef TLS_DISABLED +CFLAGS += -DTLS_DISABLED +TLS_LIBS := +else +TLS_LIBS := lib/libFDBLibTLS.a $(shell gcc --print-file-name=libtls.a) $(shell gcc --print-file-name=libssl.a) $(shell gcc --print-file-name=libcrypto.a) +endif + CXXFLAGS += -Wno-deprecated LDFLAGS := LIBS := diff --git a/bindings/c/local.mk b/bindings/c/local.mk index ef40e487ff4..995337fa039 100644 --- a/bindings/c/local.mk +++ b/bindings/c/local.mk @@ -22,14 +22,10 @@ fdb_c_CFLAGS := $(fdbclient_CFLAGS) fdb_c_LDFLAGS := $(fdbrpc_LDFLAGS) -fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a +fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(TLS_LIBS) fdb_c_tests_LIBS := -Llib -lfdb_c fdb_c_tests_HEADERS := -Ibindings/c -ifndef TLS_DISABLED -fdb_c_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a -endif - CLEAN_TARGETS += fdb_c_tests_clean ifeq ($(PLATFORM),linux) diff --git a/build/Dockerfile b/build/Dockerfile index aa14fa2daf4..17c72a72fba 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -22,7 +22,7 @@ RUN cd /opt/ && wget https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.6.4 gpg --import libressl.asc && gpg --verify libressl-2.6.4.tar.gz.asc libressl-2.6.4.tar.gz &&\ tar -xzf libressl-2.6.4.tar.gz && cd libressl-2.6.4 &&\ ./configure CFLAGS="-fPIC -O3" && make -j4 && make install &&\ - cd /opt/ # && rm -r libressl-2.6.4/ libressl-2.6.4.tar.gz libressl-2.6.4.tar.gz.asc libressl.asc + cd /opt/ && rm -r libressl-2.6.4/ libressl-2.6.4.tar.gz libressl-2.6.4.tar.gz.asc libressl.asc RUN LANGUAGE=en_US.UTF-8 LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 locale-gen en_US.UTF-8 @@ -33,3 +33,6 @@ ENV TARGET_LIBC_VERSION=$TARGET_LIBC_VERSION ARG CC=/usr/bin/gcc ENV CC=$CC + +ARG LIBRARY_PATH=/usr/local/lib +ENV LIBRARY_PATH=$LD_FLAGS diff --git a/documentation/sphinx/source/tls.rst b/documentation/sphinx/source/tls.rst index 786458996a1..3d6efcb5384 100644 --- a/documentation/sphinx/source/tls.rst +++ b/documentation/sphinx/source/tls.rst @@ -9,7 +9,7 @@ Introduction Transport Layer Security (TLS) and its predecessor, Secure Sockets Layer (SSL), are protocols designed to provide communication security over public networks. Users exchange a symmetric session key that is used to encrypt data exchanged between the parties. -By default, a FoundationDB cluster uses *unencrypted* connections among client and server processes. This document describes the `Transport Layer Security `_ (TLS) capabilities of FoundationDB, which enable security and authentication through a public/private key infrastructure. TLS is available within each FoundationDB binary. This document will describe the basic TLS capabilities of FoundationDB and document its implementation, which is based on `LibreSSL `_. TLS-enabled servers will only communicate with other TLS-enabled servers and TLS-enabled clients. Therefore, a cluster's machines must all enable TLS in order for TLS to be used. +By default, a FoundationDB cluster uses *unencrypted* connections among client and server processes. This document describes the `Transport Layer Security `_ (TLS) capabilities of FoundationDB, which enable security and authentication through a public/private key infrastructure. TLS is compiled into each FoundationDB binary. This document will describe the basic TLS capabilities of FoundationDB and document its implementation, which is based on `LibreSSL `_. TLS-enabled servers will only communicate with other TLS-enabled servers and TLS-enabled clients. Therefore, a cluster's machines must all enable TLS in order for TLS to be used. Setting Up FoundationDB to use TLS @@ -72,7 +72,7 @@ The value for each setting can be specified in more than one way. The actual va As with all other command-line options to ``fdbserver``, the TLS settings can be specified in the :ref:`[fdbserver] section of the configuration file `. -The settings for certificate file, key file, peer verification, password and CA file are interpreted by the TLS. +The settings for certificate file, key file, peer verification, password and CA file are interpreted by the software. Default Values -------------- @@ -94,12 +94,12 @@ The default peer verification is ``Check.Valid=0``. Default Password ^^^^^^^^^^^^^^^^^^^^^^^^^ -There is no default password. If no password is specified, TLS assumes that private key is unencrypted. +There is no default password. If no password is specified, it is assumed that the private key is unencrypted. CA file default location ^^^^^^^^^^^^^^^^^^^^^^^^^ -If a value is not specified, TLS searches for certs in the default openssl certs location. +If a value is not specified, the software searches for certs in the default openssl certs location. Parameters and client bindings ------------------------------ @@ -107,7 +107,7 @@ Parameters and client bindings The default LibreSSL-based implementation ================================= -FoundationDB offers TLS based on the LibreSSL library. By default, it will be loaded automatically when participating in a TLS-enabled cluster. +FoundationDB offers TLS based on the LibreSSL library. By default, it will be enabled automatically when participating in a TLS-enabled cluster. For TLS to operate, each process (both server and client) must have an X509 certificate, its corresponding private key, and potentially the certificates with which is was signed. When a process begins to communicate with a FoundationDB server process, the peer's certificate is checked to see if it is trusted and the fields of the peer certificate are verified. Peers must share the same root trusted certificate, and they must both present certificates whose signing chain includes this root certificate. @@ -116,7 +116,7 @@ If the local certificate and chain is invalid, a FoundationDB server process bou Formats ------- -The LibreSSL TLS can read certificates and their private keys in base64-encoded DER-formatted X.509 format (which is known as PEM). A PEM file can contain both certificates and a private key or the two can be stored in separate files. +LibreSSL can read certificates and their private keys in base64-encoded DER-formatted X.509 format (which is known as PEM). A PEM file can contain both certificates and a private key or the two can be stored in separate files. Required files -------------- diff --git a/fdbbackup/local.mk b/fdbbackup/local.mk index 0896bf6c051..88198633d61 100644 --- a/fdbbackup/local.mk +++ b/fdbbackup/local.mk @@ -22,11 +22,7 @@ fdbbackup_CFLAGS := $(fdbclient_CFLAGS) fdbbackup_LDFLAGS := $(fdbrpc_LDFLAGS) -fdbbackup_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a - -ifndef TLS_DISABLED -fdbbackup_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a -endif +fdbbackup_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(TLS_LIBS) ifeq ($(PLATFORM),linux) fdbbackup_LIBS += -ldl -lpthread -lrt diff --git a/fdbcli/local.mk b/fdbcli/local.mk index 4bbfe35941e..002e1aab997 100644 --- a/fdbcli/local.mk +++ b/fdbcli/local.mk @@ -22,13 +22,9 @@ fdbcli_CFLAGS := $(fdbclient_CFLAGS) fdbcli_LDFLAGS := $(fdbrpc_LDFLAGS) -fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ldl +fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a -ldl $(TLS_LIBS) fdbcli_STATIC_LIBS := -ifndef TLS_DISABLED -fdbcli_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a -endif - fdbcli_GENERATED_SOURCES += versions.h ifeq ($(PLATFORM),linux) diff --git a/fdbrpc/local.mk b/fdbrpc/local.mk index 11f35aa436a..2e0bd42933f 100644 --- a/fdbrpc/local.mk +++ b/fdbrpc/local.mk @@ -25,10 +25,6 @@ fdbrpc_BUILD_SOURCES += fdbrpc/libeio/eio.c fdbrpc_CFLAGS := -I$(BOOSTDIR) -I. -Ifdbrpc -Ifdbrpc/libeio -DUSE_UCONTEXT fdbrpc_LDFLAGS := -ifdef TLS_DISABLED -fdbrpc_CFLAGS += -DTLS_DISABLED -endif - ifeq ($(PLATFORM),osx) fdbrpc_CFLAGS += -fasynchronous-unwind-tables -fno-omit-frame-pointer fdbrpc_BUILD_SOURCES += fdbrpc/libcoroutine/asm.S fdbrpc/libcoroutine/context.c diff --git a/fdbserver/local.mk b/fdbserver/local.mk index d4235895585..3b8b40321b4 100644 --- a/fdbserver/local.mk +++ b/fdbserver/local.mk @@ -22,11 +22,7 @@ fdbserver_CFLAGS := $(fdbclient_CFLAGS) fdbserver_LDFLAGS := $(fdbrpc_LDFLAGS) -fdbserver_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a - -ifndef TLS_DISABLED -fdbserver_LIBS += lib/libFDBLibTLS.a /usr/local/lib/libtls.a /usr/local/lib/libssl.a /usr/local/lib/libcrypto.a -endif +fdbserver_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(TLS_LIBS) ifeq ($(PLATFORM),linux) fdbserver_LIBS += -ldl -lpthread -lrt