From c391fc81686004bf148a3facf8f2d3bd0a25b764 Mon Sep 17 00:00:00 2001 From: Oknet Xu Date: Thu, 1 Sep 2016 12:56:13 +0800 Subject: [PATCH] TS-4322: Proposal: NetProfileSM --- iocore/eventsystem/I_Event.h | 1 + iocore/eventsystem/I_Thread.h | 2 + iocore/net/I_NetProcessor.h | 2 + iocore/net/I_NetProfileSM.h | 107 ++ iocore/net/I_NetVConnection.h | 69 +- iocore/net/I_SSLM.h | 152 +++ ...xtProtocolSet.h => I_SSLNextProtocolSet.h} | 6 +- iocore/net/Makefile.am | 16 +- .../net/{SSLNetAccept.cc => NetProfileSM.cc} | 32 +- iocore/net/NetVConnection.cc | 69 + iocore/net/P_Net.h | 4 +- iocore/net/P_NetAccept.h | 2 + iocore/net/P_SSLNetAccept.h | 58 - iocore/net/P_SSLNetProcessor.h | 3 - iocore/net/P_SSLNextProtocolAccept.h | 66 - iocore/net/P_SSLProfileSM.h | 230 ++++ iocore/net/P_SSLUtils.h | 17 +- iocore/net/P_UnixNetProcessor.h | 1 + iocore/net/P_UnixNetProfileSM.h | 138 ++ iocore/net/P_UnixNetVConnection.h | 110 +- iocore/net/SSLClientUtils.cc | 19 +- iocore/net/SSLInternal.cc | 2 +- iocore/net/SSLM.cc | 117 ++ iocore/net/SSLNetProcessor.cc | 22 - iocore/net/SSLNextProtocolAccept.cc | 176 --- iocore/net/SSLNextProtocolSet.cc | 2 +- .../{SSLNetVConnection.cc => SSLProfileSM.cc} | 1175 ++++++++--------- iocore/net/SSLUtils.cc | 85 +- iocore/net/UnixNet.cc | 34 +- iocore/net/UnixNetAccept.cc | 15 +- iocore/net/UnixNetProcessor.cc | 12 + iocore/net/UnixNetProfileSM.cc | 547 ++++++++ iocore/net/UnixNetVConnection.cc | 535 +------- lib/ts/apidefs.h.in | 9 +- proxy/InkAPI.cc | 70 +- proxy/InkAPIInternal.h | 2 + proxy/PluginVC.h | 50 + proxy/ProtocolProbeSessionAccept.cc | 44 +- proxy/ProtocolProbeSessionAccept.h | 8 +- proxy/http/HttpDebugNames.cc | 4 + proxy/http/HttpProxyServerMain.cc | 83 +- proxy/http/HttpSM.cc | 46 +- proxy/shared/UglyLogStubs.cc | 6 + 43 files changed, 2433 insertions(+), 1715 deletions(-) create mode 100644 iocore/net/I_NetProfileSM.h create mode 100644 iocore/net/I_SSLM.h rename iocore/net/{P_SSLNextProtocolSet.h => I_SSLNextProtocolSet.h} (94%) rename iocore/net/{SSLNetAccept.cc => NetProfileSM.cc} (66%) delete mode 100644 iocore/net/P_SSLNetAccept.h delete mode 100644 iocore/net/P_SSLNextProtocolAccept.h create mode 100644 iocore/net/P_SSLProfileSM.h create mode 100644 iocore/net/P_UnixNetProfileSM.h create mode 100644 iocore/net/SSLM.cc delete mode 100644 iocore/net/SSLNextProtocolAccept.cc rename iocore/net/{SSLNetVConnection.cc => SSLProfileSM.cc} (57%) create mode 100644 iocore/net/UnixNetProfileSM.cc diff --git a/iocore/eventsystem/I_Event.h b/iocore/eventsystem/I_Event.h index a73eed7a464..76f75f9d4a6 100644 --- a/iocore/eventsystem/I_Event.h +++ b/iocore/eventsystem/I_Event.h @@ -86,6 +86,7 @@ #define BLOCK_CACHE_EVENT_EVENTS_START 4000 #define UTILS_EVENT_EVENTS_START 5000 #define CONGESTION_EVENT_EVENTS_START 5100 +#define IOCORE_EVENTS_START 7000 #define INK_API_EVENT_EVENTS_START 60000 #define SRV_EVENT_EVENTS_START 62000 #define REMAP_EVENT_EVENTS_START 63000 diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 528cdbfd238..cd17b5220d0 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -138,6 +138,8 @@ class Thread ProxyAllocator ioDataAllocator; ProxyAllocator ioAllocator; ProxyAllocator ioBlockAllocator; + ProxyAllocator tcpProfileSMAllocator; + ProxyAllocator sslProfileSMAllocator; private: // prevent unauthorized copies (Not implemented) diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index a6dffadb9d1..35b35bbfdc2 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -95,6 +95,8 @@ class NetProcessor : public Processor */ bool f_inbound_transparent; + bool isSSL; + /// Default constructor. /// Instance is constructed with default values. AcceptOptions() { this->reset(); } diff --git a/iocore/net/I_NetProfileSM.h b/iocore/net/I_NetProfileSM.h new file mode 100644 index 00000000000..2955d931859 --- /dev/null +++ b/iocore/net/I_NetProfileSM.h @@ -0,0 +1,107 @@ +/** @file + A brief file description + + @section license License + + */ + +/******** + + The NetProfileSM is designed to setup a plugable mechanism for NetVConnection + to handle different type of I/O operation. + + ********/ + +#ifndef __I_NET_PROFILESM_H__ +#define __I_NET_PROFILESM_H__ + +#include "ts/ink_sock.h" +#include "I_NetVConnection.h" + +class NetVConnection; + +#define IOCORE_EVENTS_READ (IOCORE_EVENTS_START + 1) +#define IOCORE_EVENTS_WRITE (IOCORE_EVENTS_START + 2) + +typedef enum { + PROFILE_SM_UNDEFINED, + PROFILE_SM_TCP, + PROFILE_SM_UDP, + PROFILE_SM_SSL, + PROFILE_SM_SOCKS, +} NetProfileSMType_t; + +static int DEFAULT = 0; + +class NetProfileSM : public Continuation +{ +public: + NetProfileSM(ProxyMutex *aMutex = NULL) + : Continuation(aMutex), low_profileSM(NULL), vc(NULL), netTrace(false), globally_allocated(false) + { + } + virtual void free(EThread *t) = 0; + void + clear() + { + low_profileSM = NULL; + vc = NULL; + type = PROFILE_SM_UNDEFINED; + mutex = NULL; + netTrace = false; + } + virtual int mainEvent(int event, void *data) = 0; + + // for READ & WRITE + virtual int64_t read(void *buf, int64_t len, int &err = DEFAULT) = 0; + virtual int64_t readv(struct iovec *vector, int count) = 0; + virtual int64_t write(void *buf, int64_t len, int &err = DEFAULT) = 0; + virtual int64_t writev(struct iovec *vector, int count) = 0; + virtual int64_t raw_read(void *buf, int64_t len) = 0; + virtual int64_t raw_readv(struct iovec *vector, int count) = 0; + virtual int64_t raw_write(void *buf, int64_t len) = 0; + virtual int64_t raw_writev(struct iovec *vector, int count) = 0; + virtual int64_t read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf) = 0; + virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) = 0; + + // for netTrace + bool getTrace() const; + + void + setTrace(bool trace) + { + netTrace = trace; + } + + virtual void + set_mutex(ProxyMutex *aMutex) + { + mutex = aMutex; + } + + virtual NetProfileSMType_t + get_type() const + { + return type; + } + + virtual void + reenable() + { + return; + } + + virtual const char *get_protocol_tag() const = 0; + + NetProfileSM *low_profileSM; + NetVConnection *vc; + NetProfileSMType_t type; + bool netTrace; + bool globally_allocated; + +private: + NetProfileSM(const NetProfileSM &); + NetProfileSM &operator=(const NetProfileSM &); +}; + +#endif /* __I_NET_PROFILESM_H__ */ diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index c45526be06e..b6ad4bfaac0 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -33,6 +33,7 @@ #include "I_IOBuffer.h" #include "I_Socks.h" #include +#include "I_NetProfileSM.h" #define CONNECT_SUCCESS 1 #define CONNECT_FAILURE 0 @@ -71,6 +72,8 @@ struct NetVCOptions { USE_UDP ///< UDP protocol. }; + bool isSSL; + /// IP (TCP or UDP) protocol to use on socket. ip_protocol_t ip_proto; @@ -468,6 +471,7 @@ class NetVConnection : public VConnection /** @return current inactivity_timeout value in nanosecs */ virtual ink_hrtime get_inactivity_timeout() = 0; + virtual int getWriteBufferEmpty(); /** Force an @a event if a write operation empties the write buffer. This event will be sent to the VIO, the same place as other IO events. @@ -541,6 +545,31 @@ class NetVConnection : public VConnection unsigned int attributes; EThread *thread; + // es - origin_trace associated connections + bool + getOriginTrace() const + { + return origin_trace; + } + + void + setOriginTraceAddr(const sockaddr *addr = NULL) + { + if (addr) { + origin_trace = true; + origin_trace_addr.assign(addr); + } else { + origin_trace = false; + ink_zero(origin_trace_addr); + } + } + + IpEndpoint const * + getOriginTraceAddr() const + { + return &origin_trace_addr; + } + /// PRIVATE: The public interface is VIO::reenable() virtual void reenable(VIO *vio) = 0; @@ -610,6 +639,29 @@ class NetVConnection : public VConnection return NULL; } + /// Add/Del/Free ProfileSM for NetVConnection. + virtual void add_profile_sm(NetProfileSMType_t sm_type, EThread *t = NULL); + virtual void del_profile_sm(EThread *t); + virtual void free_profile_sm(EThread *t); + + /** + * The following functions are transfered from class UnixNetVConection. + * Here, These functions are declared as pure virtual function for + * - Implement them in UnixNetVConnection or other delivered class of NetVConnection + * - Calling them by NetVConnection interface. + */ + virtual void readDisable() = 0; + virtual void writeDisable() = 0; + virtual void readSignalError(int err) = 0; + virtual void writeSignalError(int err) = 0; + virtual int readSignalDone(int event) = 0; + virtual int writeSignalDone(int event) = 0; + virtual int readSignalAndUpdate(int event) = 0; + virtual int writeSignalAndUpdate(int event) = 0; + virtual void readReschedule() = 0; + virtual void writeReschedule() = 0; + virtual void netActivity(EThread *lthread) = 0; + private: NetVConnection(const NetVConnection &); NetVConnection &operator=(const NetVConnection &); @@ -617,9 +669,11 @@ class NetVConnection : public VConnection protected: IpEndpoint local_addr; IpEndpoint remote_addr; + IpEndpoint origin_trace_addr; bool got_local_addr; bool got_remote_addr; + bool origin_trace; bool is_internal_request; /// Set if this connection is transparent. @@ -628,6 +682,10 @@ class NetVConnection : public VConnection int write_buffer_empty_event; /// NetVConnection Context. NetVConnectionContext_t netvc_context; + +public: + /// Net ProfileSM + NetProfileSM *profile_sm; }; inline NetVConnection::NetVConnection() @@ -636,13 +694,22 @@ inline NetVConnection::NetVConnection() thread(NULL), got_local_addr(0), got_remote_addr(0), + origin_trace(false), is_internal_request(false), is_transparent(false), write_buffer_empty_event(0), - netvc_context(NET_VCONNECTION_UNSET) + netvc_context(NET_VCONNECTION_UNSET), + profile_sm(NULL) { ink_zero(local_addr); ink_zero(remote_addr); + ink_zero(origin_trace_addr); +} + +inline int +NetVConnection::getWriteBufferEmpty() +{ + return write_buffer_empty_event; } inline void diff --git a/iocore/net/I_SSLM.h b/iocore/net/I_SSLM.h new file mode 100644 index 00000000000..6fb993b8df3 --- /dev/null +++ b/iocore/net/I_SSLM.h @@ -0,0 +1,152 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/**************************************************************************** + + I_SSLM.h + reference from P_SSLNetVConnection.h + + This file implements an interface for SSL Module + + ****************************************************************************/ +#ifndef __I_SSLM_H__ +#define __I_SSLM_H__ + +#include "I_SessionAccept.h" +#include "I_SSLNextProtocolSet.h" +#include +#include + +class SSLNextProtocolSet; + +typedef enum { + SSL_HOOK_OP_DEFAULT, ///< Null / initialization value. Do normal processing. + SSL_HOOK_OP_TUNNEL, ///< Switch to blind tunnel + SSL_HOOK_OP_TERMINATE, ///< Termination connection / transaction. + SSL_HOOK_OP_LAST = SSL_HOOK_OP_TERMINATE ///< End marker value. +} SslVConnOp; + +////////////////////////////////////////////////////////////////// +// +// class SSLM +// +// A Interface for SSL Module +// +////////////////////////////////////////////////////////////////// +class SSLM +{ +public: + SSLM(); + virtual ~SSLM(); + + virtual bool + getSSLHandShakeComplete() const + { + return sslHandShakeComplete; + } + + virtual void + setSSLHandShakeComplete(bool state) + { + sslHandShakeComplete = state; + } + + void + setSSLSessionCacheHit(bool state) + { + sslSessionCacheHit = state; + } + + bool + getSSLSessionCacheHit() const + { + return sslSessionCacheHit; + } + + void registerNextProtocolSet(const SSLNextProtocolSet *); + + static int advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned *outlen, void *); + static int select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, + unsigned inlen, void *); + + Continuation * + endpoint() const + { + return npnEndpoint; + } + + bool + getSSLClientRenegotiationAbort() const + { + return sslClientRenegotiationAbort; + } + + void + setSSLClientRenegotiationAbort(bool state) + { + sslClientRenegotiationAbort = state; + } + + bool + getTransparentPassThrough() const + { + return transparentPassThrough; + } + + void + setTransparentPassThrough(bool val) + { + transparentPassThrough = val; + } + + const char * + getSSLProtocol(void) const + { + return ssl ? SSL_get_version(ssl) : NULL; + } + + const char * + getSSLCipherSuite(void) const + { + return ssl ? SSL_get_cipher_name(ssl) : NULL; + } + + void clear(); + + SSL *ssl; + + /// Set by asynchronous hooks to request a specific operation. + SslVConnOp hookOpRequested; + + bool transparentPassThrough; + bool sslHandShakeComplete; + bool sslClientRenegotiationAbort; + bool sslSessionCacheHit; + + const SSLNextProtocolSet *npnSet; + Continuation *npnEndpoint; + SessionAccept *sessionAcceptPtr; + unsigned long error_code; +}; + +#endif /* __I_SSLM_H__ */ diff --git a/iocore/net/P_SSLNextProtocolSet.h b/iocore/net/I_SSLNextProtocolSet.h similarity index 94% rename from iocore/net/P_SSLNextProtocolSet.h rename to iocore/net/I_SSLNextProtocolSet.h index 9a043701e30..e3f6c0d8f28 100644 --- a/iocore/net/P_SSLNextProtocolSet.h +++ b/iocore/net/I_SSLNextProtocolSet.h @@ -21,8 +21,8 @@ limitations under the License. */ -#ifndef P_SSLNextProtocolSet_H_ -#define P_SSLNextProtocolSet_H_ +#ifndef __I_SSLNextProtocolSet_H__ +#define __I_SSLNextProtocolSet_H__ #include "ts/List.h" #include "I_Net.h" @@ -64,4 +64,4 @@ class SSLNextProtocolSet NextProtocolEndpoint::list_type endpoints; }; -#endif /* P_SSLNextProtocolSet_H_ */ +#endif /* __I_SSLNextProtocolSet_H__ */ diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index a49a83cca10..48b747b001e 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -60,6 +60,8 @@ libinknet_a_SOURCES = \ I_UDPPacket.h \ Inline.cc \ I_SessionAccept.h \ + I_NetProfileSM.h \ + I_SSLM.h \ SessionAccept.cc \ Net.cc \ NetVConnection.cc \ @@ -73,13 +75,11 @@ libinknet_a_SOURCES = \ P_Socks.h \ P_SSLCertLookup.h \ P_SSLConfig.h \ - P_SSLNetAccept.h \ P_SSLNetProcessor.h \ - P_SSLNetVConnection.h \ - P_SSLNextProtocolAccept.h \ - P_SSLNextProtocolSet.h \ + I_SSLNextProtocolSet.h \ P_SSLUtils.h \ P_SSLClientUtils.h \ + P_SSLProfileSM.h \ P_OCSPStapling.h \ P_Socks.h \ P_UDPConnection.h \ @@ -91,6 +91,7 @@ libinknet_a_SOURCES = \ P_UnixNetProcessor.h \ P_UnixNetState.h \ P_UnixNetVConnection.h \ + P_UnixNetProfileSM.h \ P_UnixPollDescriptor.h \ P_UnixUDPConnection.h \ Socks.cc \ @@ -98,13 +99,12 @@ libinknet_a_SOURCES = \ SSLSessionCache.cc \ SSLConfig.cc \ SSLInternal.cc \ - SSLNetAccept.cc \ SSLNetProcessor.cc \ - SSLNetVConnection.cc \ - SSLNextProtocolAccept.cc \ SSLNextProtocolSet.cc \ SSLUtils.cc \ SSLClientUtils.cc \ + SSLProfileSM.cc \ + SSLM.cc \ OCSPStapling.cc \ Socks.cc \ UDPIOEvent.cc \ @@ -114,6 +114,8 @@ libinknet_a_SOURCES = \ UnixNetPages.cc \ UnixNetProcessor.cc \ UnixNetVConnection.cc \ + NetProfileSM.cc \ + UnixNetProfileSM.cc \ UnixUDPConnection.cc \ UnixUDPNet.cc \ SSLDynlock.cc diff --git a/iocore/net/SSLNetAccept.cc b/iocore/net/NetProfileSM.cc similarity index 66% rename from iocore/net/SSLNetAccept.cc rename to iocore/net/NetProfileSM.cc index 8d6a5c7fd20..ce136646d64 100644 --- a/iocore/net/SSLNetAccept.cc +++ b/iocore/net/NetProfileSM.cc @@ -1,5 +1,7 @@ /** @file + A brief file description + @section license License Licensed to the Apache Software Foundation (ASF) under one @@ -19,20 +21,26 @@ limitations under the License. */ -#include "ts/ink_config.h" #include "P_Net.h" +#include "I_NetProfileSM.h" -NetProcessor * -SSLNetAccept::getNetProcessor() const +bool +NetProfileSM::getTrace() const { - return &sslNetProcessor; -} + bool isTrace = false; -NetAccept * -SSLNetAccept::clone() const -{ - NetAccept *na; - na = new SSLNetAccept; - *na = *this; - return na; + switch (type) { + case PROFILE_SM_TCP: + isTrace = netTrace || vc->getOriginTrace(); + break; + case PROFILE_SM_SSL: + ink_assert(low_profileSM); + isTrace = netTrace || low_profileSM->getTrace(); + break; + case PROFILE_SM_UDP: + // TODO: + default: + ink_assert(!"not reached!"); + } + return isTrace; } diff --git a/iocore/net/NetVConnection.cc b/iocore/net/NetVConnection.cc index 25a0bd8ce26..6db5ec639d2 100644 --- a/iocore/net/NetVConnection.cc +++ b/iocore/net/NetVConnection.cc @@ -75,3 +75,72 @@ NetVCOptions::get_family_string() const } return retval; } + +// Add ProfileSM for NetVConnection +TS_INLINE void +NetVConnection::add_profile_sm(NetProfileSMType_t sm_type, EThread *t) +{ + switch (sm_type) { + case PROFILE_SM_TCP: + if (TcpProfileSM::check_dep(this->profile_sm)) { + this->profile_sm = TcpProfileSM::allocate(t); + this->profile_sm->vc = this; + this->profile_sm->mutex = mutex; + } else { + ink_assert(!"ProfileSM: check dependency failed!"); + } + break; + case PROFILE_SM_UDP: + // TODO: + break; + case PROFILE_SM_SSL: + if (SSLProfileSM::check_dep(this->profile_sm)) { + NetProfileSM *curr = this->profile_sm; + this->profile_sm = SSLProfileSM::allocate(t); + this->profile_sm->vc = this; + this->profile_sm->mutex = mutex; + this->profile_sm->low_profileSM = curr; + } else { + ink_assert(!"ProfileSM: check dependency failed!"); + } + break; + default: + ink_assert(!"not reached!"); + } +} + +// Delete / Deattach current ProfileSM then free it and restore to Low ProfileSM +TS_INLINE void +NetVConnection::del_profile_sm(EThread *t) +{ + NetProfileSM *tmp; + + switch (profile_sm->type) { + case PROFILE_SM_SSL: + ink_assert(this->profile_sm->low_profileSM != NULL); + tmp = this->profile_sm; + this->profile_sm = tmp->low_profileSM; + tmp->free(t); + break; + case PROFILE_SM_TCP: + ink_assert(!"Can not delete a base ProfileSM!"); + break; + case PROFILE_SM_UDP: + ink_assert(!"Can not delete a base ProfileSM!"); + break; + default: + ink_assert(!"not reached!"); + } +} + +// Free all attached ProfileSM +TS_INLINE void +NetVConnection::free_profile_sm(EThread *t) +{ + NetProfileSM *profilesm; + + while ((profilesm = this->profile_sm) != NULL) { + this->profile_sm = profilesm->low_profileSM; + profilesm->free(t); + } +} diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index f8ffe591a2f..ce6cb2a274d 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -92,6 +92,7 @@ extern RecRawStatBlock *net_rsb; #include "I_Net.h" #include "P_NetVConnection.h" #include "P_UnixNet.h" +#include "P_UnixNetProfileSM.h" #include "P_UnixNetProcessor.h" #include "P_NetAccept.h" #include "P_UnixNetVConnection.h" @@ -101,9 +102,8 @@ extern RecRawStatBlock *net_rsb; #include "P_NetVCTest.h" #include "P_LibBulkIO.h" -#include "P_SSLNetVConnection.h" +#include "P_SSLProfileSM.h" #include "P_SSLNetProcessor.h" -#include "P_SSLNetAccept.h" #include "P_SSLCertLookup.h" #undef NET_SYSTEM_MODULE_VERSION diff --git a/iocore/net/P_NetAccept.h b/iocore/net/P_NetAccept.h index bbd165cd13a..61c0cfa2f13 100644 --- a/iocore/net/P_NetAccept.h +++ b/iocore/net/P_NetAccept.h @@ -96,6 +96,8 @@ struct NetAccept : public Continuation { UnixNetVConnection *epoll_vc; // only storage for epoll events EventIO ep; + bool isSSL; + virtual NetProcessor *getNetProcessor() const; void init_accept_loop(const char *); diff --git a/iocore/net/P_SSLNetAccept.h b/iocore/net/P_SSLNetAccept.h deleted file mode 100644 index c20e9178688..00000000000 --- a/iocore/net/P_SSLNetAccept.h +++ /dev/null @@ -1,58 +0,0 @@ -/** @file - - A brief file description - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -/**************************************************************************** - - P_SSLNetAccept.h - - - NetAccept is a generalized facility which allows - Connections of different classes to be accepted either - from a blockable thread or by adaptive polling. - - It is used by the NetProcessor and the ClusterProcessor - and should be considered PRIVATE to processor implementations. - - - - ****************************************************************************/ -#if !defined(_SSLNetAccept_h_) -#define _SSLNetAccept_h_ - -#include "ts/ink_platform.h" -#include "P_Connection.h" -#include "P_NetAccept.h" - -// -// NetAccept -// Handles accepting connections. -// -struct SSLNetAccept : public NetAccept { - virtual NetProcessor *getNetProcessor() const; - virtual NetAccept *clone() const; - - SSLNetAccept(){}; - - virtual ~SSLNetAccept(){}; -}; -#endif diff --git a/iocore/net/P_SSLNetProcessor.h b/iocore/net/P_SSLNetProcessor.h index a490404267b..83bed3cc4d4 100644 --- a/iocore/net/P_SSLNetProcessor.h +++ b/iocore/net/P_SSLNetProcessor.h @@ -72,9 +72,6 @@ struct SSLNetProcessor : public UnixNetProcessor { // Private // - virtual NetAccept *createNetAccept(); - virtual NetVConnection *allocate_vc(EThread *t); - private: SSLNetProcessor(const SSLNetProcessor &); SSLNetProcessor &operator=(const SSLNetProcessor &); diff --git a/iocore/net/P_SSLNextProtocolAccept.h b/iocore/net/P_SSLNextProtocolAccept.h deleted file mode 100644 index 94b2553bb47..00000000000 --- a/iocore/net/P_SSLNextProtocolAccept.h +++ /dev/null @@ -1,66 +0,0 @@ -/** @file - - SSLNextProtocolAccept - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#ifndef P_SSLNextProtocolAccept_H_ -#define P_SSLNextProtocolAccept_H_ - -#include "P_Net.h" -#include "P_EventSystem.h" -#include "P_UnixNet.h" -#include "P_SSLNetVConnection.h" -#include "P_SSLNextProtocolSet.h" -#include "I_IOBuffer.h" - -class SSLNextProtocolAccept : public SessionAccept -{ -public: - SSLNextProtocolAccept(Continuation *, bool); - ~SSLNextProtocolAccept(); - - bool accept(NetVConnection *, MIOBuffer *, IOBufferReader *); - - // Register handler as an endpoint for the specified protocol. Neither - // handler nor protocol are copied, so the caller must guarantee their - // lifetime is at least as long as that of the acceptor. - bool registerEndpoint(const char *protocol, Continuation *handler); - - // Unregister the handler. Returns false if this protocol is not registered - // or if it is not registered for the specified handler. - bool unregisterEndpoint(const char *protocol, Continuation *handler); - - SLINK(SSLNextProtocolAccept, link); - -private: - int mainEvent(int event, void *netvc); - SSLNextProtocolAccept(const SSLNextProtocolAccept &); // disabled - SSLNextProtocolAccept &operator=(const SSLNextProtocolAccept &); // disabled - - MIOBuffer *buffer; // XXX do we really need this? - Continuation *endpoint; - SSLNextProtocolSet protoset; - bool transparent_passthrough; - - friend struct SSLNextProtocolTrampoline; -}; - -#endif /* P_SSLNextProtocolAccept_H_ */ diff --git a/iocore/net/P_SSLProfileSM.h b/iocore/net/P_SSLProfileSM.h new file mode 100644 index 00000000000..1df8d4ee40d --- /dev/null +++ b/iocore/net/P_SSLProfileSM.h @@ -0,0 +1,230 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/**************************************************************************** + + P_SSLProfileSM.h + reference from P_SSLNetVConnection.h + + This file implements an I/O Processor for TCP-SSL network I/O. + + ****************************************************************************/ +#ifndef __P_SSLPROFILESM_H__ +#define __P_SSLPROFILESM_H__ + +#include "ts/ink_sock.h" +#include "I_NetVConnection.h" +#include "P_UnixNetProfileSM.h" +#include "I_SSLM.h" +#include "P_UnixNetState.h" +#include "P_Connection.h" +#include "ts/ink_platform.h" +#include "P_EventSystem.h" +#include "P_UnixNetVConnection.h" +#include "P_UnixNet.h" +#include "ts/apidefs.h" + +#include +#include + +// These are included here because older OpenSSL libraries don't have them. +// Don't copy these defines, or use their values directly, they are merely +// here to avoid compiler errors. +#ifndef SSL_TLSEXT_ERR_OK +#define SSL_TLSEXT_ERR_OK 0 +#endif + +#ifndef SSL_TLSEXT_ERR_NOACK +#define SSL_TLSEXT_ERR_NOACK 3 +#endif + +#define SSL_OP_HANDSHAKE 0x16 +#define SSL_OP_SSLV2_HANDSHAKE 0x80 + +// TS-2503: dynamic TLS record sizing +// For smaller records, we should also reserve space for various TCP options +// (timestamps, SACKs.. up to 40 bytes [1]), and account for TLS record overhead +// (another 20-60 bytes on average, depending on the negotiated ciphersuite [2]). +// All in all: 1500 - 40 (IP) - 20 (TCP) - 40 (TCP options) - TLS overhead (60-100) +// For larger records, the size is determined by TLS protocol record size +#define SSL_DEF_TLS_RECORD_SIZE 1300 // 1500 - 40 (IP) - 20 (TCP) - 40 (TCP options) - TLS overhead (60-100) +#define SSL_MAX_TLS_RECORD_SIZE 16383 // 2^14 - 1 +#define SSL_DEF_TLS_RECORD_BYTE_THRESHOLD 1000000 +#define SSL_DEF_TLS_RECORD_MSEC_THRESHOLD 1000 + +////////////////////////////////////////////////////////////////// +// +// class SSLProfileSM +// +// A NetProfileSM to implement SSL/TLS layer on a socket +// +////////////////////////////////////////////////////////////////// +class SSLProfileSM : public UnixNetProfileSM, public SSLM +{ +public: + SSLProfileSM(); + static SSLProfileSM *allocate(EThread *t); + void free(EThread *t); + void clear(); + + int mainEvent(int event, void *data); + + int64_t read(void *buf, int64_t len, int &err); + int64_t + readv(struct iovec *vector, int count) + { + return 0; + } + int64_t write(void *buf, int64_t len, int &err = DEFAULT); + int64_t + writev(struct iovec *vector, int count) + { + return 0; + } + int64_t + raw_read(void *buf, int64_t len) + { + return low_profileSM->read(buf, len); + } + int64_t + raw_readv(struct iovec *vector, int count) + { + return low_profileSM->readv(vector, count); + } + int64_t + raw_write(void *buf, int64_t len) + { + return low_profileSM->write(buf, len); + } + int64_t + raw_writev(struct iovec *vector, int count) + { + return low_profileSM->writev(vector, count); + } + static bool + check_dep(NetProfileSM *low_profile_sm) + { + // The low_profile_sm is current profilesm attached to the vc. + // It should be a base profileSM (TCP or UDP) + if (low_profile_sm->low_profileSM == NULL) { + // Currently, SSLProfileSM only support TcpProfileSM as a low ProfileSM + if (low_profile_sm->type == PROFILE_SM_TCP) { + return true; + } + } + return false; + } + /// Reenable the VC after a pre-accept or SNI hook is called. + virtual void reenable(); + virtual const char *get_protocol_tag() const; + void close(); + +private: + SSLProfileSM(const SSLProfileSM &); + SSLProfileSM &operator=(const SSLProfileSM &); + +public: + int handshakeEvent(int event, void *data); + void handle_handshake(int event, NetHandler *nh, EThread *lthread); + + int sslStartHandShake(int event, int &err); + int sslServerHandShakeEvent(int &err); + int sslClientHandShakeEvent(int &err); + + void + initialize_handshake_buffers() + { + this->handShakeBuffer = new_MIOBuffer(); + this->handShakeReader = this->handShakeBuffer->alloc_reader(); + this->handShakeHolder = this->handShakeReader->clone(); + this->handShakeBioStored = 0; + } + void + free_handshake_buffers() + { + if (this->handShakeReader) { + this->handShakeReader->dealloc(); + } + if (this->handShakeHolder) { + this->handShakeHolder->dealloc(); + } + if (this->handShakeBuffer) { + free_MIOBuffer(this->handShakeBuffer); + } + this->handShakeReader = NULL; + this->handShakeHolder = NULL; + this->handShakeBuffer = NULL; + this->handShakeBioStored = 0; + } + + int64_t read_raw_data(); + int64_t read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf); + int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); + + // Returns true if all the hooks reenabled + bool callHooks(TSEvent eventId); + + // Returns true if we have already called at + // least some of the hooks + bool calledHooks(TSEvent /* eventId */) const { return (this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE); } + bool computeSSLTrace(); + + SSL *make_ssl_connection(SSL_CTX *ctx); + + ink_hrtime sslHandshakeBeginTime; + ink_hrtime sslLastWriteTime; + int64_t sslTotalBytesSent; + +private: + MIOBuffer *handShakeBuffer; + IOBufferReader *handShakeHolder; + IOBufferReader *handShakeReader; + int handShakeBioStored; + + /// The current hook. + /// @note For @C SSL_HOOKS_INVOKE, this is the hook to invoke. + class APIHook *curHook; + + // PreAcceptHook: Probe starting on a SSLProfileSM attached NetVC. + // HandshakeDoneHook: SSLProfileSM has finished a SSL Handshake. + enum { + SSL_HOOKS_INIT, ///< Initial state, no hooks called yet. + SSL_HOOKS_INVOKE, ///< Waiting to invoke hook. + SSL_HOOKS_ACTIVE, ///< Hook invoked, waiting for it to complete. + SSL_HOOKS_CONTINUE, ///< All hooks have been called and completed + SSL_HOOKS_DONE ///< All hooks have been called and completed + } sslPreAcceptHookState, + sslHandshakeDoneHookState; + + enum SSLHandshakeHookState { + HANDSHAKE_HOOKS_PRE, + HANDSHAKE_HOOKS_CERT, + HANDSHAKE_HOOKS_POST, + HANDSHAKE_HOOKS_INVOKE, + HANDSHAKE_HOOKS_DONE + } sslHandshakeHookState; +}; + +extern ClassAllocator sslProfileSMVCAllocator; + +#endif /* __P_SSLPROFILESM_H__ */ diff --git a/iocore/net/P_SSLUtils.h b/iocore/net/P_SSLUtils.h index 4f7f2f4966e..72bc80b3a7d 100644 --- a/iocore/net/P_SSLUtils.h +++ b/iocore/net/P_SSLUtils.h @@ -36,7 +36,8 @@ struct SSLConfigParams; struct SSLCertLookup; -class SSLNetVConnection; +class NetVConnection; +class SSLProfileSM; struct RecRawStatBlock; typedef int ssl_error_t; @@ -151,7 +152,7 @@ ssl_error_t SSLConnect(SSL *ssl); RecIncrRawStat(ssl_rsb, NULL, (int)x, 1); \ } while (0) -void SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); +void SSLDiagnostic(const SourceLocation &loc, bool debug, NetVConnection *vc, const char *fmt, ...) TS_PRINTFLIKE(4, 5); // Return a static string name for a SSL_ERROR constant. const char *SSLErrorName(int ssl_error); @@ -162,14 +163,14 @@ void SSLDebugBufferPrint(const char *tag, const char *buffer, unsigned buflen, c // Load the SSL certificate configuration. bool SSLParseCertificateConfiguration(const SSLConfigParams *params, SSLCertLookup *lookup); -// Attach a SSL NetVC back pointer to a SSL session. -void SSLNetVCAttach(SSL *ssl, SSLNetVConnection *vc); +// Attach a SSLProfileSM back pointer to a SSL session. +void SSLProfileSMAttach(SSL *ssl, SSLProfileSM *pSM); -// Detach from SSL NetVC back pointer to a SSL session. -void SSLNetVCDetach(SSL *ssl); +// Detach from SSLProfileSM pointer to a SSL session. +void SSLProfileSMDetach(SSL *ssl); -// Return the SSLNetVConnection (if any) attached to this SSL session. -SSLNetVConnection *SSLNetVCAccess(const SSL *ssl); +// Return the SSLProfileSM (if any) attached to this SSL session. +SSLProfileSM *SSLProfileSMAccess(const SSL *ssl); namespace ssl { diff --git a/iocore/net/P_UnixNetProcessor.h b/iocore/net/P_UnixNetProcessor.h index bf24b8044de..0fea9a6c8e7 100644 --- a/iocore/net/P_UnixNetProcessor.h +++ b/iocore/net/P_UnixNetProcessor.h @@ -42,6 +42,7 @@ struct UnixNetProcessor : public NetProcessor { virtual NetAccept *createNetAccept(); virtual NetVConnection *allocate_vc(EThread *t); + virtual void attach_profile_sm(EThread *t, NetVConnection *vc); virtual int start(int number_of_net_threads, size_t stacksize); diff --git a/iocore/net/P_UnixNetProfileSM.h b/iocore/net/P_UnixNetProfileSM.h new file mode 100644 index 00000000000..299c351dcbf --- /dev/null +++ b/iocore/net/P_UnixNetProfileSM.h @@ -0,0 +1,138 @@ +/** @file + A brief file description + + @section license License + */ + +/******** + + This file implements a NetProfileSM edition for Unix/Linux. + + ********/ + +#ifndef __P_UNIXNET_PROFILESM_H__ +#define __P_UNIXNET_PROFILESM_H__ + +#include "ts/ink_sock.h" +#include "I_NetVConnection.h" +#include "P_UnixNetState.h" +#include "P_Connection.h" + +class NetHandler; + +class UnixNetProfileSM : public NetProfileSM +{ +public: + UnixNetProfileSM(ProxyMutex *aMutex = NULL) : NetProfileSM(aMutex) {} + virtual void free(EThread *t) = 0; + virtual int mainEvent(int event, void *data) = 0; + virtual void handle_read(NetHandler *nh, EThread *lthread); + virtual void handle_write(NetHandler *nh, EThread *lthread); + + // for READ & WRITE + virtual int64_t read(void *buf, int64_t len, int &err = DEFAULT) = 0; + virtual int64_t readv(struct iovec *vector, int count) = 0; + virtual int64_t write(void *buf, int64_t len, int &err = DEFAULT) = 0; + virtual int64_t writev(struct iovec *vector, int count) = 0; + virtual int64_t raw_read(void *buf, int64_t len) = 0; + virtual int64_t raw_readv(struct iovec *vector, int count) = 0; + virtual int64_t raw_write(void *buf, int64_t len) = 0; + virtual int64_t raw_writev(struct iovec *vector, int count) = 0; + virtual int64_t read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf) = 0; + virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) = 0; + + virtual void + reenable() + { + return; + } + + virtual const char *get_protocol_tag() const = 0; +}; + +/******** + TcpProfileSM + One of base NetProfileSM for any TCP based protocol. + ********/ + +class TcpProfileSM : public UnixNetProfileSM +{ +public: + TcpProfileSM(); + static TcpProfileSM *allocate(EThread *t); + static bool + check_dep(NetProfileSM *current_netprofile_sm) + { + // TcpProfileSM is base NetProfileSM. + // The current NetProfileSM of netVC should not set before attach TcpProfileSM. + return current_netprofile_sm ? false : true; + } + void free(EThread *t); + int mainEvent(int event, void *data); + + // for READ & WRITE + int64_t read(void *buf, int64_t len, int &err = DEFAULT); + int64_t readv(struct iovec *vector, int count); + int64_t write(void *buf, int64_t len, int &err = DEFAULT); + int64_t writev(struct iovec *vector, int count); + int64_t + raw_read(void *buf, int64_t len) + { + return 0; + } + int64_t + raw_readv(struct iovec *vector, int count) + { + return 0; + } + int64_t + raw_write(void *buf, int64_t len) + { + return 0; + } + int64_t + raw_writev(struct iovec *vector, int count) + { + return 0; + } + const char * + get_protocol_tag() const + { + return vc->options.get_proto_string(); + } + int64_t read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf); + int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); + +private: + TcpProfileSM(const TcpProfileSM &); + TcpProfileSM &operator=(const TcpProfileSM &); +}; + +extern ClassAllocator tcpProfileSMAllocator; + +/******** + TODO: UdpProfileSM + ********/ +/* +class UdpProfileSM : public ProfileSM +{ +public: + UdpProfileSM(); + static UdpProfileSM *allocate(EThread *t); + static bool + check_dep(NetProfileSM *current_netprofile_sm) + { + // UdpProfileSM is base NetProfileSM. + // The current NetProfileSM of netVC should not set before attach UdpProfileSM. + return current_netprofile_sm ? false : true; + } + void free(EThread *t); + int mainEvent(int event, void *data); +private: + UdpProfileSM(const UdpProfileSM &); + UdpProfileSM &operator=(const UdpProfileSM &); +}; +extern ClassAllocator udpProfileSMAllocator; +*/ + +#endif /* __P_UNIXNET_PROFILESM_H__ */ diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 0e634b9ff8e..17af4f166c6 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -44,6 +44,7 @@ struct PollDescriptor; TS_INLINE void NetVCOptions::reset() { + isSSL = false; ip_proto = USE_TCP; ip_family = AF_INET; local_ip.invalidate(); @@ -108,28 +109,6 @@ class UnixNetVConnection : public NetVConnection virtual Action *send_OOB(Continuation *cont, char *buf, int len); virtual void cancel_OOB(); - virtual void - setSSLHandshakeWantsRead(bool /* flag */) - { - return; - } - virtual bool - getSSLHandshakeWantsRead() - { - return false; - } - virtual void - setSSLHandshakeWantsWrite(bool /* flag */) - { - return; - } - - virtual bool - getSSLHandshakeWantsWrite() - { - return false; - } - virtual void do_io_close(int lerrno = -1); virtual void do_io_shutdown(ShutdownHowTo_t howto); @@ -173,12 +152,16 @@ class UnixNetVConnection : public NetVConnection int populate_protocol(char const **results, int n) const { - int retval = 0; + int retval = 0; + NetProfileSM *pSM = this->profile_sm; + + while (pSM && n > 0) { + results[retval++] = pSM->get_protocol_tag(); + n--; + pSM = pSM->low_profileSM; + } if (n > 0) { - results[retval++] = options.get_proto_string(); - if (n > 1) { - results[retval++] = options.get_family_string(); - } + results[retval++] = options.get_family_string(); } return retval; } @@ -187,11 +170,18 @@ class UnixNetVConnection : public NetVConnection protocol_contains(const char *tag) const { const char *retval = NULL; + const char *test_tag = NULL; unsigned int tag_len = strlen(tag); - const char *test_tag = options.get_proto_string(); - if (strncmp(tag, test_tag, tag_len) == 0) { - retval = test_tag; - } else { + NetProfileSM *pSM = this->profile_sm; + + while (pSM && retval == NULL) { + test_tag = pSM->get_protocol_tag(); + if (strncmp(tag, test_tag, tag_len) == 0) { + retval = test_tag; + } + pSM = pSM->low_profileSM; + } + if (retval == NULL) { test_tag = options.get_family_string(); if (strncmp(tag, test_tag, tag_len) == 0) { retval = test_tag; @@ -212,31 +202,16 @@ class UnixNetVConnection : public NetVConnection void get_local_sa(); - // these are not part of the pure virtual interface. They were - // added to reduce the amount of duplicate code in classes inherited - // from NetVConnection (SSL). - virtual int - sslStartHandShake(int event, int &err) - { - (void)event; - (void)err; - return EVENT_ERROR; - } - - virtual bool - getSSLHandShakeComplete() const - { - return (true); - } - - virtual void net_read_io(NetHandler *nh, EThread *lthread); - virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); - void readDisable(NetHandler *nh); - void readSignalError(NetHandler *nh, int err); - int readSignalDone(int event, NetHandler *nh); + void readDisable(); + void writeDisable(); + void readSignalError(int err); + void writeSignalError(int err); + int readSignalDone(int event); + int writeSignalDone(int event); int readSignalAndUpdate(int event); - void readReschedule(NetHandler *nh); - void writeReschedule(NetHandler *nh); + int writeSignalAndUpdate(int event); + void readReschedule(); + void writeReschedule(); void netActivity(EThread *lthread); /** * If the current object's thread does not match the t argument, create a new @@ -288,11 +263,6 @@ class UnixNetVConnection : public NetVConnection OOB_callback *oob_ptr; bool from_accept_thread; - // es - origin_trace associated connections - bool origin_trace; - const sockaddr *origin_trace_addr; - int origin_trace_port; - int startEvent(int event, Event *e); int acceptEvent(int event, Event *e); int mainEvent(int event, Event *e); @@ -313,26 +283,6 @@ class UnixNetVConnection : public NetVConnection virtual int set_tcp_init_cwnd(int init_cwnd); virtual int set_tcp_congestion_control(const char *name, int len); virtual void apply_options(); - - friend void write_to_net_io(NetHandler *, UnixNetVConnection *, EThread *); - - void - setOriginTrace(bool t) - { - origin_trace = t; - } - - void - setOriginTraceAddr(const sockaddr *addr) - { - origin_trace_addr = addr; - } - - void - setOriginTracePort(int port) - { - origin_trace_port = port; - } }; extern ClassAllocator netVCAllocator; diff --git a/iocore/net/SSLClientUtils.cc b/iocore/net/SSLClientUtils.cc index bfc3cf75c39..5831165f716 100644 --- a/iocore/net/SSLClientUtils.cc +++ b/iocore/net/SSLClientUtils.cc @@ -42,7 +42,8 @@ verify_callback(int preverify_ok, X509_STORE_CTX *ctx) int depth; int err; SSL *ssl; - SSLNetVConnection *netvc; + SSLProfileSM *ssl_profile_sm; + NetVConnection *netvc = NULL; SSLDebug("Entered verify cb"); depth = X509_STORE_CTX_get_error_depth(ctx); @@ -59,12 +60,16 @@ verify_callback(int preverify_ok, X509_STORE_CTX *ctx) // Not server cert.... return preverify_ok; } - - // Retrieve the pointer to the SSL of the connection currently treated - // and the application specific data stored into the SSL object. - ssl = static_cast(X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); - netvc = SSLNetVCAccess(ssl); - + /* + * Retrieve the pointer to the SSL of the connection currently treated + * and the application specific data stored into the SSL object. + */ + ssl = static_cast(X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); + ssl_profile_sm = SSLProfileSMAccess(ssl); + + if (ssl_profile_sm != NULL) { + netvc = ssl_profile_sm->vc; + } if (netvc != NULL) { // Match SNI if present if (netvc->options.sni_servername) { diff --git a/iocore/net/SSLInternal.cc b/iocore/net/SSLInternal.cc index deaf70ebedc..676d3363a24 100644 --- a/iocore/net/SSLInternal.cc +++ b/iocore/net/SSLInternal.cc @@ -32,7 +32,7 @@ #include #include "P_Net.h" -#include "P_SSLNetVConnection.h" +#include "P_SSLProfileSM.h" void SSL_set_rbio(SSL *ssl, BIO *rbio) diff --git a/iocore/net/SSLM.cc b/iocore/net/SSLM.cc new file mode 100644 index 00000000000..154b06505f6 --- /dev/null +++ b/iocore/net/SSLM.cc @@ -0,0 +1,117 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "I_SSLM.h" +#include "P_SSLUtils.h" + +SSLM::SSLM() + : ssl(NULL), + hookOpRequested(SSL_HOOK_OP_DEFAULT), + transparentPassThrough(false), + sslHandShakeComplete(false), + sslClientRenegotiationAbort(false), + sslSessionCacheHit(false), + npnSet(NULL), + npnEndpoint(NULL), + error_code(0) +{ +} + +SSLM::~SSLM() +{ + return; +} + +void +SSLM::clear() +{ + if (ssl != NULL) { + SSL_free(ssl); + ssl = NULL; + } + + sslHandShakeComplete = false; + transparentPassThrough = false; + hookOpRequested = SSL_HOOK_OP_DEFAULT; + sslClientRenegotiationAbort = false; + sslSessionCacheHit = false; + npnSet = NULL; + npnEndpoint = NULL; + error_code = 0; +} + +void +SSLM::registerNextProtocolSet(const SSLNextProtocolSet *s) +{ + ink_release_assert(this->npnSet == NULL); + this->npnSet = s; +} + +// NextProtocolNegotiation TLS extension callback. The NPN extension +// allows the client to select a preferred protocol, so all we have +// to do here is tell them what out protocol set is. +int +SSLM::advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned int *outlen, void * /*arg ATS_UNUSED */) +{ + NetProfileSM *profile_sm = (NetProfileSM *)SSLProfileSMAccess(ssl); + SSLM *sslm = dynamic_cast(profile_sm); + + ink_release_assert(sslm != NULL); + + if (sslm->npnSet && sslm->npnSet->advertiseProtocols(out, outlen)) { + // Successful return tells OpenSSL to advertise. + return SSL_TLSEXT_ERR_OK; + } + + return SSL_TLSEXT_ERR_NOACK; +} + +// ALPN TLS extension callback. Given the client's set of offered +// protocols, we have to select a protocol to use for this session. +int +SSLM::select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in ATS_UNUSED, + unsigned inlen ATS_UNUSED, void *) +{ + NetProfileSM *profile_sm = (NetProfileSM *)SSLProfileSMAccess(ssl); + SSLM *sslm = dynamic_cast(profile_sm); + const unsigned char *npn = NULL; + unsigned npnsz = 0; + + ink_release_assert(sslm != NULL); + + if (sslm->npnSet && sslm->npnSet->advertiseProtocols(&npn, &npnsz)) { +// SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the +// server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short. + +#if HAVE_SSL_SELECT_NEXT_PROTO + if (SSL_select_next_proto((unsigned char **)out, outlen, npn, npnsz, in, inlen) == OPENSSL_NPN_NEGOTIATED) { + Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out); + return SSL_TLSEXT_ERR_OK; + } +#endif /* HAVE_SSL_SELECT_NEXT_PROTO */ + } + + *out = NULL; + *outlen = 0; + return SSL_TLSEXT_ERR_NOACK; +} diff --git a/iocore/net/SSLNetProcessor.cc b/iocore/net/SSLNetProcessor.cc index 366dcd1b509..a3a629b4653 100644 --- a/iocore/net/SSLNetProcessor.cc +++ b/iocore/net/SSLNetProcessor.cc @@ -91,28 +91,6 @@ SSLNetProcessor::start(int, size_t stacksize) return 0; } -NetAccept * -SSLNetProcessor::createNetAccept() -{ - return (NetAccept *)new SSLNetAccept; -} - -NetVConnection * -SSLNetProcessor::allocate_vc(EThread *t) -{ - SSLNetVConnection *vc; - - if (t) { - vc = THREAD_ALLOC_INIT(sslNetVCAllocator, t); - } else { - if (likely(vc = sslNetVCAllocator.alloc())) { - vc->from_accept_thread = true; - } - } - - return vc; -} - SSLNetProcessor::SSLNetProcessor() : client_ctx(NULL) { } diff --git a/iocore/net/SSLNextProtocolAccept.cc b/iocore/net/SSLNextProtocolAccept.cc deleted file mode 100644 index 26f935bcd2e..00000000000 --- a/iocore/net/SSLNextProtocolAccept.cc +++ /dev/null @@ -1,176 +0,0 @@ -/** @file - - SSLNextProtocolAccept - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "P_SSLNextProtocolAccept.h" - -static void -send_plugin_event(Continuation *plugin, int event, void *edata) -{ - if (plugin->mutex) { - SCOPED_MUTEX_LOCK(lock, plugin->mutex, this_ethread()); - plugin->handleEvent(event, edata); - } else { - plugin->handleEvent(event, edata); - } -} - -static SSLNetVConnection * -ssl_netvc_cast(int event, void *edata) -{ - union { - VIO *vio; - NetVConnection *vc; - } ptr; - - switch (event) { - case NET_EVENT_ACCEPT: - ptr.vc = static_cast(edata); - return dynamic_cast(ptr.vc); - case VC_EVENT_INACTIVITY_TIMEOUT: - case VC_EVENT_READ_COMPLETE: - case VC_EVENT_ERROR: - ptr.vio = static_cast(edata); - return dynamic_cast(ptr.vio->vc_server); - default: - return NULL; - } -} - -// SSLNextProtocolTrampoline is the receiver of the I/O event generated when we perform a 0-length read on the new SSL -// connection. The 0-length read forces the SSL handshake, which allows us to bind an endpoint that is selected by the -// NPN extension. The Continuation that receives the read event *must* have a mutex, but we don't want to take a global -// lock across the handshake, so we make a trampoline to bounce the event from the SSL acceptor to the ultimate session -// acceptor. -struct SSLNextProtocolTrampoline : public Continuation { - SSLNextProtocolTrampoline(const SSLNextProtocolAccept *npn, Ptr &mutex) : Continuation(mutex), npnParent(npn) - { - SET_HANDLER(&SSLNextProtocolTrampoline::ioCompletionEvent); - } - - int - ioCompletionEvent(int event, void *edata) - { - VIO *vio; - Continuation *plugin; - SSLNetVConnection *netvc; - - vio = static_cast(edata); - netvc = dynamic_cast(vio->vc_server); - ink_assert(netvc != NULL); - - switch (event) { - case VC_EVENT_EOS: - case VC_EVENT_ERROR: - case VC_EVENT_ACTIVE_TIMEOUT: - case VC_EVENT_INACTIVITY_TIMEOUT: - // Cancel the read before we have a chance to delete the continuation - netvc->do_io_read(NULL, 0, NULL); - netvc->do_io(VIO::CLOSE); - delete this; - return EVENT_ERROR; - case VC_EVENT_READ_COMPLETE: - break; - default: - return EVENT_ERROR; - } - - // Cancel the action, so later timeouts and errors don't try to - // send the event to the Accept object. After this point, the accept - // object does not care. - netvc->set_action(NULL); - - // Cancel the read before we have a chance to delete the continuation - netvc->do_io_read(NULL, 0, NULL); - plugin = netvc->endpoint(); - if (plugin) { - send_plugin_event(plugin, NET_EVENT_ACCEPT, netvc); - } else if (npnParent->endpoint) { - // Route to the default endpoint - send_plugin_event(npnParent->endpoint, NET_EVENT_ACCEPT, netvc); - } else { - // No handler, what should we do? Best to just kill the VC while we can. - netvc->do_io(VIO::CLOSE); - } - - delete this; - return EVENT_CONT; - } - - const SSLNextProtocolAccept *npnParent; -}; - -int -SSLNextProtocolAccept::mainEvent(int event, void *edata) -{ - SSLNetVConnection *netvc = ssl_netvc_cast(event, edata); - - Debug("ssl", "[SSLNextProtocolAccept:mainEvent] event %d netvc %p", event, netvc); - - switch (event) { - case NET_EVENT_ACCEPT: - ink_release_assert(netvc != NULL); - - netvc->setTransparentPassThrough(transparent_passthrough); - - // Register our protocol set with the VC and kick off a zero-length read to - // force the SSLNetVConnection to complete the SSL handshake. Don't tell - // the endpoint that there is an accept to handle until the read completes - // and we know which protocol was negotiated. - netvc->registerNextProtocolSet(&this->protoset); - netvc->do_io(VIO::READ, new SSLNextProtocolTrampoline(this, netvc->mutex), 0, this->buffer, 0); - return EVENT_CONT; - default: - netvc->do_io(VIO::CLOSE); - return EVENT_DONE; - } -} - -bool -SSLNextProtocolAccept::accept(NetVConnection *, MIOBuffer *, IOBufferReader *) -{ - ink_release_assert(0); - return false; -} - -bool -SSLNextProtocolAccept::registerEndpoint(const char *protocol, Continuation *handler) -{ - return this->protoset.registerEndpoint(protocol, handler); -} - -bool -SSLNextProtocolAccept::unregisterEndpoint(const char *protocol, Continuation *handler) -{ - return this->protoset.unregisterEndpoint(protocol, handler); -} - -SSLNextProtocolAccept::SSLNextProtocolAccept(Continuation *ep, bool transparent_passthrough) - : SessionAccept(NULL), buffer(new_empty_MIOBuffer()), endpoint(ep), transparent_passthrough(transparent_passthrough) -{ - SET_HANDLER(&SSLNextProtocolAccept::mainEvent); -} - -SSLNextProtocolAccept::~SSLNextProtocolAccept() -{ - free_MIOBuffer(this->buffer); -} diff --git a/iocore/net/SSLNextProtocolSet.cc b/iocore/net/SSLNextProtocolSet.cc index dc2ac939c08..8920bdf5d05 100644 --- a/iocore/net/SSLNextProtocolSet.cc +++ b/iocore/net/SSLNextProtocolSet.cc @@ -24,7 +24,7 @@ #include "ts/ink_config.h" #include "ts/apidefs.h" #include "ts/ink_platform.h" -#include "P_SSLNextProtocolSet.h" +#include "I_SSLNextProtocolSet.h" // For currently defined protocol strings, see // http://technotes.googlecode.com/git/nextprotoneg.html. The OpenSSL diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLProfileSM.cc similarity index 57% rename from iocore/net/SSLNetVConnection.cc rename to iocore/net/SSLProfileSM.cc index c534982e241..ca5c57f4d89 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLProfileSM.cc @@ -24,7 +24,8 @@ #include "ts/EventNotify.h" #include "records/I_RecHttp.h" #include "P_Net.h" -#include "P_SSLNextProtocolSet.h" +#include "ts/ink_platform.h" +#include "ts/InkErrno.h" #include "P_SSLUtils.h" #include "InkAPIInternal.h" // Added to include the ssl_hook definitions #include "P_SSLConfig.h" @@ -57,7 +58,7 @@ void SSL_set_rbio(SSL *ssl, BIO *rbio); #define SSL_WRITE_WOULD_BLOCK 10 #define SSL_WAIT_FOR_HOOK 11 -ClassAllocator sslNetVCAllocator("sslNetVCAllocator"); +ClassAllocator sslProfileSMAllocator("sslProfileSMAllocator"); namespace { @@ -133,26 +134,22 @@ class ContWrapper : public Continuation // Private // -static SSL * -make_ssl_connection(SSL_CTX *ctx, SSLNetVConnection *netvc) +SSL * +SSLProfileSM::make_ssl_connection(SSL_CTX *ctx) { - SSL *ssl; - if (likely(ssl = SSL_new(ctx))) { - netvc->ssl = ssl; - // Only set up the bio stuff for the server side - if (netvc->get_context() == NET_VCONNECTION_OUT) { - SSL_set_fd(ssl, netvc->get_socket()); + if (vc->get_context() == NET_VCONNECTION_OUT) { + SSL_set_fd(ssl, vc->get_socket()); } else { - netvc->initialize_handshake_buffers(); + initialize_handshake_buffers(); BIO *rbio = BIO_new(BIO_s_mem()); - BIO *wbio = BIO_new_fd(netvc->get_socket(), BIO_NOCLOSE); + BIO *wbio = BIO_new_fd(vc->get_socket(), BIO_NOCLOSE); BIO_set_mem_eof_return(wbio, -1); SSL_set_bio(ssl, rbio, wbio); } - SSLNetVCAttach(ssl, netvc); + SSLProfileSMAttach(ssl, this); } return ssl; @@ -182,19 +179,90 @@ debug_certificate_name(const char *msg, X509_NAME *name) BIO_free(bio); } -static int -ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) +int64_t +SSLProfileSM::read(void *buf, int64_t len, int &err) { - NetState *s = &sslvc->read; - MIOBufferAccessor &buf = s->vio.buffer; - int event = SSL_READ_ERROR_NONE; - int64_t bytes_read = 0; - ssl_error_t sslErr = SSL_ERROR_NONE; - int64_t nread = 0; + int64_t r = 0; + bool trace = this->getTrace(); + err = SSLReadBuffer(ssl, buf, len, r); + if (r > 0) { + if (!vc->getOriginTrace()) { + TraceIn((trace), vc->get_remote_addr(), vc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)r, (int)r, (char *)buf); + } else { + char origin_trace_ip[INET6_ADDRSTRLEN]; + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + TraceIn((trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)r, (int)r, (char *)buf); + } + } + return r; +} - bool trace = sslvc->getSSLTrace(); +int64_t +SSLProfileSM::write(void *buf, int64_t len, int &err) +{ + int64_t r = 0; + bool trace = this->getTrace(); + err = SSLWriteBuffer(ssl, buf, len, r); + if (r > 0) { + if (!vc->getOriginTrace()) { + TraceOut((trace), vc->get_remote_addr(), vc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)r, (int)r, (char *)buf); + } else { + char origin_trace_ip[INET6_ADDRSTRLEN]; + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + TraceOut((trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)r, (int)r, (char *)buf); + } + } + return r; +} + +// reference from SSLNetVConnection::net_read_io() +int64_t +SSLProfileSM::read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf) +{ + UnixNetVConnection *netvc = static_cast(vc); + int event = SSL_READ_ERROR_NONE; + ssl_error_t sslErr = SSL_ERROR_NONE; + int64_t nread = 0; - bytes_read = 0; + // referenced from SSLNetVConnection::net_read_io() + // At this point we are at the post-handshake SSL processing + // If the read BIO is not already a socket, consider changing it + if (this->handShakeReader) { + // Check out if there is anything left in the current bio + if (!BIO_eof(SSL_get_rbio(this->ssl))) { + // Still data remaining in the current BIO block + } else { + // Consume what SSL has read so far. + this->handShakeReader->consume(this->handShakeBioStored); + + // If we are empty now, switch over + if (this->handShakeReader->read_avail() <= 0) { + // Switch the read bio over to a socket bio + SSL_set_rfd(this->ssl, vc->get_socket()); + this->free_handshake_buffers(); + } else { + // Setup the next iobuffer block to drain + char *start = this->handShakeReader->start(); + char *end = this->handShakeReader->end(); + this->handShakeBioStored = end - start; + + // Sets up the buffer as a read only bio target + // Must be reset on each read + BIO *rbio = BIO_new_mem_buf(start, this->handShakeBioStored); + BIO_set_mem_eof_return(rbio, -1); + SSL_set_rbio(this->ssl, rbio); + } + } + } + // Otherwise, we already replaced the buffer bio with a socket bio + + // referenced from ssl_read_from_net() + bool trace = this->getTrace(); + + rattempted = 0; // non used + total_read = 0; while (sslErr == SSL_ERROR_NONE) { int64_t block_write_avail = buf.writer()->block_write_avail(); if (block_write_avail <= 0) { @@ -206,20 +274,11 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) } } - Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] b->write_avail()=%" PRId64, block_write_avail); + Debug("ssl", "[SSLProfileSM::read_from_net] b->write_avail()=%" PRId64, block_write_avail); char *current_block = buf.writer()->end(); - sslErr = SSLReadBuffer(sslvc->ssl, current_block, block_write_avail, nread); + nread = this->read(current_block, block_write_avail, sslErr); - Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] nread=%d", (int)nread); - if (!sslvc->origin_trace) { - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)nread, - (int)nread, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(sslvc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, sslvc->origin_trace_port, (int)nread, (int)nread, current_block); - } + Debug("ssl", "[SSLNetProfileSM::read_from_net] nread=%d", (int)nread); switch (sslErr) { case SSL_ERROR_NONE: @@ -228,76 +287,76 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) SSLDebugBufferPrint("ssl_buff", current_block, nread, "SSL Read"); #endif ink_assert(nread); - bytes_read += nread; + total_read += nread; if (nread > 0) { buf.writer()->fill(nread); // Tell the buffer, we've used the bytes } break; case SSL_ERROR_WANT_WRITE: - event = SSL_WRITE_WOULD_BLOCK; + // event = SSL_WRITE_WOULD_BLOCK; + event = -EAGAIN; SSL_INCREMENT_DYN_STAT(ssl_error_want_write); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(write)"); + Debug("ssl.error", "[SSLProfileSM::read_from_net] SSL_ERROR_WOULD_BLOCK(write)"); break; case SSL_ERROR_WANT_READ: - event = SSL_READ_WOULD_BLOCK; + // event = SSL_READ_WOULD_BLOCK; + event = -EAGAIN; SSL_INCREMENT_DYN_STAT(ssl_error_want_read); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); + Debug("ssl.error", "[SSLProfileSM::read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); break; case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Want X509 lookup"); - event = SSL_READ_WOULD_BLOCK; + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "Want X509 lookup"); + // event = SSL_READ_WOULD_BLOCK; + event = -EAGAIN; SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); + Debug("ssl.error", "[SSLProfileSM::read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); break; case SSL_ERROR_SYSCALL: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Syscall Error: %s", strerror(errno)); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "Syscall Error: %s", strerror(errno)); SSL_INCREMENT_DYN_STAT(ssl_error_syscall); if (nread != 0) { // not EOF - event = SSL_READ_ERROR; - ret = errno; - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Underlying IO error: %d", errno); + // event = SSL_READ_ERROR; + event = -errno; + // ret = errno; + Debug("ssl.error", "[SSLProfileSM::read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "Underlying IO error: %d", errno); } else { // then EOF observed, treat it as EOS - // Error("[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol"); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "EOF observed violating SSL protocol"); - event = SSL_READ_EOS; + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "EOF observed violating SSL protocol"); + // event = SSL_READ_EOS; + event = 0; } break; case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Connection closed by peer"); - event = SSL_READ_EOS; + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "Connection closed by peer"); + // event = SSL_READ_EOS; + event = 0; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_ZERO_RETURN"); + Debug("ssl.error", "[SSLProfileSM::read_from_net] SSL_ERROR_ZERO_RETURN"); break; case SSL_ERROR_SSL: default: { char buf[512]; unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", - sslErr, e, buf, errno); - event = SSL_READ_ERROR; - ret = errno; - SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "[SSL_NetVConnection::ssl_read_from_net]: errno=%d", errno); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", sslErr, + e, buf, errno); + // event = SSL_READ_ERROR; + event = -errno; + // ret = errno; + SSL_CLR_ERR_INCR_DYN_STAT(netvc, ssl_error_ssl, "[SSLProfileSM::read_from_net]: errno=%d", errno); } break; } // switch } // while - if (bytes_read > 0) { - Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] bytes_read=%" PRId64, bytes_read); - - s->vio.ndone += bytes_read; - sslvc->netActivity(lthread); - - ret = bytes_read; - - event = (s->vio.ntodo() <= 0) ? SSL_READ_COMPLETE : SSL_READ_READY; - } else { // if( bytes_read > 0 ) + if (total_read > 0) { + Debug("ssl", "[SSLProfileSM::read_from_net] total_read=%" PRId64, total_read); + event = toread; + } else { #if defined(_DEBUG) - if (bytes_read == 0) { - Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] bytes_read == 0"); + if (total_read == 0) { + Debug("ssl", "[SSLProfileSM::read_from_net] total_read == 0"); } #endif } @@ -312,7 +371,7 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) * the stored data (e.g. back out to blind tunneling) */ int64_t -SSLNetVConnection::read_raw_data() +SSLProfileSM::read_raw_data() { int64_t r = 0; int64_t toread = INT_MAX; @@ -344,7 +403,7 @@ SSLNetVConnection::read_raw_data() ink_assert(niov > 0); ink_assert(niov <= countof(tiovec)); - r = socketManager.readv(this->con.fd, &tiovec[0], niov); + r = this->raw_readv(&tiovec[0], niov); NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); total_read += rattempted; @@ -382,256 +441,8 @@ SSLNetVConnection::read_raw_data() return r; } -// changed by YTS Team, yamsat -void -SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) -{ - int ret; - int64_t r = 0; - int64_t bytes = 0; - NetState *s = &this->read; - - if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) { - this->super::net_read_io(nh, lthread); - return; - } - - MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, lthread, s->vio._cont); - if (!lock.is_locked()) { - readReschedule(nh); - return; - } - // Got closed by the HttpSessionManager thread during a migration - // The closed flag should be stable once we get the s->vio.mutex in that case - // (the global session pool mutex). - if (this->closed) { - this->super::net_read_io(nh, lthread); - return; - } - // If the key renegotiation failed it's over, just signal the error and finish. - if (sslClientRenegotiationAbort == true) { - this->read.triggered = 0; - readSignalError(nh, (int)r); - Debug("ssl", "[SSLNetVConnection::net_read_io] client renegotiation setting read signal error"); - return; - } - - // If it is not enabled, lower its priority. This allows - // a fast connection to speed match a slower connection by - // shifting down in priority even if it could read. - if (!s->enabled || s->vio.op != VIO::READ) { - read_disable(nh, this); - return; - } - - MIOBufferAccessor &buf = s->vio.buffer; - int64_t ntodo = s->vio.ntodo(); - ink_assert(buf.writer()); - - // Continue on if we are still in the handshake - if (!getSSLHandShakeComplete()) { - int err; - - if (get_context() == NET_VCONNECTION_OUT) { - ret = sslStartHandShake(SSL_EVENT_CLIENT, err); - } else { - ret = sslStartHandShake(SSL_EVENT_SERVER, err); - } - // If we have flipped to blind tunnel, don't read ahead - if (this->handShakeReader && this->attributes != HttpProxyPort::TRANSPORT_BLIND_TUNNEL) { - // Check and consume data that has been read - if (BIO_eof(SSL_get_rbio(this->ssl))) { - this->handShakeReader->consume(this->handShakeBioStored); - this->handShakeBioStored = 0; - } - } else if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) { - // Now in blind tunnel. Set things up to read what is in the buffer - // Must send the READ_COMPLETE here before considering - // forwarding on the handshake buffer, so the - // SSLNextProtocolTrampoline has a chance to do its - // thing before forwarding the buffers. - this->readSignalDone(VC_EVENT_READ_COMPLETE, nh); - - // If the handshake isn't set yet, this means the tunnel - // decision was make in the SNI callback. We must move - // the client hello message back into the standard read.vio - // so it will get forwarded onto the origin server - if (!this->getSSLHandShakeComplete()) { - this->sslHandShakeComplete = 1; - - // Copy over all data already read in during the SSL_accept - // (the client hello message) - NetState *s = &this->read; - MIOBufferAccessor &buf = s->vio.buffer; - int64_t r = buf.writer()->write(this->handShakeHolder); - s->vio.nbytes += r; - s->vio.ndone += r; - - // Clean up the handshake buffers - this->free_handshake_buffers(); - - if (r > 0) { - // Kick things again, so the data that was copied into the - // vio.read buffer gets processed - this->readSignalDone(VC_EVENT_READ_COMPLETE, nh); - } - } - return; - } - if (ret == EVENT_ERROR) { - this->read.triggered = 0; - readSignalError(nh, err); - } else if (ret == SSL_HANDSHAKE_WANT_READ || ret == SSL_HANDSHAKE_WANT_ACCEPT) { - if (SSLConfigParams::ssl_handshake_timeout_in > 0) { - double handshake_time = ((double)(Thread::get_hrtime() - sslHandshakeBeginTime) / 1000000000); - Debug("ssl", "ssl handshake for vc %p, took %.3f seconds, configured handshake_timer: %d", this, handshake_time, - SSLConfigParams::ssl_handshake_timeout_in); - if (handshake_time > SSLConfigParams::ssl_handshake_timeout_in) { - Debug("ssl", "ssl handshake for vc %p, expired, release the connection", this); - read.triggered = 0; - nh->read_ready_list.remove(this); - readSignalError(nh, VC_EVENT_EOS); - return; - } - } - read.triggered = 0; - nh->read_ready_list.remove(this); - readReschedule(nh); - } else if (ret == SSL_HANDSHAKE_WANT_CONNECT || ret == SSL_HANDSHAKE_WANT_WRITE) { - write.triggered = 0; - nh->write_ready_list.remove(this); - writeReschedule(nh); - } else if (ret == EVENT_DONE) { - // If this was driven by a zero length read, signal complete when - // the handshake is complete. Otherwise set up for continuing read - // operations. - if (ntodo <= 0) { - readSignalDone(VC_EVENT_READ_COMPLETE, nh); - } else { - read.triggered = 1; - if (read.enabled) - nh->read_ready_list.in_or_enqueue(this); - } - } else if (ret == SSL_WAIT_FOR_HOOK) { - // avoid readReschedule - done when the plugin calls us back to reenable - } else { - readReschedule(nh); - } - return; - } - - // If there is nothing to do or no space available, disable connection - if (ntodo <= 0 || !buf.writer()->write_avail()) { - read_disable(nh, this); - return; - } - - // At this point we are at the post-handshake SSL processing - // If the read BIO is not already a socket, consider changing it - if (this->handShakeReader) { - // Check out if there is anything left in the current bio - if (!BIO_eof(SSL_get_rbio(this->ssl))) { - // Still data remaining in the current BIO block - } else { - // Consume what SSL has read so far. - this->handShakeReader->consume(this->handShakeBioStored); - - // If we are empty now, switch over - if (this->handShakeReader->read_avail() <= 0) { - // Switch the read bio over to a socket bio - SSL_set_rfd(this->ssl, this->get_socket()); - this->free_handshake_buffers(); - } else { - // Setup the next iobuffer block to drain - char *start = this->handShakeReader->start(); - char *end = this->handShakeReader->end(); - this->handShakeBioStored = end - start; - - // Sets up the buffer as a read only bio target - // Must be reset on each read - BIO *rbio = BIO_new_mem_buf(start, this->handShakeBioStored); - BIO_set_mem_eof_return(rbio, -1); - SSL_set_rbio(this->ssl, rbio); - } - } - } - // Otherwise, we already replaced the buffer bio with a socket bio - - // not sure if this do-while loop is really needed here, please replace - // this comment if you know - do { - ret = ssl_read_from_net(this, lthread, r); - if (ret == SSL_READ_READY || ret == SSL_READ_ERROR_NONE) { - bytes += r; - } - ink_assert(bytes >= 0); - } while ((ret == SSL_READ_READY && bytes == 0) || ret == SSL_READ_ERROR_NONE); - - if (bytes > 0) { - if (ret == SSL_READ_WOULD_BLOCK || ret == SSL_READ_READY) { - if (readSignalAndUpdate(VC_EVENT_READ_READY) != EVENT_CONT) { - Debug("ssl", "ssl_read_from_net, readSignal != EVENT_CONT"); - return; - } - } - } - - switch (ret) { - case SSL_READ_ERROR_NONE: - case SSL_READ_READY: - readReschedule(nh); - return; - break; - case SSL_WRITE_WOULD_BLOCK: - case SSL_READ_WOULD_BLOCK: - if (lock.get_mutex() != s->vio.mutex.get()) { - Debug("ssl", "ssl_read_from_net, mutex switched"); - if (ret == SSL_READ_WOULD_BLOCK) - readReschedule(nh); - else - writeReschedule(nh); - return; - } - // reset the trigger and remove from the ready queue - // we will need to be retriggered to read from this socket again - read.triggered = 0; - nh->read_ready_list.remove(this); - Debug("ssl", "read_from_net, read finished - would block"); -#ifdef TS_USE_PORT - if (ret == SSL_READ_WOULD_BLOCK) - readReschedule(nh); - else - writeReschedule(nh); -#endif - break; - - case SSL_READ_EOS: - // close the connection if we have SSL_READ_EOS, this is the return value from ssl_read_from_net() if we get an - // SSL_ERROR_ZERO_RETURN from SSL_get_error() - // SSL_ERROR_ZERO_RETURN means that the origin server closed the SSL connection - read.triggered = 0; - readSignalDone(VC_EVENT_EOS, nh); - - if (bytes > 0) { - Debug("ssl", "read_from_net, read finished - EOS"); - } else { - Debug("ssl", "read_from_net, read finished - 0 useful bytes read, bytes used by SSL layer"); - } - break; - case SSL_READ_COMPLETE: - readSignalDone(VC_EVENT_READ_COMPLETE, nh); - Debug("ssl", "read_from_net, read finished - signal done"); - break; - case SSL_READ_ERROR: - this->read.triggered = 0; - readSignalError(nh, (int)r); - Debug("ssl", "read_from_net, read finished - read error"); - break; - } -} - int64_t -SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) +SSLProfileSM::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) { int64_t try_to_write; int64_t num_really_written = 0; @@ -649,15 +460,11 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf // reset sslTotalBytesSent upon inactivity for SSL_DEF_TLS_RECORD_MSEC_THRESHOLD sslTotalBytesSent = 0; } - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, now %" PRId64 ",lastwrite %" PRId64 " ,msec_since_last_write %d", now, + Debug("ssl", "[SSLProfileSM::load_buffer_and_write] now %" PRId64 ", lastwrite %" PRId64 ", msec_since_last_write %d", now, sslLastWriteTime, msec_since_last_write); } - if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) { - return this->super::load_buffer_and_write(towrite, buf, total_written, needs); - } - - bool trace = getSSLTrace(); + bool trace = getTrace(); do { // What is remaining left in the next block? @@ -695,19 +502,9 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf try_to_write = l; num_really_written = 0; - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, before SSLWriteBuffer, l=%" PRId64 ", towrite=%" PRId64 ", b=%p", l, + Debug("ssl", "SSLProfileSM::loadBufferAndCallWrite, before SSLWriteBuffer, l=%" PRId64 ", towrite=%" PRId64 ", b=%p", l, towrite, current_block); - err = SSLWriteBuffer(ssl, current_block, l, num_really_written); - - if (!origin_trace) { - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", - (int)num_really_written, (int)num_really_written, current_block); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, origin_trace_port, (int)num_really_written, (int)num_really_written, current_block); - } + num_really_written = this->write(current_block, l, err); // We wrote all that we thought we should if (num_really_written > 0) { @@ -715,8 +512,8 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf buf.reader()->consume(num_really_written); } - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite,Number of bytes written=%" PRId64 " , total=%" PRId64 "", - num_really_written, total_written); + Debug("ssl", "SSLProfileSM::loadBufferAndCallWrite,Number of bytes written=%" PRId64 " , total=%" PRId64 "", num_really_written, + total_written); NET_INCREMENT_DYN_STAT(net_calls_to_write_stat); } while (num_really_written == try_to_write && total_written < towrite); @@ -726,6 +523,7 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf } if (num_really_written > 0) { needs |= EVENTIO_WRITE; + return total_written; } else { switch (err) { case SSL_ERROR_NONE: @@ -743,7 +541,7 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf SSL_INCREMENT_DYN_STAT(ssl_error_want_write); } else if (SSL_ERROR_WANT_X509_LOOKUP == err) { SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - TraceOut(trace, get_remote_addr(), get_remote_port(), "Want X509 lookup"); + TraceOut(trace, vc->get_remote_addr(), vc->get_remote_port(), "Want X509 lookup"); } needs |= EVENTIO_WRITE; @@ -752,14 +550,14 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf break; } case SSL_ERROR_SYSCALL: - TraceOut(trace, get_remote_addr(), get_remote_port(), "Syscall Error: %s", strerror(errno)); + TraceOut(trace, vc->get_remote_addr(), vc->get_remote_port(), "Syscall Error: %s", strerror(errno)); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSL_write-SSL_ERROR_SYSCALL"); break; // end of stream case SSL_ERROR_ZERO_RETURN: - TraceOut(trace, get_remote_addr(), get_remote_port(), "SSL Error: zero return"); + TraceOut(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL Error: zero return"); num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSL_write-SSL_ERROR_ZERO_RETURN"); @@ -769,39 +567,181 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf char buf[512]; unsigned long e = ERR_peek_last_error(); ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", err, e, buf, - errno); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", err, e, + buf, errno); num_really_written = -errno; - SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno); + SSL_CLR_ERR_INCR_DYN_STAT(vc, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno); } break; } } return num_really_written; } -SSLNetVConnection::SSLNetVConnection() - : ssl(NULL), +SSLProfileSM::SSLProfileSM() + : UnixNetProfileSM(NULL), sslHandshakeBeginTime(0), sslLastWriteTime(0), sslTotalBytesSent(0), - hookOpRequested(SSL_HOOK_OP_DEFAULT), - sslHandShakeComplete(false), - sslClientRenegotiationAbort(false), - sslSessionCacheHit(false), handShakeBuffer(NULL), handShakeHolder(NULL), handShakeReader(NULL), handShakeBioStored(0), sslPreAcceptHookState(SSL_HOOKS_INIT), - sslHandshakeHookState(HANDSHAKE_HOOKS_PRE), - npnSet(NULL), - npnEndpoint(NULL), - sslTrace(false) + sslHandshakeDoneHookState(SSL_HOOKS_INIT), + sslHandshakeHookState(HANDSHAKE_HOOKS_PRE) +{ + type = PROFILE_SM_SSL; + SET_HANDLER(&SSLProfileSM::handshakeEvent); +} + +int +SSLProfileSM::mainEvent(int event, void *data) +{ + Debug("ssl", "SSLProfileSM::mainEvent event = %d", event); + NetHandler *nh = static_cast(data); + EThread *lthread = nh->trigger_event->ethread; + UnixNetVConnection *netvc = static_cast(vc); + NetState *s; + + ink_assert(this->getSSLHandShakeComplete()); + + // Get lock first + switch (event) { + case IOCORE_EVENTS_READ: + s = &netvc->read; + break; + case IOCORE_EVENTS_WRITE: + s = &netvc->write; + break; + default: + ink_assert(!"not reach"); + return EVENT_DONE; + } + MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, lthread, s->vio._cont); + if (!lock.is_locked() || lock.get_mutex() != s->vio.mutex.get()) { + switch (event) { + case IOCORE_EVENTS_READ: + netvc->readReschedule(); + break; + case IOCORE_EVENTS_WRITE: + netvc->writeReschedule(); + break; + default: + ink_assert(!"not reach"); + } + return EVENT_DONE; + } + + ink_release_assert(HttpProxyPort::TRANSPORT_BLIND_TUNNEL != vc->attributes); + + if (event == IOCORE_EVENTS_READ) { + // If the key renegotiation failed it's over, just signal the error and finish. + if (this->sslClientRenegotiationAbort == true) { + netvc->read.triggered = 0; + netvc->readSignalError(0); + Debug("ssl", "[SSLProfileSM::handle_read] client renegotiation setting read signal error"); + return EVENT_DONE; + } + } + + // No TRY LOCK in handle_read and handle_write + switch (event) { + case IOCORE_EVENTS_READ: + handle_read(nh, nh->trigger_event->ethread); + break; + case IOCORE_EVENTS_WRITE: + handle_write(nh, nh->trigger_event->ethread); + break; + default: + ink_assert(!"not reach"); + return EVENT_DONE; + } + + return EVENT_DONE; +} + +int +SSLProfileSM::handshakeEvent(int event, void *data) { + Debug("ssl", "SSLProfileSM::handshakeEvent event = %d", event); + NetHandler *nh = static_cast(data); + EThread *lthread = nh->trigger_event->ethread; + UnixNetVConnection *netvc = static_cast(vc); + NetState *s; + + // Get lock first + switch (event) { + case IOCORE_EVENTS_READ: + s = &netvc->read; + break; + case IOCORE_EVENTS_WRITE: + s = &netvc->write; + break; + default: + ink_assert(!"not reach"); + return EVENT_DONE; + } + MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, lthread, s->vio._cont); + if (!lock.is_locked() || lock.get_mutex() != s->vio.mutex.get()) { + switch (event) { + case IOCORE_EVENTS_READ: + netvc->readReschedule(); + break; + case IOCORE_EVENTS_WRITE: + netvc->writeReschedule(); + break; + default: + ink_assert(!"not reach"); + } + return EVENT_DONE; + } + + ink_release_assert(HttpProxyPort::TRANSPORT_BLIND_TUNNEL != vc->attributes); + + if (event == IOCORE_EVENTS_READ) { + // If the key renegotiation failed it's over, just signal the error and finish. + if (this->sslClientRenegotiationAbort == true) { + netvc->read.triggered = 0; + netvc->readSignalError(0); + Debug("ssl", "[SSLProfileSM::handshakeEvent] client renegotiation setting read signal error"); + return EVENT_DONE; + } + + // If it is not enabled, lower its priority. This allows + // a fast connection to speed match a slower connection by + // shifting down in priority even if it could read. + + if (!s->enabled || s->vio.op != VIO::READ) { + netvc->readDisable(); + return EVENT_DONE; + } + } + + // No TRY LOCK in handle_handshake + handle_handshake(event, nh, nh->trigger_event->ethread); + return EVENT_DONE; } void -SSLNetVConnection::do_io_close(int lerrno) +SSLProfileSM::clear() +{ + close(); + sslHandshakeBeginTime = 0; + sslLastWriteTime = 0; + sslTotalBytesSent = 0; + if (SSL_HOOKS_ACTIVE == sslPreAcceptHookState) { + Error("SSLProfileSM::clear freed with outstanding hook"); + } + sslPreAcceptHookState = SSL_HOOKS_INIT; + curHook = 0; + free_handshake_buffers(); + + SSLM::clear(); + NetProfileSM::clear(); +} + +void +SSLProfileSM::close() { if (this->ssl != NULL && sslHandShakeComplete) { int shutdown_mode = SSL_get_shutdown(ssl); @@ -821,7 +761,7 @@ SSLNetVConnection::do_io_close(int lerrno) // at the same time we send the close-notify. If so, the client will likely // send RST anyway char c; - ssize_t x = recv(this->con.fd, &c, 1, MSG_PEEK); + ssize_t x = recv(vc->get_socket(), &c, 1, MSG_PEEK); // x < 0 means error. x == 0 means fin sent if (x != 0) { // Send the close-notify @@ -829,86 +769,191 @@ SSLNetVConnection::do_io_close(int lerrno) Debug("ssl-shutdown", "SSL_shutdown %s", (ret) ? "success" : "failed"); } } - // Go on and do the unix socket cleanups - super::do_io_close(lerrno); } void -SSLNetVConnection::free(EThread *t) +SSLProfileSM::free(EThread *t) { - got_remote_addr = 0; - got_local_addr = 0; - read.vio.mutex.clear(); - write.vio.mutex.clear(); - this->mutex.clear(); - action_.mutex.clear(); - this->ep.stop(); - this->con.close(); - flags = 0; - - SET_CONTINUATION_HANDLER(this, (SSLNetVConnHandler)&SSLNetVConnection::startEvent); - - if (nh) { - nh->read_ready_list.remove(this); - nh->write_ready_list.remove(this); - nh = NULL; - } + clear(); - read.triggered = 0; - write.triggered = 0; - read.enabled = 0; - write.enabled = 0; - read.vio._cont = NULL; - write.vio._cont = NULL; - read.vio.vc_server = NULL; - write.vio.vc_server = NULL; + if (this->globally_allocated) { + sslProfileSMAllocator.free(this); + } else { + THREAD_FREE(this, sslProfileSMAllocator, t); + } +} - closed = 0; - options.reset(); - con.close(); +SSLProfileSM * +SSLProfileSM::allocate(EThread *t) +{ + SSLProfileSM *ssl_profile_sm; - ink_assert(con.fd == NO_FD); + if (t) { + ssl_profile_sm = THREAD_ALLOC_INIT(sslProfileSMAllocator, t); + } else { + if (likely(ssl_profile_sm = sslProfileSMAllocator.alloc())) + ssl_profile_sm->globally_allocated = true; + } + return ssl_profile_sm; +} - if (ssl != NULL) { - SSL_free(ssl); - ssl = NULL; +// reference from SSLNetVConnection::net_read_io() && write_to_net() +void +SSLProfileSM::handle_handshake(int event, NetHandler *nh, EThread *lthread) +{ + int err, ret; + UnixNetVConnection *netvc = static_cast(vc); + TSSslHookInternalID hookId = TS_SSL_CLIENT_HANDSHAKE_INTERNAL_HOOK; + TSHttpHookID eventId = TS_SSL_CLIENT_HANDSHAKE_HOOK; + + if (getSSLHandShakeComplete()) { + ret = EVENT_DONE; + } else { // Call ssl StartHandShake + if (netvc->get_context() == NET_VCONNECTION_OUT) { + ret = sslStartHandShake(SSL_EVENT_CLIENT, err); + } else { + ret = sslStartHandShake(SSL_EVENT_SERVER, err); + } } - sslHandShakeComplete = false; - sslHandshakeBeginTime = 0; - sslLastWriteTime = 0; - sslTotalBytesSent = 0; - sslClientRenegotiationAbort = false; - sslSessionCacheHit = false; + // Check for blind tunnel (only on IOCORE_EVENTS_READ and SSL_EVENT_SERVER) + if (event == IOCORE_EVENTS_READ) { + // If we have flipped to blind tunnel, don't read ahead + if (vc->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) { + // If the handshake isn't set yet, this means the tunnel + // decision was make in the SNI callback. We must move + // the client hello message back into the standard read.vio + // so it will get forwarded onto the origin server + if (!this->getSSLHandShakeComplete()) { + this->sslHandShakeComplete = 1; + + // Copy over all data already read in during the SSL_accept + // (the client hello message) + NetState *s = &netvc->read; + MIOBufferAccessor &buf = s->vio.buffer; + int64_t r = buf.writer()->write(this->handShakeHolder); + s->vio.nbytes += r; + s->vio.ndone += r; - if (SSL_HOOKS_ACTIVE == sslPreAcceptHookState) { - Error("SSLNetVconnection freed with outstanding hook"); + // Clean up the handshake buffers + this->free_handshake_buffers(); + } + netvc->del_profile_sm(lthread); + netvc->readSignalDone(VC_EVENT_READ_COMPLETE); + return; + } + } + if (this->handShakeReader) { + // Check and consume data that has been read + if (BIO_eof(SSL_get_rbio(this->ssl))) { + this->handShakeReader->consume(this->handShakeBioStored); + this->handShakeBioStored = 0; + } } - sslPreAcceptHookState = SSL_HOOKS_INIT; - curHook = 0; - hookOpRequested = SSL_HOOK_OP_DEFAULT; - npnSet = NULL; - npnEndpoint = NULL; - sslHandShakeComplete = false; - free_handshake_buffers(); - sslTrace = false; + // Check for return value from sslStartHandShake + switch (ret) { + case EVENT_ERROR: + if (event == IOCORE_EVENTS_READ) { + netvc->read.triggered = 0; + netvc->readSignalError(err); + } else if (event == IOCORE_EVENTS_WRITE) { + netvc->write.triggered = 0; + netvc->writeSignalError(err); + } + break; - if (from_accept_thread) { - sslNetVCAllocator.free(this); - } else { - ink_assert(con.fd == NO_FD); - THREAD_FREE(this, sslNetVCAllocator, t); + case SSL_HANDSHAKE_WANT_READ: + case SSL_HANDSHAKE_WANT_ACCEPT: + if (event == IOCORE_EVENTS_READ && netvc->get_context() == NET_VCONNECTION_IN && + SSLConfigParams::ssl_handshake_timeout_in > 0) { + double handshake_time = ((double)(Thread::get_hrtime() - sslHandshakeBeginTime) / 1000000000); + Debug("ssl", "ssl handshake for vc %p, took %.3f seconds, configured handshake_timer: %d", this->vc, handshake_time, + SSLConfigParams::ssl_handshake_timeout_in); + if (handshake_time > SSLConfigParams::ssl_handshake_timeout_in) { + Debug("ssl", "ssl handshake for vc %p, expired, release the connection", this->vc); + netvc->read.triggered = 0; + nh->read_ready_list.remove(netvc); + netvc->readSignalError(VC_EVENT_EOS); + return; + } + } + netvc->read.triggered = 0; + netvc->readReschedule(); + break; + + case SSL_HANDSHAKE_WANT_WRITE: + case SSL_HANDSHAKE_WANT_CONNECT: + netvc->write.triggered = 0; + netvc->writeReschedule(); + break; + + case EVENT_DONE: + Debug("ssl", "EVENT_DONE netvc->read.triggered=%d netvc->write.triggered=%d event=%d", netvc->read.triggered, + netvc->write.triggered, event); + if (netvc->get_context() == NET_VCONNECTION_IN) { + hookId = TS_SSL_SERVER_HANDSHAKE_INTERNAL_HOOK; + eventId = TS_SSL_SERVER_HANDSHAKE_HOOK; + } + if (SSL_HOOKS_DONE != sslHandshakeDoneHookState) { + // Get the first hook if we haven't started invoking yet. + if (SSL_HOOKS_INIT == sslHandshakeDoneHookState) { + curHook = ssl_hooks->get(hookId); + sslHandshakeDoneHookState = SSL_HOOKS_INVOKE; + } else if (SSL_HOOKS_INVOKE == sslHandshakeDoneHookState) { + // if the state is anything else, we haven't finished the previous hook yet. + curHook = curHook->next(); + } + if (SSL_HOOKS_INVOKE == sslHandshakeDoneHookState) { + if (0 == curHook) { // no hooks left, we're done + sslHandshakeDoneHookState = SSL_HOOKS_DONE; + } else { + sslHandshakeDoneHookState = SSL_HOOKS_ACTIVE; + ContWrapper::wrap(mutex.get(), curHook->m_cont, eventId, vc); + return; + } + } else { // waiting for hook to complete + return; + } + } + + SET_HANDLER(&SSLProfileSM::mainEvent); + if (event == IOCORE_EVENTS_READ) { + if (endpoint()) { // for ProtocolProbeSessionAccept + netvc->readSignalDone(VC_EVENT_READ_COMPLETE); + return; + } + netvc->read.triggered = 1; + netvc->readReschedule(); + } else if (event == IOCORE_EVENTS_WRITE) { + netvc->write.triggered = 1; + netvc->writeReschedule(); + } + break; + + case SSL_WAIT_FOR_HOOK: + // avoid read & write Reschedule - done when the plugin calls us back to reenable + return; + break; + + case EVENT_CONT: + default: + if (event == IOCORE_EVENTS_READ) { + netvc->readReschedule(); + } else if (event == IOCORE_EVENTS_WRITE) { + netvc->writeReschedule(); + } + break; } } int -SSLNetVConnection::sslStartHandShake(int event, int &err) +SSLProfileSM::sslStartHandShake(int event, int &err) { if (sslHandshakeBeginTime == 0) { sslHandshakeBeginTime = Thread::get_hrtime(); // net_activity will not be triggered until after the handshake - set_inactivity_timeout(HRTIME_SECONDS(SSLConfigParams::ssl_handshake_timeout_in)); + vc->set_inactivity_timeout(HRTIME_SECONDS(SSLConfigParams::ssl_handshake_timeout_in)); } switch (event) { @@ -917,15 +962,15 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) SSLCertificateConfig::scoped_config lookup; IpEndpoint ip; int namelen = sizeof(ip); - safe_getsockname(this->get_socket(), &ip.sa, &namelen); + safe_getsockname(vc->get_socket(), &ip.sa, &namelen); SSLCertContext *cc = lookup->find(ip); if (is_debug_tag_set("ssl")) { IpEndpoint src, dst; ip_port_text_buffer ipb1, ipb2; int ip_len; - safe_getsockname(this->get_socket(), &dst.sa, &(ip_len = sizeof ip)); - safe_getpeername(this->get_socket(), &src.sa, &(ip_len = sizeof ip)); + safe_getsockname(vc->get_socket(), &dst.sa, &(ip_len = sizeof ip)); + safe_getpeername(vc->get_socket(), &src.sa, &(ip_len = sizeof ip)); ats_ip_nptop(&dst, ipb1, sizeof(ipb1)); ats_ip_nptop(&src, ipb2, sizeof(ipb2)); Debug("ssl", "IP context is %p for [%s] -> [%s], default context %p", cc, ipb2, ipb1, lookup->defaultContext()); @@ -934,8 +979,8 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) // Escape if this is marked to be a tunnel. // No data has been read at this point, so we can go // directly into blind tunnel mode - if (cc && SSLCertContext::OPT_TUNNEL == cc->opt && this->is_transparent) { - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + if (cc && SSLCertContext::OPT_TUNNEL == cc->opt && vc->get_is_transparent()) { + vc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; sslHandShakeComplete = 1; SSL_free(this->ssl); this->ssl = NULL; @@ -945,35 +990,33 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) // Attach the default SSL_CTX to this SSL session. The default context is never going to be able // to negotiate a SSL session, but it's enough to trampoline us into the SNI callback where we // can select the right server certificate. - this->ssl = make_ssl_connection(lookup->defaultContext(), this); + this->make_ssl_connection(lookup->defaultContext()); #if !(TS_USE_TLS_SNI) // set SSL trace if (SSLConfigParams::ssl_wire_trace_enabled) { bool trace = computeSSLTrace(); - Debug("ssl", "sslnetvc. setting trace to=%s", trace ? "true" : "false"); - setSSLTrace(trace); + Debug("ssl", "netvc with SSLProfileSM. setting trace to=%s", trace ? "true" : "false"); + this->setTrace(trace); } #endif } if (this->ssl == NULL) { - SSLErrorVC(this, "failed to create SSL server session"); + SSLErrorVC(vc, "failed to create SSL server session"); return EVENT_ERROR; } return sslServerHandShakeEvent(err); case SSL_EVENT_CLIENT: - if (this->ssl == NULL) { - this->ssl = make_ssl_connection(ssl_NetProcessor.client_ctx, this); - + if (this->ssl == NULL && this->make_ssl_connection(ssl_NetProcessor.client_ctx) != NULL) { #if TS_USE_TLS_SNI - if (this->options.sni_servername) { - if (SSL_set_tlsext_host_name(this->ssl, this->options.sni_servername)) { - Debug("ssl", "using SNI name '%s' for client handshake", this->options.sni_servername.get()); + if (vc->options.sni_servername) { + if (SSL_set_tlsext_host_name(ssl, vc->options.sni_servername)) { + Debug("ssl", "using SNI name '%s' for client handshake", vc->options.sni_servername.get()); } else { - Debug("ssl.error", "failed to set SNI name '%s' for client handshake", this->options.sni_servername.get()); + Debug("ssl.error", "failed to set SNI name '%s' for client handshake", vc->options.sni_servername.get()); SSL_INCREMENT_DYN_STAT(ssl_sni_name_set_failure); } } @@ -981,7 +1024,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) } if (this->ssl == NULL) { - SSLErrorVC(this, "failed to create SSL client session"); + SSLErrorVC(vc, "failed to create SSL client session"); return EVENT_ERROR; } @@ -994,7 +1037,7 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) } int -SSLNetVConnection::sslServerHandShakeEvent(int &err) +SSLProfileSM::sslServerHandShakeEvent(int &err) { if (SSL_HOOKS_DONE != sslPreAcceptHookState) { // Get the first hook if we haven't started invoking yet. @@ -1012,7 +1055,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) sslPreAcceptHookState = SSL_HOOKS_DONE; } else { sslPreAcceptHookState = SSL_HOOKS_ACTIVE; - ContWrapper::wrap(mutex.get(), curHook->m_cont, TS_EVENT_VCONN_PRE_ACCEPT, this); + ContWrapper::wrap(mutex.get(), curHook->m_cont, TS_EVENT_VCONN_PRE_ACCEPT, vc); return SSL_WAIT_FOR_HOOK; } } else { // waiting for hook to complete @@ -1036,7 +1079,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // without data replay. // Note we can't arrive here if a hook is active. if (SSL_HOOK_OP_TUNNEL == hookOpRequested) { - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + vc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; SSL_free(this->ssl); this->ssl = NULL; // Don't mark the handshake as complete yet, @@ -1058,27 +1101,13 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) retval = this->read_raw_data(); if (retval == 0) { // EOF, go away, we stopped in the handshake - SSLDebugVC(this, "SSL handshake error: EOF"); + SSLDebugVC(vc, "SSL handshake error: EOF"); return EVENT_ERROR; } } ssl_error_t ssl_error = SSLAccept(ssl); - bool trace = getSSLTrace(); - - if (ssl_error != SSL_ERROR_NONE) { - err = errno; - SSLDebugVC(this, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err); - - // start a blind tunnel if tr-pass is set and data does not look like ClientHello - char *buf = handShakeBuffer->buf(); - if (getTransparentPassThrough() && buf && *buf != SSL_OP_HANDSHAKE) { - SSLDebugVC(this, "Data does not look like SSL handshake, starting blind tunnel"); - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; - sslHandShakeComplete = 0; - return EVENT_CONT; - } - } + bool trace = this->getTrace(); switch (ssl_error) { case SSL_ERROR_NONE: @@ -1095,7 +1124,7 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) sslHandShakeComplete = true; - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake completed successfully"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake completed successfully"); // do we want to include cert info in trace? if (sslHandshakeBeginTime) { @@ -1138,32 +1167,32 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) } Debug("ssl", "client selected next protocol '%.*s'", len, proto); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client selected next protocol'%.*s'", len, proto); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "client selected next protocol'%.*s'", len, proto); } else { Debug("ssl", "client did not select a next protocol"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "client did not select a next protocol"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "client did not select a next protocol"); } } return EVENT_DONE; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_CONNECT"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_CONNECT"); return SSL_HANDSHAKE_WANT_CONNECT; case SSL_ERROR_WANT_WRITE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_WRITE"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_READ"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_READ"); if (retval == -EAGAIN) { // No data at the moment, hang tight - SSLDebugVC(this, "SSL handshake: EAGAIN"); + SSLDebugVC(vc, "SSL handshake: EAGAIN"); return SSL_HANDSHAKE_WANT_READ; } else if (retval < 0) { // An error, make us go away - SSLDebugVC(this, "SSL handshake error: read_retval=%d", retval); + SSLDebugVC(vc, "SSL handshake error: read_retval=%d", retval); return EVENT_ERROR; } return SSL_HANDSHAKE_WANT_READ; @@ -1172,17 +1201,17 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) // enable the sni callback to break out of the SSL_accept processing #ifdef SSL_ERROR_WANT_SNI_RESOLVE case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); return EVENT_CONT; case SSL_ERROR_WANT_SNI_RESOLVE: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_SNI_RESOLVE"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_SNI_RESOLVE"); #elif SSL_ERROR_WANT_X509_LOOKUP case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_X509_LOOKUP"); #endif #if defined(SSL_ERROR_WANT_SNI_RESOLVE) || defined(SSL_ERROR_WANT_X509_LOOKUP) - if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || SSL_HOOK_OP_TUNNEL == hookOpRequested) { - this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + if (vc->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || SSL_HOOK_OP_TUNNEL == hookOpRequested) { + vc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; sslHandShakeComplete = 0; return EVENT_CONT; } else { @@ -1192,38 +1221,52 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err) #endif case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_WANT_ACCEPT"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_WANT_ACCEPT"); + return EVENT_CONT; + } + + if (getTransparentPassThrough()) { + err = errno; + SSLDebugVC(vc, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err); + + // start a blind tunnel if tr-pass is set and data does not look like ClientHello + SSLDebugVC(vc, "Data does not look like SSL handshake, starting blind tunnel"); + vc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; + sslHandShakeComplete = 0; return EVENT_CONT; + } + switch (ssl_error) { case SSL_ERROR_SSL: { - SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslServerHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); + SSL_CLR_ERR_INCR_DYN_STAT(vc, ssl_error_ssl, "SSLProfileSM::sslServerHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL server handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); + error_code = ERR_peek_last_error(); + ERR_error_string_n(error_code, buf, sizeof(buf)); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), + "SSL server handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, error_code, buf, errno); return EVENT_ERROR; } case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_ZERO_RETURN"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_ZERO_RETURN"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_SYSCALL"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_SYSCALL"); return EVENT_ERROR; default: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL server handshake ERROR_OTHER"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL server handshake ERROR_OTHER"); return EVENT_ERROR; } } int -SSLNetVConnection::sslClientHandShakeEvent(int &err) +SSLProfileSM::sslClientHandShakeEvent(int &err) { - bool trace = getSSLTrace(); + bool trace = this->getTrace(); ssl_error_t ssl_error; + UnixNetVConnection *netvc = static_cast(vc); - ink_assert(SSLNetVCAccess(ssl) == this); + ink_assert(SSLProfileSMAccess(ssl) == this); ssl_error = SSLConnect(ssl); switch (ssl_error) { @@ -1233,8 +1276,8 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) Debug("ssl", "SSL client handshake completed successfully"); // if the handshake is complete and write is enabled reschedule the write - if (closed == 0 && write.enabled) - writeReschedule(nh); + if (netvc->closed == 0 && netvc->write.enabled) + netvc->writeReschedule(); if (cert) { debug_certificate_name("server certificate subject CN is", X509_get_subject_name(cert)); debug_certificate_name("server certificate issuer CN is", X509_get_issuer_name(cert)); @@ -1243,49 +1286,49 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) } SSL_INCREMENT_DYN_STAT(ssl_total_success_handshake_count_out_stat); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake completed successfully"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake completed successfully"); // do we want to include cert info in trace? sslHandShakeComplete = true; return EVENT_DONE; case SSL_ERROR_WANT_WRITE: - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_WRITE"); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_WANT_WRITE"); SSL_INCREMENT_DYN_STAT(ssl_error_want_write); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_WRITE"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake ERROR_WANT_WRITE"); return SSL_HANDSHAKE_WANT_WRITE; case SSL_ERROR_WANT_READ: SSL_INCREMENT_DYN_STAT(ssl_error_want_read); - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_READ"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_READ"); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_WANT_READ"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake ERROR_WANT_READ"); return SSL_HANDSHAKE_WANT_READ; case SSL_ERROR_WANT_X509_LOOKUP: SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_WANT_X509_LOOKUP"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_X509_LOOKUP"); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_WANT_X509_LOOKUP"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake ERROR_WANT_X509_LOOKUP"); break; case SSL_ERROR_WANT_ACCEPT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_ACCEPT"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake ERROR_WANT_ACCEPT"); return SSL_HANDSHAKE_WANT_ACCEPT; case SSL_ERROR_WANT_CONNECT: - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake ERROR_WANT_CONNECT"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake ERROR_WANT_CONNECT"); break; case SSL_ERROR_ZERO_RETURN: SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, EOS"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake EOS"); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, EOS"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake EOS"); return EVENT_ERROR; case SSL_ERROR_SYSCALL: err = errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, syscall"); - TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL client handshake Syscall Error: %s", strerror(errno)); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, syscall"); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), "SSL client handshake Syscall Error: %s", strerror(errno)); return EVENT_ERROR; break; @@ -1293,14 +1336,14 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) default: { err = errno; // FIXME -- This triggers a retry on cases of cert validation errors.... - Debug("ssl", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); - SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); - Debug("ssl.error", "SSLNetVConnection::sslClientHandShakeEvent, SSL_ERROR_SSL"); + Debug("ssl", "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_SSL"); + SSL_CLR_ERR_INCR_DYN_STAT(vc, ssl_error_ssl, "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_SSL errno=%d", errno); + Debug("ssl.error", "SSLProfileSM::sslClientHandShakeEvent, SSL_ERROR_SSL"); char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, get_remote_addr(), get_remote_port(), - "SSL client handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, e, buf, errno); + error_code = ERR_peek_last_error(); + ERR_error_string_n(error_code, buf, sizeof(buf)); + TraceIn(trace, vc->get_remote_addr(), vc->get_remote_port(), + "SSL client handshake ERROR_SSL: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", ssl_error, error_code, buf, errno); return EVENT_ERROR; } break; } @@ -1308,61 +1351,7 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) } void -SSLNetVConnection::registerNextProtocolSet(const SSLNextProtocolSet *s) -{ - ink_release_assert(this->npnSet == NULL); - this->npnSet = s; -} - -// NextProtocolNegotiation TLS extension callback. The NPN extension -// allows the client to select a preferred protocol, so all we have -// to do here is tell them what out protocol set is. -int -SSLNetVConnection::advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned int *outlen, void * /*arg ATS_UNUSED */) -{ - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - - ink_release_assert(netvc != NULL); - - if (netvc->npnSet && netvc->npnSet->advertiseProtocols(out, outlen)) { - // Successful return tells OpenSSL to advertise. - return SSL_TLSEXT_ERR_OK; - } - - return SSL_TLSEXT_ERR_NOACK; -} - -// ALPN TLS extension callback. Given the client's set of offered -// protocols, we have to select a protocol to use for this session. -int -SSLNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, - const unsigned char *in ATS_UNUSED, unsigned inlen ATS_UNUSED, void *) -{ - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - const unsigned char *npn = NULL; - unsigned npnsz = 0; - - ink_release_assert(netvc != NULL); - - if (netvc->npnSet && netvc->npnSet->advertiseProtocols(&npn, &npnsz)) { -// SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the -// server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short. - -#if HAVE_SSL_SELECT_NEXT_PROTO - if (SSL_select_next_proto((unsigned char **)out, outlen, npn, npnsz, in, inlen) == OPENSSL_NPN_NEGOTIATED) { - Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out); - return SSL_TLSEXT_ERR_OK; - } -#endif /* HAVE_SSL_SELECT_NEXT_PROTO */ - } - - *out = NULL; - *outlen = 0; - return SSL_TLSEXT_ERR_NOACK; -} - -void -SSLNetVConnection::reenable(NetHandler *nh) +SSLProfileSM::reenable() { if (sslPreAcceptHookState != SSL_HOOKS_DONE) { sslPreAcceptHookState = SSL_HOOKS_INVOKE; @@ -1385,27 +1374,16 @@ SSLNetVConnection::reenable(NetHandler *nh) // empty, set state to HOOKS_DONE this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE; } + } else if (this->sslHandshakeDoneHookState == SSL_HOOKS_ACTIVE) { + Debug("ssl", "SSLProfileSM::reenable sslHandshakeDoneHookState = %d, set to SSL_HOOKS_INVOKE", sslHandshakeDoneHookState); + this->sslHandshakeDoneHookState = SSL_HOOKS_INVOKE; } - this->readReschedule(nh); -} - -bool -SSLNetVConnection::sslContextSet(void *ctx) -{ -#if TS_USE_TLS_SNI - bool zret = true; - if (ssl) - SSL_set_SSL_CTX(ssl, static_cast(ctx)); - else - zret = false; -#else - bool zret = false; -#endif - return zret; + vc->readReschedule(); + vc->writeReschedule(); } bool -SSLNetVConnection::callHooks(TSEvent eventId) +SSLProfileSM::callHooks(TSEvent eventId) { // Only dealing with the SNI/CERT hook so far. ink_assert(eventId == TS_EVENT_SSL_CERT); @@ -1440,7 +1418,7 @@ SSLNetVConnection::callHooks(TSEvent eventId) } bool -SSLNetVConnection::computeSSLTrace() +SSLProfileSM::computeSSLTrace() { // this has to happen before the handshake or else sni_servername will be NULL #if TS_USE_TLS_SNI @@ -1458,7 +1436,7 @@ SSLNetVConnection::computeSSLTrace() #endif // count based on ip only if they set an IP value - const sockaddr *remote_addr = get_remote_addr(); + const sockaddr *remote_addr = vc->get_remote_addr(); bool ip_trace = false; if (SSLConfigParams::ssl_wire_trace_ip) { ip_trace = (*SSLConfigParams::ssl_wire_trace_ip == remote_addr); @@ -1489,23 +1467,8 @@ SSLNetVConnection::computeSSLTrace() return trace; } -int -SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg) -{ - int retval = super::populate(con, c, arg); - if (retval != EVENT_DONE) - return retval; - // Add in the SSL data - this->ssl = (SSL *)arg; - // Maybe bring over the stats? - - this->sslHandShakeComplete = true; - SSLNetVCAttach(this->ssl, this); - return EVENT_DONE; -} - const char * -SSLNetVConnection::map_tls_protocol_to_tag(char const *proto_string) const +SSLProfileSM::get_protocol_tag() const { const char *retval = NULL; const char *ssl_proto = getSSLProtocol(); @@ -1524,33 +1487,3 @@ SSLNetVConnection::map_tls_protocol_to_tag(char const *proto_string) const } return retval; } - -int -SSLNetVConnection::populate_protocol(char const **results, int n) const -{ - int retval = 0; - if (n > 0) { - results[0] = map_tls_protocol_to_tag(getSSLProtocol()); - if (results[0] != NULL) { - retval++; - } - if (n > retval) { - retval += super::populate_protocol(results + retval, n - retval); - } - } - return retval; -} - -const char * -SSLNetVConnection::protocol_contains(const char *tag) const -{ - const char *retval = NULL; - const char *tls_tag = map_tls_protocol_to_tag(getSSLProtocol()); - unsigned int tag_len = strlen(tag); - if (tag_len <= strlen(tls_tag) && strncmp(tag, tls_tag, tag_len) == 0) { - retval = tls_tag; - } else { - retval = super::protocol_contains(tag); - } - return retval; -} diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc index d7ef940cbcf..2f1c07ad031 100644 --- a/iocore/net/SSLUtils.cc +++ b/iocore/net/SSLUtils.cc @@ -136,7 +136,7 @@ static int ssl_session_ticket_index = -1; static ssl_ticket_key_block *global_default_keyblock = NULL; #endif -static int ssl_vc_index = -1; +static int ssl_profilesm_index = -1; static ink_mutex *mutex_buf = NULL; static bool open_ssl_initialized = false; @@ -258,9 +258,9 @@ ssl_get_cached_session(SSL *ssl, unsigned char *id, int len, int *copy) session = NULL; #endif } else { - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); SSL_INCREMENT_DYN_STAT(ssl_session_cache_hit); - netvc->setSSLSessionCacheHit(true); + ssl_profile_sm->setSSLSessionCacheHit(true); } } else { SSL_INCREMENT_DYN_STAT(ssl_session_cache_miss); @@ -313,21 +313,23 @@ set_context_cert(SSL *ssl) SSL_CTX *ctx = NULL; SSLCertContext *cc = NULL; SSLCertificateConfig::scoped_config lookup; - const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - bool found = true; - int retval = 1; - - Debug("ssl", "set_context_cert ssl=%p server=%s handshake_complete=%d", ssl, servername, netvc->getSSLHandShakeComplete()); + const char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); + NetVConnection *netvc = ssl_profile_sm->vc; + bool found = true; + int retval = 1; + + Debug("ssl", "set_context_cert ssl=%p server=%s handshake_complete=%d", ssl, servername, + ssl_profile_sm->getSSLHandShakeComplete()); // set SSL trace (we do this a little later in the USE_TLS_SNI case so we can get the servername if (SSLConfigParams::ssl_wire_trace_enabled) { - bool trace = netvc->computeSSLTrace(); - Debug("ssl", "sslnetvc. setting trace to=%s", trace ? "true" : "false"); - netvc->setSSLTrace(trace); + bool trace = ssl_profile_sm->computeSSLTrace(); + Debug("ssl", "netvc with SSLProfileSM. setting trace to=%s", trace ? "true" : "false"); + ssl_profile_sm->setTrace(trace); } // catch the client renegotiation early on - if (SSLConfigParams::ssl_allow_client_renegotiation == false && netvc->getSSLHandShakeComplete()) { + if (SSLConfigParams::ssl_allow_client_renegotiation == false && ssl_profile_sm->getSSLHandShakeComplete()) { Debug("ssl", "set_context_cert trying to renegotiate from the client"); retval = 0; // Error goto done; @@ -342,7 +344,7 @@ set_context_cert(SSL *ssl) ctx = cc->ctx; if (cc && SSLCertContext::OPT_TUNNEL == cc->opt && netvc->get_is_transparent()) { netvc->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL; - netvc->setSSLHandShakeComplete(true); + ssl_profile_sm->setSSLHandShakeComplete(true); retval = -1; goto done; } @@ -390,13 +392,13 @@ set_context_cert(SSL *ssl) static int ssl_cert_callback(SSL *ssl, void * /*arg*/) { - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); bool reenabled; int retval = 1; // Do the common certificate lookup only once. If we pause // and restart processing, do not execute the common logic again - if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { + if (!ssl_profile_sm->calledHooks(TS_EVENT_SSL_CERT)) { retval = set_context_cert(ssl); if (retval != 1) { return retval; @@ -404,7 +406,7 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) } // Call the plugin cert code - reenabled = netvc->callHooks(TS_EVENT_SSL_CERT); + reenabled = ssl_profile_sm->callHooks(TS_EVENT_SSL_CERT); // If it did not re-enable, return the code to // stop the accept processing if (!reenabled) { @@ -418,13 +420,13 @@ ssl_cert_callback(SSL *ssl, void * /*arg*/) static int ssl_servername_callback(SSL *ssl, int * /* ad */, void * /*arg*/) { - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); bool reenabled; int retval = 1; // Do the common certificate lookup only once. If we pause // and restart processing, do not execute the common logic again - if (!netvc->calledHooks(TS_EVENT_SSL_CERT)) { + if (!ssl_profile_sm->calledHooks(TS_EVENT_SSL_CERT)) { retval = set_context_cert(ssl); if (retval != 1) { goto done; @@ -432,7 +434,7 @@ ssl_servername_callback(SSL *ssl, int * /* ad */, void * /*arg*/) } // Call the plugin SNI code - reenabled = netvc->callHooks(TS_EVENT_SSL_CERT); + reenabled = ssl_profile_sm->callHooks(TS_EVENT_SSL_CERT); // If it did not re-enable, return the code to // stop the accept processing if (!reenabled) { @@ -959,7 +961,7 @@ SSLInitializeLibrary() // Reserve an application data index so that we can attach // the SSLNetVConnection to the SSL session. - ssl_vc_index = SSL_get_ex_new_index(0, (void *)"NetVC index", NULL, NULL, NULL); + ssl_profilesm_index = SSL_get_ex_new_index(0, (void *)"NetVC index", NULL, NULL, NULL); open_ssl_initialized = true; } @@ -1252,7 +1254,7 @@ increment_ssl_server_error(unsigned long err) } void -SSLDiagnostic(const SourceLocation &loc, bool debug, SSLNetVConnection *vc, const char *fmt, ...) +SSLDiagnostic(const SourceLocation &loc, bool debug, NetVConnection *vc, const char *fmt, ...) { unsigned long l; char buf[256]; @@ -1496,9 +1498,10 @@ static void ssl_callback_info(const SSL *ssl, int where, int ret) { Debug("ssl", "ssl_callback_info ssl: %p where: %d ret: %d", ssl, where, ret); - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); + Debug("ssl", "SSLProfileSM ssl_profile_sm = %p", ssl_profile_sm); - if ((where & SSL_CB_ACCEPT_LOOP) && netvc->getSSLHandShakeComplete() == true && + if ((where & SSL_CB_ACCEPT_LOOP) && ssl_profile_sm->getSSLHandShakeComplete() == true && SSLConfigParams::ssl_allow_client_renegotiation == false) { int state = SSL_get_state(ssl); @@ -1513,7 +1516,7 @@ ssl_callback_info(const SSL *ssl, int where, int ret) if (state == TLS_ST_SR_CLNT_HELLO) { #endif #endif - netvc->setSSLClientRenegotiationAbort(true); + ssl_profile_sm->setSSLClientRenegotiationAbort(true); Debug("ssl", "ssl_callback_info trying to renegotiate from the client"); } } @@ -1821,11 +1824,11 @@ SSLInitServerContext(const SSLConfigParams *params, const ssl_user_config *sslMu SSL_CTX_set_info_callback(ctx, ssl_callback_info); #if TS_USE_TLS_NPN - SSL_CTX_set_next_protos_advertised_cb(ctx, SSLNetVConnection::advertise_next_protocol, NULL); + SSL_CTX_set_next_protos_advertised_cb(ctx, SSLProfileSM::advertise_next_protocol, NULL); #endif /* TS_USE_TLS_NPN */ #if TS_USE_TLS_ALPN - SSL_CTX_set_alpn_select_cb(ctx, SSLNetVConnection::select_next_protocol, NULL); + SSL_CTX_set_alpn_select_cb(ctx, SSLProfileSM::select_next_protocol, NULL); #endif /* TS_USE_TLS_ALPN */ #ifdef HAVE_OPENSSL_OCSP_STAPLING @@ -2149,12 +2152,12 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, int enc) { SSLCertificateConfig::scoped_config lookup; - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); // Get the IP address to look up the keyblock IpEndpoint ip; int namelen = sizeof(ip); - safe_getsockname(netvc->get_socket(), &ip.sa, &namelen); + safe_getsockname(ssl_profile_sm->vc->get_socket(), &ip.sa, &namelen); SSLCertContext *cc = lookup->find(ip); ssl_ticket_key_block *keyblock = NULL; if (cc == NULL || cc->keyblock == NULL) { @@ -2188,8 +2191,8 @@ ssl_callback_session_ticket(SSL *ssl, unsigned char *keyname, unsigned char *iv, if (i != 0) // The number of tickets decrypted with "older" keys. SSL_INCREMENT_DYN_STAT(ssl_total_tickets_verified_old_key_stat); - SSLNetVConnection *netvc = SSLNetVCAccess(ssl); - netvc->setSSLSessionCacheHit(true); + SSLProfileSM *ssl_profile_sm = (SSLProfileSM *)SSLProfileSMAccess(ssl); + ssl_profile_sm->setSSLSessionCacheHit(true); // When we decrypt with an "older" key, encrypt the ticket again with the most recent key. return (i == 0) ? 1 : 2; } @@ -2212,26 +2215,26 @@ SSLReleaseContext(SSL_CTX *ctx) } void -SSLNetVCAttach(SSL *ssl, SSLNetVConnection *vc) +SSLProfileSMAttach(SSL *ssl, SSLProfileSM *pSM) { - SSL_set_ex_data(ssl, ssl_vc_index, vc); + SSL_set_ex_data(ssl, ssl_profilesm_index, pSM); } void -SSLNetVCDetach(SSL *ssl) +SSLProfileSMDetach(SSL *ssl) { - SSL_set_ex_data(ssl, ssl_vc_index, NULL); + SSL_set_ex_data(ssl, ssl_profilesm_index, NULL); } -SSLNetVConnection * -SSLNetVCAccess(const SSL *ssl) +SSLProfileSM * +SSLProfileSMAccess(const SSL *ssl) { - SSLNetVConnection *netvc; - netvc = static_cast(SSL_get_ex_data(ssl, ssl_vc_index)); + SSLProfileSM *ssl_profile_sm; + ssl_profile_sm = static_cast(SSL_get_ex_data(ssl, ssl_profilesm_index)); - ink_assert(dynamic_cast(static_cast(SSL_get_ex_data(ssl, ssl_vc_index)))); + ink_assert(dynamic_cast(ssl_profile_sm)); - return netvc; + return ssl_profile_sm; } ssl_error_t diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index 2b3964e7608..f01dd7c3650 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -519,11 +519,11 @@ NetHandler::mainNetEvent(int event, Event *e) while ((vc = read_ready_list.dequeue())) { // Initialize the thread-local continuation flags set_cont_flags(vc->control_flags); - if (vc->closed) + if (vc->closed) { close_UnixNetVConnection(vc, trigger_event->ethread); - else if (vc->read.enabled && vc->read.triggered) - vc->net_read_io(this, trigger_event->ethread); - else if (!vc->read.enabled) { + } else if (vc->read.enabled && vc->read.triggered) { + vc->profile_sm->handleEvent(IOCORE_EVENTS_READ, this); + } else if (!vc->read.enabled) { read_ready_list.remove(vc); #if defined(solaris) if (vc->read.triggered && vc->write.enabled) { @@ -536,11 +536,11 @@ NetHandler::mainNetEvent(int event, Event *e) } while ((vc = write_ready_list.dequeue())) { set_cont_flags(vc->control_flags); - if (vc->closed) + if (vc->closed) { close_UnixNetVConnection(vc, trigger_event->ethread); - else if (vc->write.enabled && vc->write.triggered) - write_to_net(this, vc, trigger_event->ethread); - else if (!vc->write.enabled) { + } else if (vc->write.enabled && vc->write.triggered) { + vc->profile_sm->handleEvent(IOCORE_EVENTS_WRITE, this); + } else if (!vc->write.enabled) { write_ready_list.remove(vc); #if defined(solaris) if (vc->write.triggered && vc->read.enabled) { @@ -554,21 +554,23 @@ NetHandler::mainNetEvent(int event, Event *e) #else /* !USE_EDGE_TRIGGER */ while ((vc = read_ready_list.dequeue())) { diags->set_override(vc->control.debug_override); - if (vc->closed) + if (vc->closed) { close_UnixNetVConnection(vc, trigger_event->ethread); - else if (vc->read.enabled && vc->read.triggered) - vc->net_read_io(this, trigger_event->ethread); - else if (!vc->read.enabled) + } else if (vc->read.enabled && vc->read.triggered) { + vc->profile_sm->handleEvent(IOCORE_EVENTS_READ, this); + } else if (!vc->read.enabled) { vc->ep.modify(-EVENTIO_READ); + } } while ((vc = write_ready_list.dequeue())) { diags->set_override(vc->control.debug_override); - if (vc->closed) + if (vc->closed) { close_UnixNetVConnection(vc, trigger_event->ethread); - else if (vc->write.enabled && vc->write.triggered) - write_to_net(this, vc, trigger_event->ethread); - else if (!vc->write.enabled) + } else if (vc->write.enabled && vc->write.triggered) { + vc->profile_sm->handleEvent(IOCORE_EVENTS_WRITE, this); + } else if (!vc->write.enabled) { vc->ep.modify(-EVENTIO_WRITE); + } } #endif /* !USE_EDGE_TRIGGER */ diff --git a/iocore/net/UnixNetAccept.cc b/iocore/net/UnixNetAccept.cc index 8750e03b6e6..7566265aaa0 100644 --- a/iocore/net/UnixNetAccept.cc +++ b/iocore/net/UnixNetAccept.cc @@ -121,6 +121,10 @@ net_accept(NetAccept *na, void *ep, bool blockable) vc->action_ = *na->action_; vc->set_is_transparent(na->server.f_inbound_transparent); vc->set_context(NET_VCONNECTION_IN); + vc->add_profile_sm(PROFILE_SM_TCP, e->ethread); + if (na->isSSL) { + vc->add_profile_sm(PROFILE_SM_SSL, e->ethread); + } SET_CONTINUATION_HANDLER(vc, (NetVConnHandler)&UnixNetVConnection::acceptEvent); if (e->ethread->is_event_type(na->etype)) @@ -305,6 +309,10 @@ NetAccept::do_blocking_accept(EThread *t) vc->options.packet_tos = packet_tos; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + vc->add_profile_sm(PROFILE_SM_TCP, NULL); + if (isSSL) { + vc->add_profile_sm(PROFILE_SM_SSL, NULL); + } SET_CONTINUATION_HANDLER(vc, (NetVConnHandler)&UnixNetVConnection::acceptEvent); // eventProcessor.schedule_imm(vc, getEtype()); eventProcessor.schedule_imm_signal(vc, etype); @@ -446,6 +454,10 @@ NetAccept::acceptFastEvent(int event, void *ep) vc->options.packet_tos = packet_tos; vc->apply_options(); vc->set_context(NET_VCONNECTION_IN); + vc->add_profile_sm(PROFILE_SM_TCP, e->ethread); + if (isSSL) { + vc->add_profile_sm(PROFILE_SM_SSL, e->ethread); + } SET_CONTINUATION_HANDLER(vc, (NetVConnHandler)&UnixNetVConnection::mainEvent); // set thread and nh as acceptEvent does @@ -522,7 +534,8 @@ NetAccept::NetAccept() sockopt_flags(0), packet_mark(0), packet_tos(0), - etype(0) + etype(-1), + isSSL(false) { } diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 67f6b77aaba..6185fe7adb2 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -48,6 +48,7 @@ NetProcessor::AcceptOptions::reset() packet_mark = 0; packet_tos = 0; f_inbound_transparent = false; + isSSL = false; return *this; } @@ -138,6 +139,7 @@ UnixNetProcessor::accept_internal(Continuation *cont, int fd, AcceptOptions cons na->backdoor = opt.backdoor; if (na->callback_on_open) na->mutex = cont->mutex; + na->isSSL = opt.isSSL; if (opt.frequent_accept) { // true if (accept_threads > 0) { if (0 == na->do_listen(BLOCKING, opt.f_inbound_transparent)) { @@ -221,6 +223,7 @@ UnixNetProcessor::connect_re_internal(Continuation *cont, sockaddr const *target vc->submit_time = Thread::get_hrtime(); vc->mutex = cont->mutex; Action *result = &vc->action_; + attach_profile_sm(t, vc); if (using_socks) { char buff[INET6_ADDRPORTSTRLEN]; @@ -459,6 +462,15 @@ UnixNetProcessor::allocate_vc(EThread *t) return vc; } +void +UnixNetProcessor::attach_profile_sm(EThread *t, NetVConnection *vc) +{ + vc->add_profile_sm(PROFILE_SM_TCP, t); + if (vc->options.isSSL) { + vc->add_profile_sm(PROFILE_SM_SSL, t); + } +} + struct socks_conf_struct *NetProcessor::socks_conf_stuff = NULL; int NetProcessor::accept_mss = 0; diff --git a/iocore/net/UnixNetProfileSM.cc b/iocore/net/UnixNetProfileSM.cc new file mode 100644 index 00000000000..0e1ec27691d --- /dev/null +++ b/iocore/net/UnixNetProfileSM.cc @@ -0,0 +1,547 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "P_Net.h" +#include "Log.h" + +// reference from read_from_net() +void +UnixNetProfileSM::handle_read(NetHandler *nh, EThread *lthread) +{ + int64_t r = 0; + UnixNetVConnection *netvc = static_cast(vc); + NetState *s = &netvc->read; + ProxyMutex *tmp_mutex = s->vio.mutex.get(); + + MIOBufferAccessor &buf = s->vio.buffer; + ink_assert(buf.writer()); + + // if there is nothing to do, disable connection + int64_t ntodo = s->vio.ntodo(); + if (ntodo <= 0 || !buf.writer()->write_avail()) { + netvc->readDisable(); + return; + } + + // It is possible that the closed flag got set from HttpSessionManager in the + // global session pool case. If so, the closed flag should be stable once we get the + // s->vio.mutex (the global session pool mutex). + if (netvc->closed) { + close_UnixNetVConnection(netvc, lthread); + return; + } + + int64_t toread = buf.writer()->write_avail(); + if (toread > ntodo) + toread = ntodo; + + // read data + int64_t rattempted = 0, total_read = 0; + r = this->read_from_net(toread, rattempted, total_read, buf); + Debug("iocore_net", "[UnixNetProfileSM::handle_read read_from_net = %ld", r); + // check for errors + if (r <= 0) { + if (r == -EAGAIN || r == -ENOTCONN) { + NET_INCREMENT_DYN_STAT(net_calls_to_read_nodata_stat); + netvc->read.triggered = 0; + nh->read_ready_list.remove(netvc); + return; + } + + if (!r || r == -ECONNRESET) { + netvc->read.triggered = 0; + nh->read_ready_list.remove(netvc); + netvc->readSignalDone(VC_EVENT_EOS); + return; + } + netvc->read.triggered = 0; + netvc->readSignalError((int)-r); + return; + } + NET_SUM_DYN_STAT(net_read_bytes_stat, r); + + // Add data to buffer and signal continuation. + buf.writer()->fill(r); +#ifdef DEBUG + if (buf.writer()->write_avail() <= 0) + Debug("iocore_net", "[UnixNetProfileSM::handle_read] read buffer full"); +#endif + s->vio.ndone += r; + netvc->netActivity(lthread); + + // Signal read ready, check if user is not done + // If there are no more bytes to read, signal read complete + ink_assert(ntodo >= 0); + if (s->vio.ntodo() <= 0) { + netvc->readSignalDone(VC_EVENT_READ_COMPLETE); + Debug("iocore_net", "[UnixNetProfileSM::handle_read] read finished - signal done"); + return; + } else { + if (netvc->readSignalAndUpdate(VC_EVENT_READ_READY) != EVENT_CONT) { + return; + } + + // change of lock... don't look at shared variables! + if (tmp_mutex != s->vio.mutex.get()) { + netvc->readReschedule(); + return; + } + } + // If here are is no more room, or nothing to do, disable the connection + if (s->vio.ntodo() <= 0 || !s->enabled || !buf.writer()->write_avail()) { + netvc->readDisable(); + return; + } + + netvc->readReschedule(); +} + +// reference from write_to_net_io() +void +UnixNetProfileSM::handle_write(NetHandler *nh, EThread *lthread) +{ + NET_INCREMENT_DYN_STAT(net_calls_to_writetonet_stat); + NET_INCREMENT_DYN_STAT(net_calls_to_writetonet_afterpoll_stat); + + UnixNetVConnection *netvc = static_cast(vc); + NetState *s = &netvc->write; + ProxyMutex *tmp_mutex = s->vio.mutex.get(); + + // If there is nothing to do, disable + int64_t ntodo = s->vio.ntodo(); + if (ntodo <= 0) { + netvc->writeDisable(); + return; + } + + MIOBufferAccessor &buf = s->vio.buffer; + ink_assert(buf.writer()); + + // Calculate amount to write + int64_t towrite = buf.reader()->read_avail(); + if (towrite > ntodo) + towrite = ntodo; + int signalled = 0; + + // signal write ready to allow user to fill the buffer + if (towrite != ntodo && buf.writer()->write_avail()) { + if (netvc->writeSignalAndUpdate(VC_EVENT_WRITE_READY) != EVENT_CONT) { + return; + } + // TODO: change of lock check. + ntodo = s->vio.ntodo(); + if (ntodo <= 0) { + netvc->writeDisable(); + return; + } + signalled = 1; + // Recalculate amount to write + towrite = buf.reader()->read_avail(); + if (towrite > ntodo) + towrite = ntodo; + } + // if there is nothing to do, disable + ink_assert(towrite >= 0); + if (towrite <= 0) { + netvc->writeDisable(); + return; + } + + int needs = 0; + int64_t total_written = 0; + int64_t r = this->load_buffer_and_write(towrite, buf, total_written, needs); + + if (total_written > 0) { + NET_SUM_DYN_STAT(net_write_bytes_stat, total_written); + s->vio.ndone += total_written; + } + + // A write of 0 makes no sense since we tried to write more than 0. + ink_assert(r != 0); + // Either we wrote something or got an error. + // check for errors + if (r < 0) { // if the socket was not ready,add to WaitList + if (r == -EAGAIN || r == -ENOTCONN) { + NET_INCREMENT_DYN_STAT(net_calls_to_write_nodata_stat); + if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { + netvc->write.triggered = 0; + nh->write_ready_list.remove(netvc); + netvc->writeReschedule(); + } + if ((needs & EVENTIO_READ) == EVENTIO_READ) { + netvc->read.triggered = 0; + nh->read_ready_list.remove(netvc); + netvc->readReschedule(); + } + return; + } + netvc->write.triggered = 0; + netvc->writeSignalError((int)-total_written); + return; + } else { // Wrote data. Finished without error + int wbe_event = netvc->getWriteBufferEmpty(); // save so we can clear if needed. + + // If the empty write buffer trap is set, clear it. + if (!(buf.reader()->is_read_avail_more_than(0))) + netvc->trapWriteBufferEmpty(0); + + netvc->netActivity(lthread); + // If there are no more bytes to write, signal write complete, + ink_assert(ntodo >= 0); + if (s->vio.ntodo() <= 0) { + netvc->writeSignalDone(VC_EVENT_WRITE_COMPLETE); + return; + } + int e = 0; + if (!signalled) { + e = VC_EVENT_WRITE_READY; + } else if (wbe_event != vc->getWriteBufferEmpty()) { + // @a signalled means we won't send an event, and the event values differing means we + // had a write buffer trap and cleared it, so we need to send it now. + e = wbe_event; + } + if (e) { + if (netvc->writeSignalAndUpdate(e) != EVENT_CONT) { + return; + } + + // change of lock... don't look at shared variables! + if (tmp_mutex != s->vio.mutex.get()) { + netvc->writeReschedule(); + return; + } + } + + if ((needs & EVENTIO_READ) == EVENTIO_READ) { + netvc->readReschedule(); + } + + if (!(buf.reader()->is_read_avail_more_than(0))) { + netvc->writeDisable(); + return; + } + + if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { + netvc->writeReschedule(); + } + return; + } +} + +// Global +ClassAllocator tcpProfileSMAllocator("tcpProfileSMAllocator"); + +TcpProfileSM::TcpProfileSM() : UnixNetProfileSM(NULL) +{ + type = PROFILE_SM_TCP; + SET_HANDLER(&TcpProfileSM::mainEvent); +} + +int +TcpProfileSM::mainEvent(int event, void *data) +{ + NetHandler *nh = static_cast(data); + UnixNetVConnection *netvc = static_cast(vc); + + NetState *s = NULL; + switch (event) { + case IOCORE_EVENTS_READ: + s = &netvc->read; + break; + case IOCORE_EVENTS_WRITE: + s = &netvc->write; + break; + default: + ink_release_assert(!"should not reached"); + break; + } + + MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, nh->trigger_event->ethread, s->vio._cont); + + if (!lock.is_locked()) { + switch (event) { + case IOCORE_EVENTS_READ: + netvc->readReschedule(); + break; + case IOCORE_EVENTS_WRITE: + netvc->writeReschedule(); + break; + default: + ink_release_assert(!"should not reached"); + break; + } + return EVENT_DONE; + } + ink_release_assert(lock.get_mutex() == s->vio.mutex.get()); + + switch (event) { + case IOCORE_EVENTS_READ: + if (s->enabled && s->vio.op == VIO::READ) + handle_read(nh, nh->trigger_event->ethread); + else + netvc->readDisable(); + break; + case IOCORE_EVENTS_WRITE: + if (s->enabled && s->vio.op == VIO::WRITE) + handle_write(nh, nh->trigger_event->ethread); + else + netvc->writeDisable(); + break; + default: + ink_release_assert(!"should not reached"); + break; + } + return EVENT_DONE; +} + +TcpProfileSM * +TcpProfileSM::allocate(EThread *t) +{ + TcpProfileSM *tcp_profile; + + if (t) { + tcp_profile = THREAD_ALLOC_INIT(tcpProfileSMAllocator, t); + } else { + if (likely(tcp_profile = tcpProfileSMAllocator.alloc())) + tcp_profile->globally_allocated = true; + } + return tcp_profile; +} + +void +TcpProfileSM::free(EThread *t) +{ + NetProfileSM::clear(); + + if (globally_allocated) { + tcpProfileSMAllocator.free(this); + } else { + THREAD_FREE(this, tcpProfileSMAllocator, t); + } +} +int64_t +TcpProfileSM::read(void *buf, int64_t size, int &err) +{ + int64_t nread = socketManager.read(vc->get_socket(), buf, size); + err = errno; + + if (vc->getOriginTrace()) { + char origin_trace_ip[INET6_ADDRSTRLEN]; + + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + + if (nread > 0) { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)nread, (int)nread, (char *)buf); + + } else if (nread == 0) { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d closed connection", origin_trace_ip, + vc->getOriginTraceAddr()->port()); + } else { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), strerror(errno)); + } + } + return nread; +} + +int64_t +TcpProfileSM::readv(struct iovec *vector, int count) +{ + int64_t nread = socketManager.readv(vc->get_socket(), vector, count); + + if (vc->getOriginTrace()) { + char origin_trace_ip[INET6_ADDRSTRLEN]; + + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + + if (nread > 0) { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)nread, (int)nread, (char *)vector[0].iov_base); + + } else if (nread == 0) { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d closed connection", origin_trace_ip, + vc->getOriginTraceAddr()->port()); + } else { + TraceIn(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), strerror(errno)); + } + } + return nread; +} + +int64_t +TcpProfileSM::write(void *buf, int64_t size, int &err) +{ + int64_t nwritten = socketManager.write(vc->get_socket(), buf, size); + err = errno; + + if (vc->getOriginTrace()) { + char origin_trace_ip[INET6_ADDRSTRLEN]; + + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + + if (nwritten > 0) { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)nwritten, (int)nwritten, (char *)buf); + + } else if (nwritten == 0) { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=0", origin_trace_ip, + vc->getOriginTraceAddr()->port()); + } else { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), strerror(errno)); + } + } + return nwritten; +} + +int64_t +TcpProfileSM::writev(struct iovec *vector, int count) +{ + int64_t nwritten = socketManager.writev(vc->get_socket(), vector, count); + + if (vc->getOriginTrace()) { + char origin_trace_ip[INET6_ADDRSTRLEN]; + + ats_ip_ntop(vc->getOriginTraceAddr(), origin_trace_ip, sizeof(origin_trace_ip)); + + if (nwritten > 0) { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), (int)nwritten, (int)nwritten, (char *)vector[0].iov_base); + + } else if (nwritten == 0) { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=0", origin_trace_ip, + vc->getOriginTraceAddr()->port()); + } else { + TraceOut(vc->getOriginTrace(), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, + vc->getOriginTraceAddr()->port(), strerror(errno)); + } + } + return nwritten; +} + +int64_t +TcpProfileSM::read_from_net(int64_t toread, int64_t &rattempted, int64_t &total_read, MIOBufferAccessor &buf) +{ + int64_t r = 0; + unsigned niov = 0; + IOVec tiovec[NET_MAX_IOV]; + if (toread) { + IOBufferBlock *b = buf.writer()->first_write_block(); + do { + niov = 0; + rattempted = 0; + while (b && niov < NET_MAX_IOV) { + int64_t a = b->write_avail(); + if (a > 0) { + tiovec[niov].iov_base = b->_end; + int64_t togo = toread - total_read - rattempted; + if (a > togo) + a = togo; + tiovec[niov].iov_len = a; + rattempted += a; + niov++; + if (a >= togo) + break; + } + b = b->next.get(); + } + + ink_assert(niov > 0); + ink_assert(niov <= countof(tiovec)); + r = this->readv(&tiovec[0], niov); + + NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); + + total_read += rattempted; + } while (rattempted && r == rattempted && total_read < toread); + + // if we have already moved some bytes successfully, summarize in r + if (total_read != rattempted) { + if (r <= 0) + r = total_read - rattempted; + else + r = total_read - rattempted + r; + } + } else + r = 0; + + return r; +} + +int64_t +TcpProfileSM::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) +{ + int64_t r = 0; + int64_t try_to_write = 0; + IOBufferReader *tmp_reader = buf.reader()->clone(); + + do { + IOVec tiovec[NET_MAX_IOV]; + unsigned niov = 0; + try_to_write = 0; + while (niov < NET_MAX_IOV) { + // check if we have done this block + int64_t l = tmp_reader->block_read_avail(); + if (l <= 0) + break; + char *current_block = tmp_reader->start(); + + // check if to amount to write exceeds that in this buffer + int64_t wavail = towrite - total_written; + if (l > wavail) { + l = wavail; + } + + if (!l) { + break; + } + + // build an iov entry + tiovec[niov].iov_len = l; + try_to_write += l; + tiovec[niov].iov_base = current_block; + niov++; + tmp_reader->consume(l); + } + + ink_assert(niov > 0); + ink_assert(niov <= countof(tiovec)); + r = this->writev(&tiovec[0], niov); + + if (r > 0) { + buf.reader()->consume(r); + } + total_written += r; + + ProxyMutex *mutex = vc->thread->mutex.get(); + NET_INCREMENT_DYN_STAT(net_calls_to_write_stat); + } while (r == try_to_write && total_written < towrite); + + tmp_reader->dealloc(); + + needs |= EVENTIO_WRITE; + + return r; +} diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 9010820846c..320b27781d8 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -235,376 +235,6 @@ write_signal_error(NetHandler *nh, UnixNetVConnection *vc, int lerrno) return write_signal_done(VC_EVENT_ERROR, nh, vc); } -// Read the data for a UnixNetVConnection. -// Rescheduling the UnixNetVConnection by moving the VC -// onto or off of the ready_list. -// Had to wrap this function with net_read_io for SSL. -static void -read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) -{ - NetState *s = &vc->read; - ProxyMutex *mutex = thread->mutex.get(); - int64_t r = 0; - - MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, thread, s->vio._cont); - - if (!lock.is_locked()) { - read_reschedule(nh, vc); - return; - } - - // It is possible that the closed flag got set from HttpSessionManager in the - // global session pool case. If so, the closed flag should be stable once we get the - // s->vio.mutex (the global session pool mutex). - if (vc->closed) { - close_UnixNetVConnection(vc, thread); - return; - } - // if it is not enabled. - if (!s->enabled || s->vio.op != VIO::READ) { - read_disable(nh, vc); - return; - } - - MIOBufferAccessor &buf = s->vio.buffer; - ink_assert(buf.writer()); - - // if there is nothing to do, disable connection - int64_t ntodo = s->vio.ntodo(); - if (ntodo <= 0) { - read_disable(nh, vc); - return; - } - int64_t toread = buf.writer()->write_avail(); - if (toread > ntodo) - toread = ntodo; - - // read data - int64_t rattempted = 0, total_read = 0; - unsigned niov = 0; - IOVec tiovec[NET_MAX_IOV]; - if (toread) { - IOBufferBlock *b = buf.writer()->first_write_block(); - do { - niov = 0; - rattempted = 0; - while (b && niov < NET_MAX_IOV) { - int64_t a = b->write_avail(); - if (a > 0) { - tiovec[niov].iov_base = b->_end; - int64_t togo = toread - total_read - rattempted; - if (a > togo) - a = togo; - tiovec[niov].iov_len = a; - rattempted += a; - niov++; - if (a >= togo) - break; - } - b = b->next.get(); - } - - ink_assert(niov > 0); - ink_assert(niov <= countof(tiovec)); - r = socketManager.readv(vc->con.fd, &tiovec[0], niov); - - NET_INCREMENT_DYN_STAT(net_calls_to_read_stat); - - if (vc->origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - - ats_ip_ntop(vc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - vc->origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d closed connection", - origin_trace_ip, vc->origin_trace_port); - } else { - TraceIn((vc->origin_trace), vc->get_remote_addr(), vc->get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, - vc->origin_trace_port, strerror(errno)); - } - } - - total_read += rattempted; - } while (rattempted && r == rattempted && total_read < toread); - - // if we have already moved some bytes successfully, summarize in r - if (total_read != rattempted) { - if (r <= 0) - r = total_read - rattempted; - else - r = total_read - rattempted + r; - } - // check for errors - if (r <= 0) { - if (r == -EAGAIN || r == -ENOTCONN) { - NET_INCREMENT_DYN_STAT(net_calls_to_read_nodata_stat); - vc->read.triggered = 0; - nh->read_ready_list.remove(vc); - return; - } - - if (!r || r == -ECONNRESET) { - vc->read.triggered = 0; - nh->read_ready_list.remove(vc); - read_signal_done(VC_EVENT_EOS, nh, vc); - return; - } - vc->read.triggered = 0; - read_signal_error(nh, vc, (int)-r); - return; - } - NET_SUM_DYN_STAT(net_read_bytes_stat, r); - - // Add data to buffer and signal continuation. - buf.writer()->fill(r); -#ifdef DEBUG - if (buf.writer()->write_avail() <= 0) - Debug("iocore_net", "read_from_net, read buffer full"); -#endif - s->vio.ndone += r; - net_activity(vc, thread); - } else - r = 0; - - // Signal read ready, check if user is not done - if (r) { - // If there are no more bytes to read, signal read complete - ink_assert(ntodo >= 0); - if (s->vio.ntodo() <= 0) { - read_signal_done(VC_EVENT_READ_COMPLETE, nh, vc); - Debug("iocore_net", "read_from_net, read finished - signal done"); - return; - } else { - if (read_signal_and_update(VC_EVENT_READ_READY, vc) != EVENT_CONT) { - return; - } - - // change of lock... don't look at shared variables! - if (lock.get_mutex() != s->vio.mutex.get()) { - read_reschedule(nh, vc); - return; - } - } - } - // If here are is no more room, or nothing to do, disable the connection - if (s->vio.ntodo() <= 0 || !s->enabled || !buf.writer()->write_avail()) { - read_disable(nh, vc); - return; - } - - read_reschedule(nh, vc); -} - -// -// Write the data for a UnixNetVConnection. -// Rescheduling the UnixNetVConnection when necessary. -// -void -write_to_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) -{ - ProxyMutex *mutex = thread->mutex.get(); - - NET_INCREMENT_DYN_STAT(net_calls_to_writetonet_stat); - NET_INCREMENT_DYN_STAT(net_calls_to_writetonet_afterpoll_stat); - - write_to_net_io(nh, vc, thread); -} - -void -write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) -{ - NetState *s = &vc->write; - ProxyMutex *mutex = thread->mutex.get(); - - MUTEX_TRY_LOCK_FOR(lock, s->vio.mutex, thread, s->vio._cont); - - if (!lock.is_locked() || lock.get_mutex() != s->vio.mutex.get()) { - write_reschedule(nh, vc); - return; - } - - // This function will always return true unless - // vc is an SSLNetVConnection. - if (!vc->getSSLHandShakeComplete()) { - int err, ret; - - if (vc->get_context() == NET_VCONNECTION_OUT) { - ret = vc->sslStartHandShake(SSL_EVENT_CLIENT, err); - } else { - ret = vc->sslStartHandShake(SSL_EVENT_SERVER, err); - } - - if (ret == EVENT_ERROR) { - vc->write.triggered = 0; - write_signal_error(nh, vc, err); - } else if (ret == SSL_HANDSHAKE_WANT_READ || ret == SSL_HANDSHAKE_WANT_ACCEPT) { - vc->read.triggered = 0; - nh->read_ready_list.remove(vc); - read_reschedule(nh, vc); - } else if (ret == SSL_HANDSHAKE_WANT_CONNECT || ret == SSL_HANDSHAKE_WANT_WRITE) { - vc->write.triggered = 0; - nh->write_ready_list.remove(vc); - write_reschedule(nh, vc); - } else if (ret == EVENT_DONE) { - vc->write.triggered = 1; - if (vc->write.enabled) { - nh->write_ready_list.in_or_enqueue(vc); - } - } else { - write_reschedule(nh, vc); - } - - return; - } - - // If it is not enabled,add to WaitList. - if (!s->enabled || s->vio.op != VIO::WRITE) { - write_disable(nh, vc); - return; - } - - // If there is nothing to do, disable - int64_t ntodo = s->vio.ntodo(); - if (ntodo <= 0) { - write_disable(nh, vc); - return; - } - - MIOBufferAccessor &buf = s->vio.buffer; - ink_assert(buf.writer()); - - // Calculate the amount to write. - int64_t towrite = buf.reader()->read_avail(); - if (towrite > ntodo) { - towrite = ntodo; - } - - int signalled = 0; - - // signal write ready to allow user to fill the buffer - if (towrite != ntodo && buf.writer()->write_avail()) { - if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) { - return; - } - - ntodo = s->vio.ntodo(); - if (ntodo <= 0) { - write_disable(nh, vc); - return; - } - - signalled = 1; - - // Recalculate amount to write - towrite = buf.reader()->read_avail(); - if (towrite > ntodo) { - towrite = ntodo; - } - } - - // if there is nothing to do, disable - ink_assert(towrite >= 0); - if (towrite <= 0) { - write_disable(nh, vc); - return; - } - - int needs = 0; - int64_t total_written = 0; - int64_t r = vc->load_buffer_and_write(towrite, buf, total_written, needs); - - if (total_written > 0) { - NET_SUM_DYN_STAT(net_write_bytes_stat, total_written); - s->vio.ndone += total_written; - } - - // check for errors - if (r <= 0) { // if the socket was not ready,add to WaitList - if (r == -EAGAIN || r == -ENOTCONN) { - NET_INCREMENT_DYN_STAT(net_calls_to_write_nodata_stat); - if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { - vc->write.triggered = 0; - nh->write_ready_list.remove(vc); - write_reschedule(nh, vc); - } - - if ((needs & EVENTIO_READ) == EVENTIO_READ) { - vc->read.triggered = 0; - nh->read_ready_list.remove(vc); - read_reschedule(nh, vc); - } - - return; - } - - if (!r || r == -ECONNRESET) { - vc->write.triggered = 0; - write_signal_done(VC_EVENT_EOS, nh, vc); - return; - } - - vc->write.triggered = 0; - write_signal_error(nh, vc, (int)-total_written); - return; - } else { // Wrote data. Finished without error - int wbe_event = vc->write_buffer_empty_event; // save so we can clear if needed. - - // If the empty write buffer trap is set, clear it. - if (!(buf.reader()->is_read_avail_more_than(0))) { - vc->write_buffer_empty_event = 0; - } - - net_activity(vc, thread); - - // If there are no more bytes to write, signal write complete, - ink_assert(ntodo >= 0); - if (s->vio.ntodo() <= 0) { - write_signal_done(VC_EVENT_WRITE_COMPLETE, nh, vc); - return; - } - - int e = 0; - if (!signalled) { - e = VC_EVENT_WRITE_READY; - } else if (wbe_event != vc->write_buffer_empty_event) { - // @a signalled means we won't send an event, and the event values differing means we - // had a write buffer trap and cleared it, so we need to send it now. - e = wbe_event; - } - - if (e) { - if (write_signal_and_update(e, vc) != EVENT_CONT) { - return; - } - - // change of lock... don't look at shared variables! - if (lock.get_mutex() != s->vio.mutex.get()) { - write_reschedule(nh, vc); - return; - } - } - - if ((needs & EVENTIO_READ) == EVENTIO_READ) { - read_reschedule(nh, vc); - } - - if (!(buf.reader()->is_read_avail_more_than(0))) { - write_disable(nh, vc); - return; - } - - if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { - write_reschedule(nh, vc); - } - - return; - } -} - bool UnixNetVConnection::get_data(int id, void *data) { @@ -878,14 +508,14 @@ UnixNetVConnection::reenable_re(VIO *vio) ep.modify(EVENTIO_READ); ep.refresh(EVENTIO_READ); if (read.triggered) - net_read_io(nh, t); + this->profile_sm->handleEvent(IOCORE_EVENTS_READ, nh); else nh->read_ready_list.remove(this); } else { ep.modify(EVENTIO_WRITE); ep.refresh(EVENTIO_WRITE); if (write.triggered) - write_to_net(nh, this, t); + this->profile_sm->handleEvent(IOCORE_EVENTS_WRITE, nh); else nh->write_ready_list.remove(this); } @@ -910,10 +540,7 @@ UnixNetVConnection::UnixNetVConnection() recursion(0), submit_time(0), oob_ptr(0), - from_accept_thread(false), - origin_trace(false), - origin_trace_addr(NULL), - origin_trace_port(0) + from_accept_thread(false) { SET_HANDLER((NetVConnHandler)&UnixNetVConnection::startEvent); } @@ -940,126 +567,64 @@ UnixNetVConnection::set_enabled(VIO *vio) } void -UnixNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) +UnixNetVConnection::readDisable() { - read_from_net(nh, this, lthread); + read_disable(nh, this); } -// This code was pulled out of write_to_net so -// I could overwrite it for the SSL implementation -// (SSL read does not support overlapped i/o) -// without duplicating all the code in write_to_net. -int64_t -UnixNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) +void +UnixNetVConnection::writeDisable() { - int64_t r = 0; - int64_t try_to_write = 0; - IOBufferReader *tmp_reader = buf.reader()->clone(); - - do { - IOVec tiovec[NET_MAX_IOV]; - unsigned niov = 0; - try_to_write = 0; - - while (niov < NET_MAX_IOV) { - int64_t wavail = towrite - total_written; - int64_t len = tmp_reader->block_read_avail(); - - // Check if we have done this block. - if (len <= 0) { - break; - } - - // Check if the amount to write exceeds that in this buffer. - if (len > wavail) { - len = wavail; - } - - if (len == 0) { - break; - } - - // build an iov entry - tiovec[niov].iov_len = len; - tiovec[niov].iov_base = tmp_reader->start(); - niov++; - - try_to_write += len; - tmp_reader->consume(len); - } - - ink_assert(niov > 0); - ink_assert(niov <= countof(tiovec)); - r = socketManager.writev(con.fd, &tiovec[0], niov); - - if (origin_trace) { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - - if (r > 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d\tbytes=%d\n%.*s", origin_trace_ip, - origin_trace_port, (int)r, (int)r, (char *)tiovec[0].iov_base); - - } else if (r == 0) { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d closed connection", origin_trace_ip, - origin_trace_port); - } else { - TraceOut(origin_trace, get_remote_addr(), get_remote_port(), "CLIENT %s:%d error=%s", origin_trace_ip, origin_trace_port, - strerror(errno)); - } - } - - if (r > 0) { - buf.reader()->consume(r); - total_written += r; - } - - ProxyMutex *mutex = thread->mutex.get(); - NET_INCREMENT_DYN_STAT(net_calls_to_write_stat); - } while (r == try_to_write && total_written < towrite); - - tmp_reader->dealloc(); - - needs |= EVENTIO_WRITE; - - return r; + write_disable(nh, this); } void -UnixNetVConnection::readDisable(NetHandler *nh) +UnixNetVConnection::readSignalError(int err) { - read_disable(nh, this); + read_signal_error(nh, this, err); } void -UnixNetVConnection::readSignalError(NetHandler *nh, int err) +UnixNetVConnection::writeSignalError(int err) { - read_signal_error(nh, this, err); + write_signal_error(nh, this, err); } int -UnixNetVConnection::readSignalDone(int event, NetHandler *nh) +UnixNetVConnection::readSignalDone(int event) { return (read_signal_done(event, nh, this)); } +int +UnixNetVConnection::writeSignalDone(int event) +{ + return (write_signal_done(event, nh, this)); +} + int UnixNetVConnection::readSignalAndUpdate(int event) { return (read_signal_and_update(event, this)); } +int +UnixNetVConnection::writeSignalAndUpdate(int event) +{ + return (write_signal_and_update(event, this)); +} + // Interface so SSL inherited class can call some static in-line functions // without affecting regular net stuff or copying a bunch of code into // the header files. void -UnixNetVConnection::readReschedule(NetHandler *nh) +UnixNetVConnection::readReschedule() { read_reschedule(nh, this); } void -UnixNetVConnection::writeReschedule(NetHandler *nh) +UnixNetVConnection::writeReschedule() { write_reschedule(nh, this); } @@ -1350,6 +915,7 @@ UnixNetVConnection::free(EThread *t) { ink_release_assert(t == this_ethread()); // clear variables for reuse + free_profile_sm(t); this->mutex.clear(); action_.mutex.clear(); got_remote_addr = 0; @@ -1381,6 +947,7 @@ UnixNetVConnection::free(EThread *t) #endif ink_assert(con.fd == NO_FD); ink_assert(t == this_ethread()); + netvc_context = NET_VCONNECTION_UNSET; if (from_accept_thread) { netVCAllocator.free(this); @@ -1411,12 +978,15 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) Connection hold_con; hold_con.move(this->con); - SSLNetVConnection *sslvc = dynamic_cast(this); - - SSL *save_ssl = (sslvc) ? sslvc->ssl : NULL; + SSL *save_ssl = NULL; + SSLProfileSM *ssl_profile_sm = NULL; + if (this->profile_sm->get_type() == PROFILE_SM_SSL) { + ssl_profile_sm = (SSLProfileSM *)(this->profile_sm); + save_ssl = ssl_profile_sm->ssl; + } if (save_ssl) { - SSLNetVCDetach(sslvc->ssl); - sslvc->ssl = NULL; + SSLProfileSMDetach(save_ssl); + ssl_profile_sm->ssl = NULL; } // Do_io_close will signal the VC to be freed on the original thread @@ -1427,26 +997,23 @@ UnixNetVConnection::migrateToCurrentThread(Continuation *cont, EThread *t) this->do_io_close(); // Create new VC: - if (save_ssl) { - SSLNetVConnection *sslvc = static_cast(sslNetProcessor.allocate_vc(t)); - if (sslvc->populate(hold_con, cont, save_ssl) != EVENT_DONE) { - sslvc->do_io_close(); - sslvc = NULL; - } else { - sslvc->set_context(get_context()); - } - return sslvc; - // Update the SSL fields + UnixNetVConnection *netvc = static_cast(netProcessor.allocate_vc(t)); + if (netvc->populate(hold_con, cont, save_ssl) != EVENT_DONE) { + netvc->do_io_close(); + netvc = NULL; } else { - UnixNetVConnection *netvc = static_cast(netProcessor.allocate_vc(t)); - if (netvc->populate(hold_con, cont, save_ssl) != EVENT_DONE) { - netvc->do_io_close(); - netvc = NULL; - } else { - netvc->set_context(get_context()); + netvc->set_context(get_context()); + if (save_ssl) + netvc->options.isSSL = true; + unix_netProcessor.attach_profile_sm(t, netvc); + if (save_ssl) { + ssl_profile_sm = (SSLProfileSM *)(netvc->profile_sm); + ssl_profile_sm->ssl = save_ssl; + ssl_profile_sm->sslHandShakeComplete = true; + SSLProfileSMAttach(save_ssl, ssl_profile_sm); } - return netvc; } + return netvc; } void diff --git a/lib/ts/apidefs.h.in b/lib/ts/apidefs.h.in index 885a7f874c9..2661824e268 100644 --- a/lib/ts/apidefs.h.in +++ b/lib/ts/apidefs.h.in @@ -288,7 +288,9 @@ typedef enum { TS_VCONN_PRE_ACCEPT_HOOK = TS_SSL_FIRST_HOOK, TS_SSL_SNI_HOOK, TS_SSL_CERT_HOOK = TS_SSL_SNI_HOOK, - TS_SSL_LAST_HOOK = TS_SSL_CERT_HOOK, + TS_SSL_CLIENT_HANDSHAKE_HOOK, // Client Side SSL Handshake Done + TS_SSL_SERVER_HANDSHAKE_HOOK, // Server Side SSL Handshake Done + TS_SSL_LAST_HOOK = TS_SSL_SERVER_HANDSHAKE_HOOK, TS_HTTP_LAST_HOOK } TSHttpHookID; @@ -448,7 +450,9 @@ typedef enum { TS_EVENT_INTERNAL_60200 = 60200, TS_EVENT_INTERNAL_60201 = 60201, TS_EVENT_INTERNAL_60202 = 60202, - TS_EVENT_SSL_CERT = 60203 + TS_EVENT_SSL_CERT = 60203, + TS_EVENT_SSL_CLIENT_HANDSHAKE = 60204, + TS_EVENT_SSL_SERVER_HANDSHAKE = 60205, } TSEvent; #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ @@ -1154,7 +1158,6 @@ extern tsapi const char *const TS_PROTO_TAG_UDP; extern tsapi const char *const TS_PROTO_TAG_IPV4; extern tsapi const char *const TS_PROTO_TAG_IPV6; - /* -------------------------------------------------------------------------- MLoc Constants */ /** diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc index 30e5fc7733b..deb0fca7cca 100644 --- a/proxy/InkAPI.cc +++ b/proxy/InkAPI.cc @@ -5391,12 +5391,13 @@ TSHttpSsnSSLConnectionGet(TSHttpSsn ssnp) return NULL; } - SSLNetVConnection *ssl_vc = dynamic_cast(cs->get_netvc()); - if (ssl_vc == NULL) { - return NULL; + NetVConnection *vc = cs->get_netvc(); + if (vc->profile_sm->get_type() == PROFILE_SM_SSL) { + SSLM *sslm = dynamic_cast(vc->profile_sm); + return sslm->ssl; } - return (void *)ssl_vc->ssl; + return NULL; } sockaddr const * @@ -9045,32 +9046,37 @@ TSHttpEventNameLookup(TSEvent event) return HttpDebugNames::get_event_name(static_cast(event)); } -/// Re-enable SSL VC. -class TSSslCallback : public Continuation +/// Re-enable NetVC with NetProfileSM. +class TSNetProfileSMCallback : public Continuation { public: - TSSslCallback(SSLNetVConnection *vc) : Continuation(vc->mutex), m_vc(vc) { SET_HANDLER(&TSSslCallback::event_handler); } + TSNetProfileSMCallback(NetProfileSM *profile_sm) : Continuation(profile_sm->mutex), sm(profile_sm) + { + SET_HANDLER(&TSNetProfileSMCallback::event_handler); + } + int event_handler(int, void *) { - m_vc->reenable(m_vc->nh); + sm->reenable(); delete this; return 0; } private: - SSLNetVConnection *m_vc; + NetProfileSM *sm; }; /// SSL Hooks TSReturnCode TSVConnTunnel(TSVConn sslp) { - NetVConnection *vc = reinterpret_cast(sslp); - SSLNetVConnection *ssl_vc = dynamic_cast(vc); - TSReturnCode zret = TS_SUCCESS; - if (0 != ssl_vc) { - ssl_vc->hookOpRequested = SSL_HOOK_OP_TUNNEL; + NetVConnection *vc = reinterpret_cast(sslp); + TSReturnCode zret = TS_SUCCESS; + + if (vc->profile_sm->get_type() == PROFILE_SM_SSL) { + SSLM *sslm = dynamic_cast(vc->profile_sm); + sslm->hookOpRequested = SSL_HOOK_OP_TUNNEL; } else { zret = TS_ERROR; } @@ -9080,13 +9086,13 @@ TSVConnTunnel(TSVConn sslp) TSSslConnection TSVConnSSLConnectionGet(TSVConn sslp) { - TSSslConnection ssl = NULL; - NetVConnection *vc = reinterpret_cast(sslp); - SSLNetVConnection *ssl_vc = dynamic_cast(vc); - if (ssl_vc != NULL) { - ssl = reinterpret_cast(ssl_vc->ssl); + NetVConnection *vc = reinterpret_cast(sslp); + + if (vc->profile_sm->get_type() == PROFILE_SM_SSL) { + SSLM *sslm = dynamic_cast(vc->profile_sm); + return reinterpret_cast(sslm->ssl); } - return ssl; + return NULL; } tsapi TSSslContext @@ -9141,35 +9147,37 @@ TSSslContextDestroy(TSSslContext ctx) tsapi int TSVConnIsSsl(TSVConn sslp) { - NetVConnection *vc = reinterpret_cast(sslp); - SSLNetVConnection *ssl_vc = dynamic_cast(vc); - return ssl_vc != NULL; + NetVConnection *vc = reinterpret_cast(sslp); + if (vc->profile_sm->get_type() == PROFILE_SM_SSL) { + return true; + } + + return false; } void TSVConnReenable(TSVConn vconn) { - NetVConnection *vc = reinterpret_cast(vconn); - SSLNetVConnection *ssl_vc = dynamic_cast(vc); - // We really only deal with a SSLNetVConnection at the moment - if (ssl_vc != NULL) { + NetVConnection *vc = reinterpret_cast(vconn); + // We really only deal with a NetVConnection with SSLProfileSM at the moment + if (vc->profile_sm->get_type() == PROFILE_SM_SSL) { EThread *eth = this_ethread(); - bool reschedule = eth != ssl_vc->thread; + bool reschedule = eth != vc->thread; if (!reschedule) { // We use the VC mutex so we don't need to reschedule again if we // can't get the lock. For this reason we need to execute the // callback on the VC thread or it doesn't work (not sure why - // deadlock or it ends up interacting with the wrong NetHandler). - MUTEX_TRY_LOCK(trylock, ssl_vc->mutex, eth); + MUTEX_TRY_LOCK(trylock, vc->mutex, eth); if (trylock.is_locked()) { - ssl_vc->reenable(ssl_vc->nh); + vc->profile_sm->reenable(); } else { reschedule = true; } } if (reschedule) { - ssl_vc->thread->schedule_imm(new TSSslCallback(ssl_vc)); + vc->thread->schedule_imm(new TSNetProfileSMCallback(vc->profile_sm)); } } } diff --git a/proxy/InkAPIInternal.h b/proxy/InkAPIInternal.h index a3248b8aad6..212ad7d68c3 100644 --- a/proxy/InkAPIInternal.h +++ b/proxy/InkAPIInternal.h @@ -285,6 +285,8 @@ typedef enum { TS_SSL_INTERNAL_FIRST_HOOK, TS_VCONN_PRE_ACCEPT_INTERNAL_HOOK = TS_SSL_INTERNAL_FIRST_HOOK, TS_SSL_CERT_INTERNAL_HOOK, + TS_SSL_CLIENT_HANDSHAKE_INTERNAL_HOOK, + TS_SSL_SERVER_HANDSHAKE_INTERNAL_HOOK, TS_SSL_INTERNAL_LAST_HOOK } TSSslHookInternalID; diff --git a/proxy/PluginVC.h b/proxy/PluginVC.h index daa9c0337a3..0879801c9ae 100644 --- a/proxy/PluginVC.h +++ b/proxy/PluginVC.h @@ -98,6 +98,56 @@ class PluginVC : public NetVConnection, public PluginIdentity virtual bool add_to_active_queue(); virtual ink_hrtime get_active_timeout(); virtual ink_hrtime get_inactivity_timeout(); + // The following function are not used in this class. + // Just to make GCC happy. + virtual void + readDisable() + { + } + virtual void + writeDisable() + { + } + virtual void + readSignalError(int err) + { + } + virtual void + writeSignalError(int err) + { + } + virtual int + readSignalDone(int event) + { + return 0; + } + virtual int + writeSignalDone(int event) + { + return 0; + } + virtual int + readSignalAndUpdate(int event) + { + return 0; + } + virtual int + writeSignalAndUpdate(int event) + { + return 0; + } + virtual void + readReschedule() + { + } + virtual void + writeReschedule() + { + } + virtual void + netActivity(EThread *lthread) + { + } // Pure virutal functions we need to compile virtual SOCKET get_socket(); diff --git a/proxy/ProtocolProbeSessionAccept.cc b/proxy/ProtocolProbeSessionAccept.cc index dd05889d37d..2ac7124cca9 100644 --- a/proxy/ProtocolProbeSessionAccept.cc +++ b/proxy/ProtocolProbeSessionAccept.cc @@ -24,6 +24,7 @@ #include "P_Net.h" #include "I_Machine.h" #include "ProtocolProbeSessionAccept.h" +#include "I_RecHttp.h" #include "http2/HTTP2.h" static bool @@ -64,6 +65,7 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio { VIO *vio; NetVConnection *netvc; + Continuation *plugin; ProtoGroupKey key = N_PROTO_GROUPS; // use this as an invalid value. vio = static_cast(edata); @@ -85,12 +87,24 @@ struct ProtocolProbeTrampoline : public Continuation, public ProtocolProbeSessio ink_assert(netvc != NULL); - if (!reader->is_read_avail_more_than(minimum_read_size - 1)) { - // Not enough data read. Well, that sucks. - goto done; + if (netvc->profile_sm->get_type() == PROFILE_SM_SSL) { + SSLM *sslm = dynamic_cast(netvc->profile_sm); + plugin = sslm->endpoint(); // NPN or ALPN + if (plugin) { + netvc->do_io_read(NULL, 0, NULL); // Disable the read IO that we started. + netvc->attributes = HttpProxyPort::TRANSPORT_SSL; + ((SessionAccept *)plugin)->accept(netvc, this->iobuf, reader); + delete this; + return EVENT_CONT; + } } - if (proto_is_http2(reader)) { + if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == netvc->attributes) { + key = PROTO_HTTP; + } else if (!reader->is_read_avail_more_than(minimum_read_size - 1)) { + // Not enough data read. Well, that sucks. + goto done; + } else if (proto_is_http2(reader)) { key = PROTO_HTTP2; } else { key = PROTO_HTTP; @@ -130,7 +144,15 @@ ProtocolProbeSessionAccept::mainEvent(int event, void *data) ink_assert(data); VIO *vio; - NetVConnection *netvc = (NetVConnection *)data; + NetVConnection *netvc = (NetVConnection *)data; + + if (netvc->profile_sm->get_type() == PROFILE_SM_SSL) { + Debug("ssl", "[ProtocolProbeSessionAccept::mainEvent] event %d netvc %p", event, netvc); + SSLM *sslm = dynamic_cast(netvc->profile_sm); + sslm->registerNextProtocolSet(&protoset); + sslm->setTransparentPassThrough(transparent_passthrough); + } + ProtocolProbeTrampoline *probe = new ProtocolProbeTrampoline(this, netvc->mutex, NULL, NULL); // XXX we need to apply accept inactivity timeout here ... @@ -164,3 +186,15 @@ ProtocolProbeSessionAccept::registerEndpoint(ProtoGroupKey key, SessionAccept *a ink_release_assert(endpoint[key] == NULL); this->endpoint[key] = ap; } + +bool +ProtocolProbeSessionAccept::registerEndpoint(const char *protocol, Continuation *handler) +{ + return this->protoset.registerEndpoint(protocol, handler); +} + +void +ProtocolProbeSessionAccept::setTransparentPassthrough(bool transparent_passthrough) +{ + this->transparent_passthrough = transparent_passthrough; +} diff --git a/proxy/ProtocolProbeSessionAccept.h b/proxy/ProtocolProbeSessionAccept.h index 03eefdaa58f..7e7180cd988 100644 --- a/proxy/ProtocolProbeSessionAccept.h +++ b/proxy/ProtocolProbeSessionAccept.h @@ -25,6 +25,7 @@ #define ProtocolProbeSessionAccept_H_ #include "I_SessionAccept.h" +#include "I_SSLNextProtocolSet.h" struct ProtocolProbeSessionAcceptEnums { /// Enumeration for related groups of protocols. @@ -40,13 +41,16 @@ struct ProtocolProbeSessionAcceptEnums { class ProtocolProbeSessionAccept : public SessionAccept, public ProtocolProbeSessionAcceptEnums { public: - ProtocolProbeSessionAccept() : SessionAccept(NULL) + ProtocolProbeSessionAccept() : SessionAccept(NULL), transparent_passthrough(false) { memset(endpoint, 0, sizeof(endpoint)); SET_HANDLER(&ProtocolProbeSessionAccept::mainEvent); } ~ProtocolProbeSessionAccept() {} void registerEndpoint(ProtoGroupKey key, SessionAccept *ap); + bool registerEndpoint(const char *protocol, Continuation *handler); + + void setTransparentPassthrough(bool transparent_passthrough); bool accept(NetVConnection *, MIOBuffer *, IOBufferReader *); @@ -62,6 +66,8 @@ class ProtocolProbeSessionAccept : public SessionAccept, public ProtocolProbeSes do range checks on the enum value. */ SessionAccept *endpoint[N_PROTO_GROUPS + 1]; + SSLNextProtocolSet protoset; + bool transparent_passthrough; friend struct ProtocolProbeTrampoline; }; diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index 7285787aa7b..3ae6efe2b9d 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -482,6 +482,10 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t) return "TS_VCONN_PRE_ACCEPT_HOOK"; case TS_SSL_CERT_HOOK: return "TS_SSL_CERT_HOOK"; + case TS_SSL_CLIENT_HANDSHAKE_HOOK: + return "TS_SSL_CLIENT_HANDSHAKE_HOOK"; + case TS_SSL_SERVER_HANDSHAKE_HOOK: + return "TS_SSL_SERVER_HANDSHAKE_HOOK"; } return "unknown hook"; diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index cbb221c6434..07a42581c44 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -35,19 +35,25 @@ #include "HttpPages.h" #include "HttpTunnel.h" #include "ts/Tokenizer.h" -#include "P_SSLNextProtocolAccept.h" #include "ProtocolProbeSessionAccept.h" #include "http2/Http2SessionAccept.h" HttpSessionAccept *plugin_http_accept = NULL; HttpSessionAccept *plugin_http_transparent_accept = 0; +/* + * break TSNetAcceptNamedProtocol + * static SLL ssl_plugin_acceptors; static Ptr ssl_plugin_mutex; + */ bool ssl_register_protocol(const char *protocol, Continuation *contp) { + /* + * break TSNetAcceptNamedProtocol + * SCOPED_MUTEX_LOCK(lock, ssl_plugin_mutex, this_ethread()); for (SSLNextProtocolAccept *ssl = ssl_plugin_acceptors.head; ssl; ssl = ssl_plugin_acceptors.next(ssl)) { @@ -55,6 +61,7 @@ ssl_register_protocol(const char *protocol, Continuation *contp) return false; } } + */ return true; } @@ -62,6 +69,9 @@ ssl_register_protocol(const char *protocol, Continuation *contp) bool ssl_unregister_protocol(const char *protocol, Continuation *contp) { + /* + * break TSNetAcceptNamedProtocol + * SCOPED_MUTEX_LOCK(lock, ssl_plugin_mutex, this_ethread()); for (SSLNextProtocolAccept *ssl = ssl_plugin_acceptors.head; ssl; ssl = ssl_plugin_acceptors.next(ssl)) { @@ -69,6 +79,7 @@ ssl_unregister_protocol(const char *protocol, Continuation *contp) // from all SSL ports. ssl->unregisterEndpoint(protocol, contp); } + */ return true; } @@ -126,6 +137,8 @@ make_net_accept_options(const HttpProxyPort &port, unsigned nthreads) net.local_ip = HttpConfig::m_master.inbound_ip4; } + net.isSSL = port.isSSL(); + return net; } @@ -167,50 +180,45 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned // XXX the protocol probe should be a configuration option. ProtocolProbeSessionAccept *probe = new ProtocolProbeSessionAccept(); - HttpSessionAccept *http = 0; // don't allocate this unless it will be used. + // ALPN selects the first server-offered protocol, + // so make sure that we offer the newest protocol first. + // But since registerEndpoint prepends you want to + // register them backwards, so you'd want to register + // the least important protocol first: + // http/1.0, http/1.1, h2 + // + // Pre registed ALPN Endpoints for dynamtic SSL/TLS probe on any port. + + // HTTP + HttpSessionAccept *http = NULL; // don't allocate this unless it will be used. if (port.m_session_protocol_preference.intersects(HTTP_PROTOCOL_SET)) { http = new HttpSessionAccept(accept_opt); probe->registerEndpoint(ProtocolProbeSessionAccept::PROTO_HTTP, http); - } - - if (port.m_session_protocol_preference.intersects(HTTP2_PROTOCOL_SET)) { - probe->registerEndpoint(ProtocolProbeSessionAccept::PROTO_HTTP2, new Http2SessionAccept(accept_opt)); - } - - if (port.isSSL()) { - SSLNextProtocolAccept *ssl = new SSLNextProtocolAccept(probe, port.m_transparent_passthrough); - - // ALPN selects the first server-offered protocol, - // so make sure that we offer the newest protocol first. - // But since registerEndpoint prepends you want to - // register them backwards, so you'd want to register - // the least important protocol first: - // http/1.0, http/1.1, h2 - - // HTTP if (port.m_session_protocol_preference.contains(TS_ALPN_PROTOCOL_INDEX_HTTP_1_0)) { - ssl->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_1_0, http); + probe->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_1_0, http); } - if (port.m_session_protocol_preference.contains(TS_ALPN_PROTOCOL_INDEX_HTTP_1_1)) { - ssl->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_1_1, http); + probe->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_1_1, http); } + } - // HTTP2 - if (port.m_session_protocol_preference.contains(TS_ALPN_PROTOCOL_INDEX_HTTP_2_0)) { - Http2SessionAccept *acc = new Http2SessionAccept(accept_opt); - - ssl->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_2_0, acc); + Http2SessionAccept *http2 = NULL; + if (port.m_session_protocol_preference.intersects(HTTP2_PROTOCOL_SET)) { + http2 = new Http2SessionAccept(accept_opt); + probe->registerEndpoint(ProtocolProbeSessionAccept::PROTO_HTTP2, http2); + } + if (port.m_session_protocol_preference.contains(TS_ALPN_PROTOCOL_INDEX_HTTP_2_0)) { + // TODO: Should be removed when h2-14 is gone and dead, and h2 is widely supported in UAs + if (http2 == NULL) { + http2 = new Http2SessionAccept(accept_opt); } + probe->registerEndpoint(TS_ALPN_PROTOCOL_HTTP_2_0, http2); + } - SCOPED_MUTEX_LOCK(lock, ssl_plugin_mutex, this_ethread()); - ssl_plugin_acceptors.push(ssl); + probe->setTransparentPassthrough(port.m_transparent_passthrough); - acceptor._accept = ssl; - } else { - acceptor._accept = probe; - } + acceptor._accept = probe; } /** Set up all the accepts and sockets. @@ -245,9 +253,6 @@ init_HttpProxyServer(int n_accept_threads) plugin_http_transparent_accept = new HttpSessionAccept(ha_opt); plugin_http_transparent_accept->mutex = new_ProxyMutex(); } - if (!ssl_plugin_mutex) { - ssl_plugin_mutex = new_ProxyMutex(); - } // Do the configuration defined ports. for (int i = 0, n = proxy_ports.length(); i < n; ++i) { @@ -271,11 +276,7 @@ start_HttpProxyServer() for (int i = 0, n = proxy_ports.length(); i < n; ++i) { HttpProxyAcceptor &acceptor = HttpProxyAcceptors[i]; HttpProxyPort &port = proxy_ports[i]; - if (port.isSSL()) { - if (NULL == sslNetProcessor.main_accept(acceptor._accept, port.m_fd, acceptor._net_opt)) { - return; - } - } else if (!port.isPlugin()) { + if (!port.isPlugin()) { if (NULL == netProcessor.main_accept(acceptor._accept, port.m_fd, acceptor._net_opt)) { return; } diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index b72e47bc66c..cde2a1a4f83 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -526,18 +526,23 @@ HttpSM::attach_client_session(ProxyClientTransaction *client_vc, IOBufferReader ua_session = client_vc; // Collect log & stats information - client_tcp_reused = (1 < ua_session->get_transact_count()); - SSLNetVConnection *ssl_vc = dynamic_cast(netvc); - if (ssl_vc != NULL) { - client_connection_is_ssl = true; - client_ssl_reused = ssl_vc->getSSLSessionCacheHit(); - const char *protocol = ssl_vc->getSSLProtocol(); - client_sec_protocol = protocol ? protocol : "-"; - const char *cipher = ssl_vc->getSSLCipherSuite(); - client_cipher_suite = cipher ? cipher : "-"; - } - const char *protocol_str = client_vc->get_protocol_string(); - client_protocol = protocol_str ? protocol_str : "-"; + client_tcp_reused = (1 < ua_session->get_transact_count()); + const char *protocol = NULL; + const char *cipher = NULL; + if (netvc->profile_sm->get_type() == PROFILE_SM_SSL) { + SSLM *sslm = dynamic_cast(netvc->profile_sm); + this->client_connection_is_ssl = true; + this->client_ssl_reused = sslm->getSSLSessionCacheHit(); + protocol = sslm->getSSLProtocol(); + cipher = sslm->getSSLCipherSuite(); + } else { + this->client_connection_is_ssl = false; + this->client_ssl_reused = false; + } + this->client_sec_protocol = protocol ? protocol : "-"; + this->client_cipher_suite = cipher ? cipher : "-"; + const char *protocol_str = client_vc->get_protocol_string(); + this->client_protocol = protocol_str ? protocol_str : "-"; ink_release_assert(ua_session->get_half_close_flag() == false); mutex = client_vc->mutex; @@ -5832,31 +5837,24 @@ HttpSM::attach_server_session(HttpServerSession *s) // Get server and client connections UnixNetVConnection *server_vc = dynamic_cast(server_session->get_netvc()); UnixNetVConnection *client_vc = (UnixNetVConnection *)(ua_session->get_netvc()); - SSLNetVConnection *ssl_vc = dynamic_cast(client_vc); bool associated_connection = false; - if (server_vc) { // if server_vc isn't a PluginVC - if (ssl_vc) { // if incoming connection is SSL - bool client_trace = ssl_vc->getSSLTrace(); + if (server_vc) { // if server_vc isn't a PluginVC + if (client_vc->profile_sm->get_type() == PROFILE_SM_SSL) { // if incoming connection is SSL + bool client_trace = client_vc->profile_sm->getTrace(); if (client_trace) { // get remote address and port to mark corresponding traces - const sockaddr *remote_addr = ssl_vc->get_remote_addr(); - uint16_t remote_port = ssl_vc->get_remote_port(); - server_vc->setOriginTrace(true); + const sockaddr *remote_addr = client_vc->get_remote_addr(); server_vc->setOriginTraceAddr(remote_addr); - server_vc->setOriginTracePort(remote_port); associated_connection = true; } } } if (!associated_connection && server_vc) { - server_vc->setOriginTrace(false); server_vc->setOriginTraceAddr(NULL); - server_vc->setOriginTracePort(0); } // set flag for server session is SSL - SSLNetVConnection *server_ssl_vc = dynamic_cast(server_vc); - if (server_ssl_vc) { + if (server_vc->profile_sm->get_type() == PROFILE_SM_SSL) { server_connection_is_ssl = true; } diff --git a/proxy/shared/UglyLogStubs.cc b/proxy/shared/UglyLogStubs.cc index a5e5069cb0a..cda287628e3 100644 --- a/proxy/shared/UglyLogStubs.cc +++ b/proxy/shared/UglyLogStubs.cc @@ -193,6 +193,12 @@ UnixNetProcessor::allocate_vc(EThread *) return NULL; } +void +UnixNetProcessor::attach_profile_sm(EThread *, NetVConnection *) +{ + ink_release_assert(false); +} + // For Intel ICC int cache_config_mutex_retry_delay = 2;