Peering support for SslBump (squid-cache#380)

Support forwarding of bumped, re­encrypted HTTPS requests through a
cache_peer using a standard HTTP CONNECT tunnel.

The new Http::Tunneler class establishes HTTP CONNECT tunnels through
forward proxies. It is used by TunnelStateData and FwdState classes.

Just like before these changes, when a cache_peer replies to CONNECT
with an error response, only the HTTP response headers are forwarded to
the client, and then the connection is closed.

No support for triggering client authentication when a cache_peer
configuration instructs the bumping Squid to relay authentication info
contained in client CONNECT request. The bumping Squid still responds
with HTTP 200 (Connection Established) to the client CONNECT request (to
see TLS client handshake) _before_ selecting the cache_peer.

HTTPS cache_peers are not yet supported primarily because Squid cannot
do TLS-in-TLS with a single fde::ssl state; SslBump and the HTTPS proxy
client/tunneling code would need a dedicated TLS connection each.

Also fixed delay pools for tunneled traffic.

This is a Measurement Factory project.

src/Debug.h

@@ -185,14 +185,16 @@ class Raw
 {
 public:
     Raw(const char *label, const char *data, const size_t size):
-        level(-1), label_(label), data_(data), size_(size), useHex_(false) {}
+        level(-1), label_(label), data_(data), size_(size), useHex_(false), useGap_(true) {}
 
     /// limit data printing to at least the given debugging level
     Raw &minLevel(const int aLevel) { level = aLevel; return *this; }
 
     /// print data using two hex digits per byte (decoder: xxd -r -p)
     Raw &hex() { useHex_ = true; return *this; }
 
+    Raw &gap(bool useGap = true) { useGap_ = useGap; return *this; }
+
     /// If debugging is prohibited by the current debugs() or section level,
     /// prints nothing. Otherwise, dumps data using one of these formats:
     ///   " label[size]=data" if label was set and data size is positive
@@ -213,6 +215,7 @@ class Raw
     const char *data_; ///< raw data to be printed
     size_t size_; ///< data length
     bool useHex_; ///< whether hex() has been called
+    bool useGap_; ///< whether to print leading space if label is missing
 };
 
 inline

src/FwdState.cc

@@ -18,6 +18,7 @@
 #include "CachePeer.h"
 #include "client_side.h"
 #include "clients/forward.h"
+#include "clients/HttpTunneler.h"
 #include "comm/Connection.h"
 #include "comm/ConnOpener.h"
 #include "comm/Loops.h"
@@ -288,11 +289,6 @@ FwdState::~FwdState()
 
     entry = NULL;
 
-    if (calls.connector != NULL) {
-        calls.connector->cancel("FwdState destructed");
-        calls.connector = NULL;
-    }
-
     if (Comm::IsConnOpen(serverConn))
         closeServerConnection("~FwdState");
 
@@ -709,6 +705,7 @@ FwdState::handleUnregisteredServerEnd()
     retryOrBail();
 }
 
+/// handles an established TCP connection to peer (including origin servers)
 void
 FwdState::connectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno)
 {
@@ -737,21 +734,104 @@ FwdState::connectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, in
     // only set when we dispatch the request to an existing (pinned) connection.
     assert(!request->flags.pinned);
 
+    if (const CachePeer *peer = serverConnection()->getPeer()) {
+        // Assume that it is only possible for the client-first from the
+        // bumping modes to try connect to a remote server. The bumped
+        // requests with other modes are using pinned connections or fails.
+        const bool clientFirstBump = request->flags.sslBumped;
+        // We need a CONNECT tunnel to send encrypted traffic through a proxy,
+        // but we do not support TLS inside TLS, so we exclude HTTPS proxies.
+        const bool originWantsEncryptedTraffic =
+            request->method == Http::METHOD_CONNECT ||
+            request->flags.sslPeek ||
+            clientFirstBump;
+        if (originWantsEncryptedTraffic && // the "encrypted traffic" part
+                !peer->options.originserver && // the "through a proxy" part
+                !peer->secure.encryptTransport) // the "exclude HTTPS proxies" part
+            return establishTunnelThruProxy();
+    }
+
+    secureConnectionToPeerIfNeeded();
+}
+
+void
+FwdState::establishTunnelThruProxy()
+{
+    AsyncCall::Pointer callback = asyncCall(17,4,
+                                            "FwdState::tunnelEstablishmentDone",
+                                            Http::Tunneler::CbDialer<FwdState>(&FwdState::tunnelEstablishmentDone, this));
+    HttpRequest::Pointer requestPointer = request;
+    const auto tunneler = new Http::Tunneler(serverConnection(), requestPointer, callback, connectingTimeout(serverConnection()), al);
+#if USE_DELAY_POOLS
+    Must(serverConnection()->getPeer());
+    if (!serverConnection()->getPeer()->options.no_delay)
+        tunneler->setDelayId(entry->mem_obj->mostBytesAllowed());
+#endif
+    AsyncJob::Start(tunneler);
+    // and wait for the tunnelEstablishmentDone() call
+}
+
+/// resumes operations after the (possibly failed) HTTP CONNECT exchange
+void
+FwdState::tunnelEstablishmentDone(Http::TunnelerAnswer &answer)
+{
+    if (answer.positive()) {
+        if (answer.leftovers.isEmpty()) {
+            secureConnectionToPeerIfNeeded();
+            return;
+        }
+        // This should not happen because TLS servers do not speak first. If we
+        // have to handle this, then pass answer.leftovers via a PeerConnector
+        // to ServerBio. See ClientBio::setReadBufData().
+        static int occurrences = 0;
+        const auto level = (occurrences++ < 100) ? DBG_IMPORTANT : 2;
+        debugs(17, level, "ERROR: Early data after CONNECT response. " <<
+               "Found " << answer.leftovers.length() << " bytes. " <<
+               "Closing " << serverConnection());
+        fail(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request, al));
+        closeServerConnection("found early data after CONNECT response");
+        retryOrBail();
+        return;
+    }
+
+    // TODO: Reuse to-peer connections after a CONNECT error response.
+
+    if (const auto peer = serverConnection()->getPeer())
+        peerConnectFailed(peer);
+
+    const auto error = answer.squidError.get();
+    Must(error);
+    answer.squidError.clear(); // preserve error for fail()
+    fail(error);
+    closeServerConnection("Squid-generated CONNECT error");
+    retryOrBail();
+}
+
+/// handles an established TCP connection to peer (including origin servers)
+void
+FwdState::secureConnectionToPeerIfNeeded()
+{
+    assert(!request->flags.pinned);
+
     const CachePeer *p = serverConnection()->getPeer();
     const bool peerWantsTls = p && p->secure.encryptTransport;
     // userWillTlsToPeerForUs assumes CONNECT == HTTPS
     const bool userWillTlsToPeerForUs = p && p->options.originserver &&
                                         request->method == Http::METHOD_CONNECT;
     const bool needTlsToPeer = peerWantsTls && !userWillTlsToPeerForUs;
-    const bool needTlsToOrigin = !p && request->url.getScheme() == AnyP::PROTO_HTTPS;
-    if (needTlsToPeer || needTlsToOrigin || request->flags.sslPeek) {
+    const bool clientFirstBump = request->flags.sslBumped; // client-first (already) bumped connection
+    const bool needsBump = request->flags.sslPeek || clientFirstBump;
+
+    // 'GET https://...' requests. If a peer is used the request is forwarded
+    // as is
+    const bool needTlsToOrigin = !p && request->url.getScheme() == AnyP::PROTO_HTTPS && !clientFirstBump;
+
+    if (needTlsToPeer || needTlsToOrigin || needsBump) {
         HttpRequest::Pointer requestPointer = request;
         AsyncCall::Pointer callback = asyncCall(17,4,
                                                 "FwdState::ConnectedToPeer",
                                                 FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this));
-        // Use positive timeout when less than one second is left.
-        const time_t connTimeout = serverDestinations[0]->connectTimeout(start_t);
-        const time_t sslNegotiationTimeout = positiveTimeout(connTimeout);
+        const auto sslNegotiationTimeout = connectingTimeout(serverDestinations[0]);
         Security::PeerConnector *connector = nullptr;
 #if USE_OPENSSL
         if (request->flags.sslPeek)
@@ -764,10 +844,10 @@ FwdState::connectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, in
     }
 
     // if not encrypting just run the post-connect actions
-    Security::EncryptorAnswer nil;
-    connectedToPeer(nil);
+    successfullyConnectedToPeer();
 }
 
+/// called when all negotiations with the TLS-speaking peer have been completed
 void
 FwdState::connectedToPeer(Security::EncryptorAnswer &answer)
 {
@@ -790,6 +870,13 @@ FwdState::connectedToPeer(Security::EncryptorAnswer &answer)
         return;
     }
 
+    successfullyConnectedToPeer();
+}
+
+/// called when all negotiations with the peer have been completed
+void
+FwdState::successfullyConnectedToPeer()
+{
     // should reach ConnStateData before the dispatched Client job starts
     CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
                  ConnStateData::notePeerConnection, serverConnection());
@@ -877,13 +964,27 @@ FwdState::connectStart()
 
     request->hier.startPeerClock();
 
-    // Do not fowrward bumped connections to parent proxy unless it is an
-    // origin server
-    if (serverDestinations[0]->getPeer() && !serverDestinations[0]->getPeer()->options.originserver && request->flags.sslBumped) {
-        debugs(50, 4, "fwdConnectStart: Ssl bumped connections through parent proxy are not allowed");
-        const auto anErr = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al);
-        fail(anErr);
-        stopAndDestroy("SslBump misconfiguration");
+    // Requests bumped at step2+ require their pinned connection. Since we
+    // failed to reuse the pinned connection, we now must terminate the
+    // bumped request. For client-first and step1 bumped requests, the
+    // from-client connection is already bumped, but the connection to the
+    // server is not established/pinned so they must be excluded. We can
+    // recognize step1 bumping by nil ConnStateData::serverBump().
+#if USE_OPENSSL
+    const auto clientFirstBump = request->clientConnectionManager.valid() &&
+        (request->clientConnectionManager->sslBumpMode == Ssl::bumpClientFirst ||
+         (request->clientConnectionManager->sslBumpMode == Ssl::bumpBump && !request->clientConnectionManager->serverBump())
+            );
+#else
+    const auto clientFirstBump = false;
+#endif /* USE_OPENSSL */
+    if (request->flags.sslBumped && !clientFirstBump) {
+        // TODO: Factor out/reuse as Occasionally(DBG_IMPORTANT, 2[, occurrences]).
+        static int occurrences = 0;
+        const auto level = (occurrences++ < 100) ? DBG_IMPORTANT : 2;
+        debugs(17, level, "BUG: Lost previously bumped from-Squid connection. Rejecting bumped request.");
+        fail(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request, al));
+        self = nullptr; // refcounted
         return;
     }
 
@@ -892,11 +993,12 @@ FwdState::connectStart()
     if (!serverDestinations[0]->getPeer())
         host = request->url.host();
 
+    bool bumpThroughPeer = request->flags.sslBumped && serverDestinations[0]->getPeer();
     Comm::ConnectionPointer temp;
     // Avoid pconns after races so that the same client does not suffer twice.
     // This does not increase the total number of connections because we just
     // closed the connection that failed the race. And re-pinning assumes this.
-    if (pconnRace != raceHappened)
+    if (pconnRace != raceHappened && !bumpThroughPeer)
         temp = pconnPop(serverDestinations[0], host);
 
     const bool openedPconn = Comm::IsConnOpen(temp);
@@ -929,9 +1031,9 @@ FwdState::connectStart()
 
     GetMarkingsToServer(request, *serverDestinations[0]);
 
-    calls.connector = commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper, this));
-    const time_t connTimeout = serverDestinations[0]->connectTimeout(start_t);
-    Comm::ConnOpener *cs = new Comm::ConnOpener(serverDestinations[0], calls.connector, connTimeout);
+    const AsyncCall::Pointer connector = commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper, this));
+    const auto connTimeout = connectingTimeout(serverDestinations[0]);
+    const auto cs = new Comm::ConnOpener(serverDestinations[0], connector, connTimeout);
     if (host)
         cs->setHost(host);
     ++n_tries;
@@ -1041,17 +1143,13 @@ FwdState::dispatch()
     }
 #endif
 
-    if (serverConnection()->getPeer() != NULL) {
-        ++ serverConnection()->getPeer()->stats.fetches;
-        request->peer_login = serverConnection()->getPeer()->login;
-        request->peer_domain = serverConnection()->getPeer()->domain;
-        request->flags.auth_no_keytab = serverConnection()->getPeer()->options.auth_no_keytab;
+    if (const auto peer = serverConnection()->getPeer()) {
+        ++peer->stats.fetches;
+        request->prepForPeering(*peer);
         httpStart(this);
     } else {
         assert(!request->flags.sslPeek);
-        request->peer_login = NULL;
-        request->peer_domain = NULL;
-        request->flags.auth_no_keytab = 0;
+        request->prepForDirect();
 
         switch (request->url.getScheme()) {
 
@@ -1315,6 +1413,13 @@ FwdState::pinnedCanRetry() const
     return true;
 }
 
+time_t
+FwdState::connectingTimeout(const Comm::ConnectionPointer &conn) const
+{
+    const auto connTimeout = conn->connectTimeout(start_t);
+    return positiveTimeout(connTimeout);
+}
+
 /**** PRIVATE NON-MEMBER FUNCTIONS ********************************************/
 
 /*

src/FwdState.h

@@ -10,6 +10,7 @@
 #define SQUID_FORWARD_H
 
 #include "base/RefCount.h"
+#include "clients/forward.h"
 #include "comm.h"
 #include "comm/Connection.h"
 #include "err_type.h"
@@ -134,6 +135,11 @@ class FwdState: public RefCountable, public PeerSelectionInitiator
     void connectedToPeer(Security::EncryptorAnswer &answer);
     static void RegisterWithCacheManager(void);
 
+    void establishTunnelThruProxy();
+    void tunnelEstablishmentDone(Http::TunnelerAnswer &answer);
+    void secureConnectionToPeerIfNeeded();
+    void successfullyConnectedToPeer();
+
     /// stops monitoring server connection for closure and updates pconn stats
     void closeServerConnection(const char *reason);
 
@@ -143,6 +149,9 @@ class FwdState: public RefCountable, public PeerSelectionInitiator
     /// whether we have used up all permitted forwarding attempts
     bool exhaustedTries() const;
 
+    /// \returns the time left for this connection to become connected or 1 second if it is less than one second left
+    time_t connectingTimeout(const Comm::ConnectionPointer &conn) const;
+
 public:
     StoreEntry *entry;
     HttpRequest *request;
@@ -157,11 +166,6 @@ class FwdState: public RefCountable, public PeerSelectionInitiator
     time_t start_t;
     int n_tries; ///< the number of forwarding attempts so far
 
-    // AsyncCalls which we set and may need cancelling.
-    struct {
-        AsyncCall::Pointer connector;  ///< a call linking us to the ConnOpener producing serverConn.
-    } calls;
-
     struct {
         bool connected_okay; ///< TCP link ever opened properly. This affects retry of POST,PUT,CONNECT,etc
         bool dont_retry;

src/HttpRequest.cc

@@ -12,6 +12,7 @@
 #include "AccessLogEntry.h"
 #include "acl/AclSizeLimit.h"
 #include "acl/FilledChecklist.h"
+#include "CachePeer.h"
 #include "client_side.h"
 #include "client_side_request.h"
 #include "dns/LookupDetails.h"
@@ -452,6 +453,25 @@ HttpRequest::bodyNibbled() const
     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 }
 
+void
+HttpRequest::prepForPeering(const CachePeer &peer)
+{
+    // XXX: Saving two pointers to memory controlled by an independent object.
+    peer_login = peer.login;
+    peer_domain = peer.domain;
+    flags.auth_no_keytab = peer.options.auth_no_keytab;
+    debugs(11, 4, this << " to " << peer.host << (!peer.options.originserver ? " proxy" : " origin"));
+}
+
+void
+HttpRequest::prepForDirect()
+{
+    peer_login = nullptr;
+    peer_domain = nullptr;
+    flags.auth_no_keytab = false;
+    debugs(11, 4, this);
+}
+
 void
 HttpRequest::detailError(err_type aType, int aDetail)
 {

src/HttpRequest.h

@@ -34,10 +34,11 @@
 #include "eui/Eui64.h"
 #endif
 
-class ConnStateData;
-class Downloader;
 class AccessLogEntry;
 typedef RefCount<AccessLogEntry> AccessLogEntryPointer;
+class CachePeer;
+class ConnStateData;
+class Downloader;
 
 /*  Http Request */
 void httpRequestPack(void *obj, Packable *p);
@@ -87,6 +88,13 @@ class HttpRequest: public Http::Message
     Adaptation::Icap::History::Pointer icapHistory() const;
 #endif
 
+    /* If a request goes through several destinations, then the following two
+     * methods will be called several times, in destinations-dependent order. */
+    /// get ready to be sent to the given cache_peer, including originserver
+    void prepForPeering(const CachePeer &peer);
+    /// get ready to be sent directly to an origin server, excluding originserver
+    void prepForDirect();
+
     void recordLookup(const Dns::LookupDetails &detail);
 
     /// sets error detail if no earlier detail was available