use the new crypto/tls QUIC Transport

.github/workflows/gotip.yml

@@ -0,0 +1,28 @@
+on: [push, pull_request]
+
+jobs:
+  integration:
+    runs-on: ${{ fromJSON(vars['GOTIP_RUNNER_UBUNTU'] || '"ubuntu-latest"') }}
+    env:
+      DEBUG: false # set this to true to export qlogs and save them as artifacts
+      TIMESCALE_FACTOR: 3
+    name: Test with gotip
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3 # need to install Go for bootstrapping
+        with:
+          go-version: '1.20.x'
+      - name: Install gotip
+        run: |
+          git clone --depth=1 https://go.googlesource.com/go $HOME/gotip
+          cd $HOME/gotip/src
+          git fetch https://go.googlesource.com/go refs/changes/15/498215/1 && git checkout FETCH_HEAD
+          ./make.bash
+          echo "GOROOT=$HOME/gotip" >> $GITHUB_ENV
+          echo "$HOME/gotip/bin:$PATH" >> $GITHUB_PATH
+      - run: go version
+      - name: set qlogger
+        if: env.DEBUG == 'true'
+        run: echo "QLOGFLAG= -qlog" >> $GITHUB_ENV
+      - name: Run self tests, using QUIC v1
+        run: go run github.com/onsi/ginkgo/v2/ginkgo -r -v -randomize-all -randomize-suites -trace integrationtests/self -- -version=1 ${{ env.QLOGFLAG }}

connection.go

@@ -52,7 +52,7 @@ type streamManager interface {
 }
 
 type cryptoStreamHandler interface {
-	RunHandshake()
+	StartHandshake() error
 	ChangeConnectionID(protocol.ConnectionID)
 	SetLargest1RTTAcked(protocol.PacketNumber) error
 	SetHandshakeConfirmed()
@@ -103,15 +103,15 @@ type connRunner interface {
 
 type handshakeRunner struct {
 	onReceivedParams    func(*wire.TransportParameters)
-	onError             func(error)
+	onReceivedReadKeys  func()
 	dropKeys            func(protocol.EncryptionLevel)
 	onHandshakeComplete func()
 }
 
 func (r *handshakeRunner) OnReceivedParams(tp *wire.TransportParameters) { r.onReceivedParams(tp) }
-func (r *handshakeRunner) OnError(e error)                               { r.onError(e) }
 func (r *handshakeRunner) DropKeys(el protocol.EncryptionLevel)          { r.dropKeys(el) }
 func (r *handshakeRunner) OnHandshakeComplete()                          { r.onHandshakeComplete() }
+func (r *handshakeRunner) OnReceivedReadKeys()                           { r.onReceivedReadKeys() }
 
 type closeError struct {
 	err       error
@@ -327,14 +327,13 @@ var newConnection = func(
 	cs := handshake.NewCryptoSetupServer(
 		initialStream,
 		handshakeStream,
+		s.oneRTTStream,
 		clientDestConnID,
-		conn.LocalAddr(),
-		conn.RemoteAddr(),
 		params,
 		&handshakeRunner{
-			onReceivedParams: s.handleTransportParameters,
-			onError:          s.closeLocal,
-			dropKeys:         s.dropEncryptionLevel,
+			onReceivedParams:   s.handleTransportParameters,
+			dropKeys:           s.dropEncryptionLevel,
+			onReceivedReadKeys: s.receivedReadKeys,
 			onHandshakeComplete: func() {
 				runner.Retire(clientDestConnID)
 				close(s.handshakeCompleteChan)
@@ -415,6 +414,7 @@ var newClientConnection = func(
 	)
 	initialStream := newCryptoStream()
 	handshakeStream := newCryptoStream()
+	oneRTTStream := newCryptoStream()
 	params := &wire.TransportParameters{
 		InitialMaxStreamDataBidiRemote: protocol.ByteCount(s.config.InitialStreamReceiveWindow),
 		InitialMaxStreamDataBidiLocal:  protocol.ByteCount(s.config.InitialStreamReceiveWindow),
@@ -440,14 +440,13 @@ var newClientConnection = func(
 	cs, clientHelloWritten := handshake.NewCryptoSetupClient(
 		initialStream,
 		handshakeStream,
+		oneRTTStream,
 		destConnID,
-		conn.LocalAddr(),
-		conn.RemoteAddr(),
 		params,
 		&handshakeRunner{
 			onReceivedParams:    s.handleTransportParameters,
-			onError:             s.closeLocal,
 			dropKeys:            s.dropEncryptionLevel,
+			onReceivedReadKeys:  s.receivedReadKeys,
 			onHandshakeComplete: func() { close(s.handshakeCompleteChan) },
 		},
 		tlsConf,
@@ -459,7 +458,7 @@ var newClientConnection = func(
 	)
 	s.clientHelloWritten = clientHelloWritten
 	s.cryptoStreamHandler = cs
-	s.cryptoStreamManager = newCryptoStreamManager(cs, initialStream, handshakeStream, newCryptoStream())
+	s.cryptoStreamManager = newCryptoStreamManager(cs, initialStream, handshakeStream, oneRTTStream)
 	s.unpacker = newPacketUnpacker(cs, s.srcConnIDLen)
 	s.packer = newPacketPacker(srcConnID, s.connIDManager.Get, initialStream, handshakeStream, s.sentPacketHandler, s.retransmissionQueue, s.RemoteAddr(), cs, s.framer, s.receivedPacketHandler, s.datagramQueue, s.perspective)
 	if len(tlsConf.ServerName) > 0 {
@@ -522,11 +521,9 @@ func (s *connection) run() error {
 
 	s.timer = *newTimer()
 
-	handshaking := make(chan struct{})
-	go func() {
-		defer close(handshaking)
-		s.cryptoStreamHandler.RunHandshake()
-	}()
+	if err := s.cryptoStreamHandler.StartHandshake(); err != nil {
+		return err
+	}
 	go func() {
 		if err := s.sendQueue.Run(); err != nil {
 			s.destroyImpl(err)
@@ -678,7 +675,6 @@ runLoop:
 	}
 
 	s.cryptoStreamHandler.Close()
-	<-handshaking
 	s.handleCloseError(&closeErr)
 	if e := (&errCloseForRecreating{}); !errors.As(closeErr.err, &e) && s.tracer != nil {
 		s.tracer.Close()
@@ -709,7 +705,9 @@ func (s *connection) supportsDatagrams() bool {
 func (s *connection) ConnectionState() ConnectionState {
 	s.connStateMutex.Lock()
 	defer s.connStateMutex.Unlock()
-	s.connState.TLS = s.cryptoStreamHandler.ConnectionState()
+	cs := s.cryptoStreamHandler.ConnectionState()
+	s.connState.TLS = cs.ConnectionState
+	s.connState.Used0RTT = cs.Used0RTT
 	return s.connState
 }
 
@@ -771,7 +769,7 @@ func (s *connection) handleHandshakeComplete() {
 	if err != nil {
 		s.closeLocal(err)
 	}
-	if ticket != nil {
+	if ticket != nil { // may be nil if session tickets are disabled via tls.Config.SessionTicketsDisabled
 		s.oneRTTStream.Write(ticket)
 		for s.oneRTTStream.HasData() {
 			s.queueControlFrame(s.oneRTTStream.PopCryptoFrame(protocol.MaxPostHandshakeCryptoFrameSize))
@@ -1369,16 +1367,13 @@ func (s *connection) handleConnectionCloseFrame(frame *wire.ConnectionCloseFrame
 }
 
 func (s *connection) handleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) error {
-	encLevelChanged, err := s.cryptoStreamManager.HandleCryptoFrame(frame, encLevel)
-	if err != nil {
-		return err
-	}
-	if encLevelChanged {
-		// Queue all packets for decryption that have been undecryptable so far.
-		s.undecryptablePacketsToProcess = s.undecryptablePackets
-		s.undecryptablePackets = nil
-	}
-	return nil
+	return s.cryptoStreamManager.HandleCryptoFrame(frame, encLevel)
+}
+
+func (s *connection) receivedReadKeys() {
+	// Queue all packets for decryption that have been undecryptable so far.
+	s.undecryptablePacketsToProcess = s.undecryptablePackets
+	s.undecryptablePackets = nil
 }
 
 func (s *connection) handleStreamFrame(frame *wire.StreamFrame) error {
@@ -1620,11 +1615,15 @@ func (s *connection) handleCloseError(closeErr *closeError) {
 }
 
 func (s *connection) dropEncryptionLevel(encLevel protocol.EncryptionLevel) {
-	s.sentPacketHandler.DropPackets(encLevel)
-	s.receivedPacketHandler.DropPackets(encLevel)
 	if s.tracer != nil {
 		s.tracer.DroppedEncryptionLevel(encLevel)
 	}
+	s.sentPacketHandler.DropPackets(encLevel)
+	s.receivedPacketHandler.DropPackets(encLevel)
+	if err := s.cryptoStreamManager.Drop(encLevel); err != nil {
+		s.closeLocal(err)
+		return
+	}
 	if encLevel == protocol.Encryption0RTT {
 		s.streamsMap.ResetFor0RTT()
 		if err := s.connFlowController.Reset(); err != nil {
@@ -1867,6 +1866,9 @@ func (s *connection) sendPacket() (bool, error) {
 		s.framer.QueueControlFrame(&wire.DataBlockedFrame{MaximumData: offset})
 	}
 	s.windowUpdateQueue.QueueAll()
+	if cf := s.cryptoStreamManager.GetPostHandshakeData(protocol.MaxPostHandshakeCryptoFrameSize); cf != nil {
+		s.queueControlFrame(cf)
+	}
 
 	now := time.Now()
 	if !s.handshakeConfirmed {

crypto_stream_manager.go

@@ -8,7 +8,7 @@ import (
 )
 
 type cryptoDataHandler interface {
-	HandleMessage([]byte, protocol.EncryptionLevel) bool
+	HandleMessage([]byte, protocol.EncryptionLevel) error
 }
 
 type cryptoStreamManager struct {
@@ -33,7 +33,7 @@ func newCryptoStreamManager(
 	}
 }
 
-func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) (bool /* encryption level changed */, error) {
+func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) error {
 	var str cryptoStream
 	//nolint:exhaustive // CRYPTO frames cannot be sent in 0-RTT packets.
 	switch encLevel {
@@ -44,18 +44,38 @@ func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLeve
 	case protocol.Encryption1RTT:
 		str = m.oneRTTStream
 	default:
-		return false, fmt.Errorf("received CRYPTO frame with unexpected encryption level: %s", encLevel)
+		return fmt.Errorf("received CRYPTO frame with unexpected encryption level: %s", encLevel)
 	}
 	if err := str.HandleCryptoFrame(frame); err != nil {
-		return false, err
+		return err
 	}
 	for {
 		data := str.GetCryptoData()
 		if data == nil {
-			return false, nil
+			return nil
 		}
-		if encLevelFinished := m.cryptoHandler.HandleMessage(data, encLevel); encLevelFinished {
-			return true, str.Finish()
+		if err := m.cryptoHandler.HandleMessage(data, encLevel); err != nil {
+			return err
 		}
 	}
 }
+
+func (m *cryptoStreamManager) GetPostHandshakeData(maxSize protocol.ByteCount) *wire.CryptoFrame {
+	if !m.oneRTTStream.HasData() {
+		return nil
+	}
+	return m.oneRTTStream.PopCryptoFrame(maxSize)
+}
+
+func (m *cryptoStreamManager) Drop(encLevel protocol.EncryptionLevel) error {
+	switch encLevel {
+	case protocol.EncryptionInitial:
+		return m.initialStream.Finish()
+	case protocol.EncryptionHandshake:
+		return m.handshakeStream.Finish()
+	case protocol.Encryption0RTT:
+		return nil
+	default:
+		panic(fmt.Sprintf("dropped unexpected encryption level: %s", encLevel))
+	}
+}

http3/client.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/quic-go/quic-go"
 	"github.com/quic-go/quic-go/internal/protocol"
-	"github.com/quic-go/quic-go/internal/qtls"
 	"github.com/quic-go/quic-go/internal/utils"
 	"github.com/quic-go/quic-go/quicvarint"
 
@@ -393,7 +392,7 @@ func (c *client) doRequest(req *http.Request, conn quic.EarlyConnection, str qui
 		return nil, newConnError(ErrCodeGeneralProtocolError, err)
 	}
 
-	connState := qtls.ToTLSConnectionState(conn.ConnectionState().TLS)
+	connState := conn.ConnectionState().TLS
 	res := &http.Response{
 		Proto:      "HTTP/3.0",
 		ProtoMajor: 3,

http3/server.go

@@ -585,7 +585,7 @@ func (s *Server) handleRequest(conn quic.Connection, str quic.Stream, decoder *q
 		return newStreamError(ErrCodeGeneralProtocolError, err)
 	}
 
-	connState := conn.ConnectionState().TLS.ConnectionState
+	connState := conn.ConnectionState().TLS
 	req.TLS = &connState
 	req.RemoteAddr = conn.RemoteAddr().String()
 	body := newRequestBody(newStream(str, onFrameError))

integrationtests/self/handshake_test.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/internal/handshake"
 	"github.com/quic-go/quic-go/internal/protocol"
 	"github.com/quic-go/quic-go/internal/qerr"
-	"github.com/quic-go/quic-go/internal/qtls"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -91,7 +91,7 @@ var _ = Describe("Handshake tests", func() {
 			suiteID := id
 
 			It(fmt.Sprintf("using %s", name), func() {
-				reset := qtls.SetCipherSuite(suiteID)
+				reset := handshake.SetCipherSuite(suiteID)
 				defer reset()
 
 				tlsConf := getTLSConfig()
@@ -198,7 +198,7 @@ var _ = Describe("Handshake tests", func() {
 			var transportErr *quic.TransportError
 			Expect(errors.As(err, &transportErr)).To(BeTrue())
 			Expect(transportErr.ErrorCode.IsCryptoError()).To(BeTrue())
-			Expect(transportErr.Error()).To(ContainSubstring("tls: bad certificate"))
+			Expect(transportErr.Error()).To(ContainSubstring("tls: certificate required"))
 		})
 
 		It("uses the ServerName in the tls.Config", func() {