From f1fd7c7a1d41dabe6a080ddc50473c2649828e54 Mon Sep 17 00:00:00 2001 From: Honfika Date: Sat, 30 Nov 2019 13:09:53 +0100 Subject: [PATCH 1/5] Ignore docfx.json in ReSharper --- src/Titanium.Web.Proxy.sln.DotSettings | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Titanium.Web.Proxy.sln.DotSettings b/src/Titanium.Web.Proxy.sln.DotSettings index 1b03ebb37..c94f8bdbc 100644 --- a/src/Titanium.Web.Proxy.sln.DotSettings +++ b/src/Titanium.Web.Proxy.sln.DotSettings @@ -1,5 +1,6 @@  True + docfx.json False True NEVER From 7cb42ed190d6aa56d76ea27116616dc936126f9d Mon Sep 17 00:00:00 2001 From: Honfika Date: Sat, 30 Nov 2019 20:07:16 +0100 Subject: [PATCH 2/5] decding websocket frame --- .../ProxyTestController.cs | 41 ++++- .../EventArguments/SessionEventArgs.cs | 4 + .../ExplicitClientHandler.cs | 45 ++++-- src/Titanium.Web.Proxy/Helpers/HttpHelper.cs | 119 ++++++++++----- src/Titanium.Web.Proxy/Helpers/HttpStream.cs | 14 +- src/Titanium.Web.Proxy/Helpers/KnownMethod.cs | 71 +++++++++ .../Network/Tcp/TcpConnectionFactory.cs | 2 + src/Titanium.Web.Proxy/WebSocketDecoder.cs | 140 ++++++++++++++++++ src/Titanium.Web.Proxy/WebSocketFrame.cs | 26 ++++ src/Titanium.Web.Proxy/WebsocketOpCode.cs | 12 ++ 10 files changed, 407 insertions(+), 67 deletions(-) create mode 100644 src/Titanium.Web.Proxy/Helpers/KnownMethod.cs create mode 100644 src/Titanium.Web.Proxy/WebSocketDecoder.cs create mode 100644 src/Titanium.Web.Proxy/WebSocketFrame.cs create mode 100644 src/Titanium.Web.Proxy/WebsocketOpCode.cs diff --git a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index 9ae48e1f2..67ecec8fb 100644 --- a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Security; +using System.Text; using System.Threading; using System.Threading.Tasks; using Titanium.Web.Proxy.EventArguments; @@ -10,6 +12,7 @@ using Titanium.Web.Proxy.Helpers; using Titanium.Web.Proxy.Http; using Titanium.Web.Proxy.Models; +using Titanium.Web.Proxy.StreamExtended.Network; namespace Titanium.Web.Proxy.Examples.Basic { @@ -22,6 +25,9 @@ public class ProxyTestController public ProxyTestController() { proxyServer = new ProxyServer(); + + proxyServer.EnableHttp2 = true; + // generate root certificate without storing it in file system //proxyServer.CertificateManager.CreateRootCertificate(false); @@ -32,11 +38,11 @@ public ProxyTestController() { if (exception is ProxyHttpException phex) { - await writeToConsole(exception.Message + ": " + phex.InnerException?.Message, true); + await writeToConsole(exception.Message + ": " + phex.InnerException?.Message, ConsoleColor.Red); } else { - await writeToConsole(exception.Message, true); + await writeToConsole(exception.Message, ConsoleColor.Red); } }; proxyServer.ForwardToUpstreamGateway = true; @@ -146,6 +152,26 @@ private async Task onBeforeTunnelConnectRequest(object sender, TunnelConnectSess } } + private void WebSocket_DataReceived(object sender, DataEventArgs e) + { + var args = (SessionEventArgs)sender; + + foreach (var frame in args.WebSocketDecoder.Decode(e.Buffer, e.Offset, e.Count)) + { + if (frame.OpCode == WebsocketOpCode.Binary) + { + var data = frame.Data.ToArray(); + string str = string.Join(",", data.ToArray().Select(x => x.ToString("X2"))); + writeToConsole(str, ConsoleColor.Blue).Wait(); + } + + if (frame.OpCode == WebsocketOpCode.Text) + { + writeToConsole(frame.GetText(), ConsoleColor.Blue).Wait(); + } + } + } + private Task onBeforeTunnelConnectResponse(object sender, TunnelConnectSessionEventArgs e) { return Task.FromResult(false); @@ -205,6 +231,11 @@ private async Task multipartRequestPartSent(object sender, MultipartRequestPartS private async Task onResponse(object sender, SessionEventArgs e) { + if (e.HttpClient.ConnectRequest?.TunnelType == TunnelType.Websocket) + { + e.DataReceived += WebSocket_DataReceived; + } + await writeToConsole("Active Server Connections:" + ((ProxyServer)sender).ServerConnectionCount); string ext = System.IO.Path.GetExtension(e.HttpClient.Request.RequestUri.AbsolutePath); @@ -277,14 +308,14 @@ public Task OnCertificateSelection(object sender, CertificateSelectionEventArgs return Task.FromResult(0); } - private async Task writeToConsole(string message, bool useRedColor = false) + private async Task writeToConsole(string message, ConsoleColor? consoleColor = null) { await @lock.WaitAsync(); - if (useRedColor) + if (consoleColor.HasValue) { ConsoleColor existing = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; + Console.ForegroundColor = consoleColor.Value; Console.WriteLine(message); Console.ForegroundColor = existing; } diff --git a/src/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/src/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index f9201877b..176d41299 100644 --- a/src/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/src/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -26,6 +26,8 @@ public class SessionEventArgs : SessionEventArgsBase /// private bool reRequest; + private WebSocketDecoder webSocketDecoder; + /// /// Is this session a HTTP/2 promise? /// @@ -58,6 +60,8 @@ public bool ReRequest } } + public WebSocketDecoder WebSocketDecoder => webSocketDecoder ??= new WebSocketDecoder(BufferPool); + /// /// Occurs when multipart request part sent. /// diff --git a/src/Titanium.Web.Proxy/ExplicitClientHandler.cs b/src/Titanium.Web.Proxy/ExplicitClientHandler.cs index f78076a80..471197455 100644 --- a/src/Titanium.Web.Proxy/ExplicitClientHandler.cs +++ b/src/Titanium.Web.Proxy/ExplicitClientHandler.cs @@ -47,9 +47,15 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect try { TunnelConnectSessionEventArgs? connectArgs = null; - + + var method = await HttpHelper.GetMethod(clientStream, BufferPool, cancellationToken); + if (clientStream.IsClosed) + { + return; + } + // Client wants to create a secure tcp tunnel (probably its a HTTPS or Websocket request) - if (await HttpHelper.IsConnectMethod(clientStream, BufferPool, cancellationToken) == 1) + if (method == KnownMethod.Connect) { // read the first line HTTP command var requestLine = await clientStream.ReadRequestLine(cancellationToken); @@ -75,6 +81,7 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect // filter out excluded host names bool decryptSsl = endPoint.DecryptSsl && connectArgs.DecryptSsl; + bool sendRawData = !decryptSsl; if (connectArgs.DenyConnect) { @@ -113,6 +120,10 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect await clientStream.WriteResponseAsync(response, cancellationToken); var clientHelloInfo = await SslTools.PeekClientHello(clientStream, BufferPool, cancellationToken); + if (clientStream.IsClosed) + { + return; + } bool isClientHello = clientHelloInfo != null; if (clientHelloInfo != null) @@ -224,31 +235,41 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect $"Couldn't authenticate host '{connectHostname}' with certificate '{certName}'.", e, connectArgs); } - if (await HttpHelper.IsConnectMethod(clientStream, BufferPool, cancellationToken) == -1) + method = await HttpHelper.GetMethod(clientStream, BufferPool, cancellationToken); + if (clientStream.IsClosed) { - decryptSsl = false; + return; } - if (!decryptSsl) + if (method == KnownMethod.Invalid) { + sendRawData = true; await tcpConnectionFactory.Release(prefetchConnectionTask, true); prefetchConnectionTask = null; } } + else if (clientHelloInfo == null) + { + method = await HttpHelper.GetMethod(clientStream, BufferPool, cancellationToken); + if (clientStream.IsClosed) + { + return; + } + } if (cancellationTokenSource.IsCancellationRequested) { throw new Exception("Session was terminated by user."); } - // Hostname is excluded or it is not an HTTPS connect - if (!decryptSsl || !isClientHello) + if (method == KnownMethod.Invalid) { - if (!isClientHello) - { - connectRequest.TunnelType = TunnelType.Websocket; - } + sendRawData = true; + } + // Hostname is excluded or it is not an HTTPS connect + if (sendRawData) + { // create new connection to server. // If we detected that client tunnel CONNECTs without SSL by checking for empty client hello then // this connection should not be HTTPS. @@ -302,7 +323,7 @@ await TcpHelper.SendRaw(clientStream, connection.Stream, BufferPool, } } - if (connectArgs != null && await HttpHelper.IsPriMethod(clientStream, BufferPool, cancellationToken) == 1) + if (connectArgs != null && method == KnownMethod.Pri) { // todo string? httpCmd = await clientStream.ReadLineAsync(cancellationToken); diff --git a/src/Titanium.Web.Proxy/Helpers/HttpHelper.cs b/src/Titanium.Web.Proxy/Helpers/HttpHelper.cs index bd7bbaa0f..d531928b5 100644 --- a/src/Titanium.Web.Proxy/Helpers/HttpHelper.cs +++ b/src/Titanium.Web.Proxy/Helpers/HttpHelper.cs @@ -167,32 +167,11 @@ internal static string GetWildCardDomainName(string hostname) } /// - /// Determines whether is connect method. + /// Gets the HTTP method from the stream. /// - /// 1: when CONNECT, 0: when valid HTTP method, -1: otherwise - internal static ValueTask IsConnectMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default) + public static async ValueTask GetMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default) { - return startsWith(httpReader, bufferPool, "CONNECT", cancellationToken); - } - - /// - /// Determines whether is pri method (HTTP/2). - /// - /// 1: when PRI, 0: when valid HTTP method, -1: otherwise - internal static ValueTask IsPriMethod(IPeekStream httpReader, IBufferPool bufferPool, CancellationToken cancellationToken = default) - { - return startsWith(httpReader, bufferPool, "PRI", cancellationToken); - } - - /// - /// Determines whether the stream starts with the given string. - /// - /// - /// 1: when starts with the given string, 0: when valid HTTP method, -1: otherwise - /// - private static async ValueTask startsWith(IPeekStream httpReader, IBufferPool bufferPool, string expectedStart, CancellationToken cancellationToken = default) - { - const int lengthToCheck = 10; + const int lengthToCheck = 20; if (bufferPool.BufferSize < lengthToCheck) { throw new Exception($"Buffer is too small. Minimum size is {lengthToCheck} bytes"); @@ -201,13 +180,12 @@ private static async ValueTask startsWith(IPeekStream httpReader, IBufferPo byte[] buffer = bufferPool.GetBuffer(bufferPool.BufferSize); try { - bool isExpected = true; int i = 0; while (i < lengthToCheck) { int peeked = await httpReader.PeekBytesAsync(buffer, i, i, lengthToCheck - i, cancellationToken); if (peeked <= 0) - return -1; + return KnownMethod.Invalid; peeked += i; @@ -216,27 +194,94 @@ private static async ValueTask startsWith(IPeekStream httpReader, IBufferPo int b = buffer[i]; if (b == ' ' && i > 2) - return isExpected ? 1 : 0; - else - { - char ch = (char)b; - if (ch < 'A' || ch > 'z' || (ch > 'Z' && ch < 'a')) // ASCII letter - return -1; - else if (i >= expectedStart.Length || ch != expectedStart[i]) - isExpected = false; - } + return getKnownMethod(buffer.AsSpan(0, i)); + + char ch = (char)b; + if ((ch < 'A' || ch > 'z' || (ch > 'Z' && ch < 'a')) && (ch != '-')) // ASCII letter + return KnownMethod.Invalid; i++; } } - // only letters - return 0; + // only letters, but no space (or shorter than 3 characters) + return KnownMethod.Invalid; } finally { bufferPool.ReturnBuffer(buffer); } } + + private static KnownMethod getKnownMethod(ReadOnlySpan method) + { + // the following methods are supported: + // Connect + // Delete + // Get + // Head + // Options + // Post + // Put + // Trace + // Pri + + // method parameter should have at least 3 bytes + byte b1 = method[0]; + byte b2 = method[1]; + byte b3 = method[2]; + + switch (method.Length) + { + case 3: + // Get or Put + if (b1 == 'G') + return b2 == 'E' && b3 == 'T' ? KnownMethod.Get : KnownMethod.Unknown; + + if (b1 == 'P') + { + if (b2 == 'U') + return b3 == 'T' ? KnownMethod.Put : KnownMethod.Unknown; + + if (b2 == 'R') + return b3 == 'I' ? KnownMethod.Pri : KnownMethod.Unknown; + } + + break; + case 4: + // Head or Post + if (b1 == 'H') + return b2 == 'E' && b3 == 'A' && method[3] == 'D' ? KnownMethod.Head : KnownMethod.Unknown; + + if (b1 == 'P') + return b2 == 'O' && b3 == 'S' && method[3] == 'T' ? KnownMethod.Post : KnownMethod.Unknown; + + break; + case 5: + // Trace + if (b1 == 'T') + return b2 == 'R' && b3 == 'A' && method[3] == 'C' && method[4] == 'E' ? KnownMethod.Trace : KnownMethod.Unknown; + + break; + case 6: + // Delete + if (b1 == 'D') + return b2 == 'E' && b3 == 'L' && method[3] == 'E' && method[4] == 'T' && method[5] == 'E' ? KnownMethod.Delete : KnownMethod.Unknown; + + break; + case 7: + // Connect or Options + if (b1 == 'C') + return b2 == 'O' && b3 == 'N' && method[3] == 'N' && method[4] == 'E' && method[5] == 'C' && method[6] == 'T' ? KnownMethod.Connect : KnownMethod.Unknown; + + if (b1 == 'O') + return b2 == 'P' && b3 == 'T' && method[3] == 'I' && method[4] == 'O' && method[5] == 'N' && method[6] == 'S' ? KnownMethod.Options : KnownMethod.Unknown; + + break; + } + + + return KnownMethod.Unknown; + } } } diff --git a/src/Titanium.Web.Proxy/Helpers/HttpStream.cs b/src/Titanium.Web.Proxy/Helpers/HttpStream.cs index 1e02631a4..c600489af 100644 --- a/src/Titanium.Web.Proxy/Helpers/HttpStream.cs +++ b/src/Titanium.Web.Proxy/Helpers/HttpStream.cs @@ -650,7 +650,7 @@ public async ValueTask FillBufferAsync(CancellationToken cancellationToken if (bufferDataLength == buffer.Length) { - resizeBuffer(ref buffer, bufferDataLength * 2); + Array.Resize(ref buffer, bufferDataLength * 2); } } } @@ -678,18 +678,6 @@ public async Task ReadAndIgnoreAllLinesAsync(CancellationToken cancellationToken } } - /// - /// Increase size of buffer and copy existing content to new buffer - /// - /// - /// - private static void resizeBuffer(ref byte[] buffer, long size) - { - var newBuffer = new byte[size]; - Buffer.BlockCopy(buffer, 0, newBuffer, 0, buffer.Length); - buffer = newBuffer; - } - /// /// Base Stream.BeginRead will call this.Read and block thread (we don't want this, Network stream handles async) /// In order to really async Reading Launch this.ReadAsync as Task will fire NetworkStream.ReadAsync diff --git a/src/Titanium.Web.Proxy/Helpers/KnownMethod.cs b/src/Titanium.Web.Proxy/Helpers/KnownMethod.cs new file mode 100644 index 000000000..eda343753 --- /dev/null +++ b/src/Titanium.Web.Proxy/Helpers/KnownMethod.cs @@ -0,0 +1,71 @@ +namespace Titanium.Web.Proxy.Helpers +{ + internal enum KnownMethod + { + Unknown, + Invalid, + + // RFC 7231: Hypertext Transfer Protocol (HTTP/1.1): Semantics and Content + Connect, + Delete, + Get, + Head, + Options, + Post, + Put, + Trace, + + // RFC 7540: Hypertext Transfer Protocol Version 2 + Pri, + + // RFC 5789: PATCH Method for HTTP + Patch, + + // RFC 3744: Web Distributed Authoring and Versioning (WebDAV) Access Control Protocol + Acl, + + // RFC 3253: Versioning Extensions to WebDAV (Web Distributed Authoring and Versioning) + BaselineControl, + Checkin, + Checkout, + Label, + Merge, + Mkactivity, + Mkworkspace, + Report, + Unckeckout, + Update, + VersionControl, + + // RFC 3648: Web Distributed Authoring and Versioning (WebDAV) Ordered Collections Protocol + Orderpatch, + + // RFC 4437: Web Distributed Authoring and Versioning (WebDAV): Redirect Reference Resources + Mkredirectref, + Updateredirectref, + + // RFC 4791: Calendaring Extensions to WebDAV (CalDAV) + Mkcalendar, + + // RFC 4918: HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV) + Copy, + Lock, + Mkcol, + Move, + Propfind, + Proppatch, + Unlock, + + // RFC 5323: Web Distributed Authoring and Versioning (WebDAV) SEARCH + Search, + + // RFC 5842: Binding Extensions to Web Distributed Authoring and Versioning (WebDAV) + Bind, + Rebind, + Unbind, + + // Internet Draft snell-link-method: HTTP Link and Unlink Methods + Link, + Unlink, + } +} diff --git a/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs b/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs index 6d3c4ae79..0c1095f0b 100644 --- a/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs +++ b/src/Titanium.Web.Proxy/Network/Tcp/TcpConnectionFactory.cs @@ -60,6 +60,7 @@ internal string GetConnectionCacheKey(string remoteHostName, int remotePort, cacheKeyBuilder.Append("-"); cacheKeyBuilder.Append(remotePort); cacheKeyBuilder.Append("-"); + // when creating Tcp client isConnect won't matter cacheKeyBuilder.Append(isHttps); @@ -408,6 +409,7 @@ private async Task createServerConnection(string remoteHost continue; } + break; } catch (Exception e) diff --git a/src/Titanium.Web.Proxy/WebSocketDecoder.cs b/src/Titanium.Web.Proxy/WebSocketDecoder.cs new file mode 100644 index 000000000..7d84860e8 --- /dev/null +++ b/src/Titanium.Web.Proxy/WebSocketDecoder.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Titanium.Web.Proxy.StreamExtended.BufferPool; + +namespace Titanium.Web.Proxy +{ + public class WebSocketDecoder + { + private byte[] buffer; + + private long bufferLength; + + internal WebSocketDecoder(IBufferPool bufferPool) + { + buffer = new byte[bufferPool.BufferSize]; + } + + public IEnumerable Decode(byte[] data, int offset, int count) + { + var buffer = data.AsMemory(offset, count); + + bool copied = false; + if (bufferLength > 0) + { + // already have remaining data + buffer = copyToBuffer(buffer); + copied = true; + } + + while (true) + { + var data1 = buffer.Span; + if (!isDataEnough(data1)) + { + break; + } + + var opCode = (WebsocketOpCode)(data1[0] & 0xf); + byte b = data1[1]; + long size = b & 0x7f; + + // todo: size > int.Max?? + + bool masked = (b & 0x80) != 0; + + int idx = 2; + if (size > 125) + { + if (size == 126) + { + size = (data1[2] << 8) + data1[3]; + idx = 4; + } + else + { + size = ((long)data1[2] << 56) + ((long)data1[3] << 48) + ((long)data1[4] << 40) + ((long)data1[5] << 32) + + ((long)data1[6] << 24) + (data1[7] << 16) + (data1[8] << 8) + data1[9]; + idx = 10; + } + } + + uint mask = 0; + if (masked) + { + //mask = (uint)(((long)data1[idx++] << 24) + (data1[idx++] << 16) + (data1[idx++] << 8) + data1[idx++]); + mask = (uint)(data1[idx++] + (data1[idx++] << 8) + (data1[idx++] << 16) + ((long)data1[idx++] << 24)); + } + + if (masked) + { + uint m = mask; + for (int i = 0; i < size; i++) + { + data[i + idx] = (byte)(data1[i + idx] ^ (byte)mask); + + m >>= 8; + + if (m == 0) + m = mask; + } + } + + var frameData = buffer.Slice(idx, (int)size); + var frame = new WebSocketFrame { Data = frameData, OpCode = opCode }; + yield return frame; + + buffer = buffer.Slice((int)(idx + size)); + } + + if (!copied && buffer.Length > 0) + { + copyToBuffer(buffer); + } + } + + private Memory copyToBuffer(ReadOnlyMemory data) + { + long requiredLength = bufferLength + data.Length; + if (requiredLength > buffer.Length) + { + Array.Resize(ref buffer, (int)Math.Min(requiredLength, buffer.Length * 2)); + } + + data.CopyTo(buffer.AsMemory((int)bufferLength)); + bufferLength += data.Length; + return buffer.AsMemory(0, (int)bufferLength); + } + + private static bool isDataEnough(ReadOnlySpan data) + { + int length = data.Length; + if (length < 2) + return false; + + byte size = data[1]; + if ((size & 0x80) != 0) // masked + length -= 4; + + size &= 0x7f; + + if (size == 126) + { + if (length < 2) + { + return false; + } + } + else if (size == 127) + { + if (length < 10) + { + return false; + } + } + + return length >= size; + } + } +} diff --git a/src/Titanium.Web.Proxy/WebSocketFrame.cs b/src/Titanium.Web.Proxy/WebSocketFrame.cs new file mode 100644 index 000000000..edb28c99d --- /dev/null +++ b/src/Titanium.Web.Proxy/WebSocketFrame.cs @@ -0,0 +1,26 @@ +using System; +using System.Text; + +namespace Titanium.Web.Proxy +{ + public class WebSocketFrame + { + public WebsocketOpCode OpCode { get; internal set; } + + public ReadOnlyMemory Data { get; internal set; } + + public string GetText() + { + return GetText(Encoding.UTF8); + } + + public string GetText(Encoding encoding) + { +#if NETSTANDARD2_1 + return encoding.GetString(Data.Span); +#else + return encoding.GetString(Data.ToArray()); +#endif + } + } +} diff --git a/src/Titanium.Web.Proxy/WebsocketOpCode.cs b/src/Titanium.Web.Proxy/WebsocketOpCode.cs new file mode 100644 index 000000000..93eb763b5 --- /dev/null +++ b/src/Titanium.Web.Proxy/WebsocketOpCode.cs @@ -0,0 +1,12 @@ +namespace Titanium.Web.Proxy +{ + public enum WebsocketOpCode : byte + { + Continuation, + Text, + Binary, + ConnectionClose = 8, + Ping, + Pong, + } +} From c46293d56cfb5b5f8a6bebebfc1daeb3042a6412 Mon Sep 17 00:00:00 2001 From: Honfika Date: Sat, 30 Nov 2019 21:31:51 +0100 Subject: [PATCH 3/5] better websocker unmask --- .../ProxyTestController.cs | 17 ++++++- src/Titanium.Web.Proxy/WebSocketDecoder.cs | 49 ++++++++++++++----- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index 67ecec8fb..c9bdd3eb7 100644 --- a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -152,9 +152,21 @@ private async Task onBeforeTunnelConnectRequest(object sender, TunnelConnectSess } } + private void WebSocket_DataSent(object sender, DataEventArgs e) + { + var args = (SessionEventArgs)sender; + WebSocketDataSentReceived(args, e, true); + } + private void WebSocket_DataReceived(object sender, DataEventArgs e) { var args = (SessionEventArgs)sender; + WebSocketDataSentReceived(args, e, false); + } + + private void WebSocketDataSentReceived(SessionEventArgs args, DataEventArgs e, bool sent) + { + var color = sent ? ConsoleColor.Green : ConsoleColor.Blue; foreach (var frame in args.WebSocketDecoder.Decode(e.Buffer, e.Offset, e.Count)) { @@ -162,12 +174,12 @@ private void WebSocket_DataReceived(object sender, DataEventArgs e) { var data = frame.Data.ToArray(); string str = string.Join(",", data.ToArray().Select(x => x.ToString("X2"))); - writeToConsole(str, ConsoleColor.Blue).Wait(); + writeToConsole(str, color).Wait(); } if (frame.OpCode == WebsocketOpCode.Text) { - writeToConsole(frame.GetText(), ConsoleColor.Blue).Wait(); + writeToConsole(frame.GetText(), color).Wait(); } } } @@ -233,6 +245,7 @@ private async Task onResponse(object sender, SessionEventArgs e) { if (e.HttpClient.ConnectRequest?.TunnelType == TunnelType.Websocket) { + e.DataSent += WebSocket_DataSent; e.DataReceived += WebSocket_DataReceived; } diff --git a/src/Titanium.Web.Proxy/WebSocketDecoder.cs b/src/Titanium.Web.Proxy/WebSocketDecoder.cs index 7d84860e8..58591b1b2 100644 --- a/src/Titanium.Web.Proxy/WebSocketDecoder.cs +++ b/src/Titanium.Web.Proxy/WebSocketDecoder.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using Titanium.Web.Proxy.StreamExtended.BufferPool; namespace Titanium.Web.Proxy @@ -60,25 +61,51 @@ public IEnumerable Decode(byte[] data, int offset, int count) } } - uint mask = 0; - if (masked) + if (size < 0) { - //mask = (uint)(((long)data1[idx++] << 24) + (data1[idx++] << 16) + (data1[idx++] << 8) + data1[idx++]); - mask = (uint)(data1[idx++] + (data1[idx++] << 8) + (data1[idx++] << 16) + ((long)data1[idx++] << 24)); + ; + } + + if (data1.Length < idx + size) + { + break; } if (masked) { - uint m = mask; - for (int i = 0; i < size; i++) - { - data[i + idx] = (byte)(data1[i + idx] ^ (byte)mask); + //mask = (uint)(((long)data1[idx++] << 24) + (data1[idx++] << 16) + (data1[idx++] << 8) + data1[idx++]); + //mask = (uint)(data1[idx++] + (data1[idx++] << 8) + (data1[idx++] << 16) + ((long)data1[idx++] << 24)); + var uData = MemoryMarshal.Cast(data1.Slice(idx, (int)size + 4)); + idx += 4; - m >>= 8; + uint mask = uData[0]; + long size1 = size; + if (size > 4) + { + uData = uData.Slice(1); + for (int i = 0; i < uData.Length; i++) + { + uData[i] = uData[i] ^ mask; + } - if (m == 0) - m = mask; + size1 -= uData.Length * 4; } + + if (size1 > 0) + { + int pos = (int)(idx + size - size1); + data1[pos] ^= (byte)mask; + + if (size1 > 1) + { + data1[pos + 1] ^= (byte)(mask >> 8); + } + + if (size1 > 2) + { + data1[pos + 2] ^= (byte)(mask >> 16); + } +; } } var frameData = buffer.Slice(idx, (int)size); From 4f43bd16c747de54f76c0651659b28002ddf5ffc Mon Sep 17 00:00:00 2001 From: Honfika Date: Sat, 30 Nov 2019 21:39:35 +0100 Subject: [PATCH 4/5] WebSocket IsFinal flad added --- src/Titanium.Web.Proxy/WebSocketDecoder.cs | 8 ++------ src/Titanium.Web.Proxy/WebSocketFrame.cs | 2 ++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Titanium.Web.Proxy/WebSocketDecoder.cs b/src/Titanium.Web.Proxy/WebSocketDecoder.cs index 58591b1b2..c65648ba2 100644 --- a/src/Titanium.Web.Proxy/WebSocketDecoder.cs +++ b/src/Titanium.Web.Proxy/WebSocketDecoder.cs @@ -38,6 +38,7 @@ public IEnumerable Decode(byte[] data, int offset, int count) } var opCode = (WebsocketOpCode)(data1[0] & 0xf); + bool isFinal = (data1[0] & 0x80) != 0; byte b = data1[1]; long size = b & 0x7f; @@ -61,11 +62,6 @@ public IEnumerable Decode(byte[] data, int offset, int count) } } - if (size < 0) - { - ; - } - if (data1.Length < idx + size) { break; @@ -109,7 +105,7 @@ public IEnumerable Decode(byte[] data, int offset, int count) } var frameData = buffer.Slice(idx, (int)size); - var frame = new WebSocketFrame { Data = frameData, OpCode = opCode }; + var frame = new WebSocketFrame { IsFinal = isFinal, Data = frameData, OpCode = opCode }; yield return frame; buffer = buffer.Slice((int)(idx + size)); diff --git a/src/Titanium.Web.Proxy/WebSocketFrame.cs b/src/Titanium.Web.Proxy/WebSocketFrame.cs index edb28c99d..b30067a99 100644 --- a/src/Titanium.Web.Proxy/WebSocketFrame.cs +++ b/src/Titanium.Web.Proxy/WebSocketFrame.cs @@ -5,6 +5,8 @@ namespace Titanium.Web.Proxy { public class WebSocketFrame { + public bool IsFinal { get; internal set; } + public WebsocketOpCode OpCode { get; internal set; } public ReadOnlyMemory Data { get; internal set; } From 4febcb0512d8b0899da92fd1ae082c1a6bb127a3 Mon Sep 17 00:00:00 2001 From: Honfika Date: Sat, 30 Nov 2019 22:53:45 +0100 Subject: [PATCH 5/5] less exceptions --- .../ProxyTestController.cs | 2 +- .../ExplicitClientHandler.cs | 41 ++-- src/Titanium.Web.Proxy/Helpers/HttpStream.cs | 194 +++++++++++++++--- src/Titanium.Web.Proxy/Helpers/NullWriter.cs | 40 ++-- src/Titanium.Web.Proxy/Net45Compatibility.cs | 2 + 5 files changed, 216 insertions(+), 63 deletions(-) diff --git a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index c9bdd3eb7..def5d9448 100644 --- a/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -26,7 +26,7 @@ public ProxyTestController() { proxyServer = new ProxyServer(); - proxyServer.EnableHttp2 = true; + //proxyServer.EnableHttp2 = true; // generate root certificate without storing it in file system //proxyServer.CertificateManager.CreateRootCertificate(false); diff --git a/src/Titanium.Web.Proxy/ExplicitClientHandler.cs b/src/Titanium.Web.Proxy/ExplicitClientHandler.cs index 471197455..6c029bf27 100644 --- a/src/Titanium.Web.Proxy/ExplicitClientHandler.cs +++ b/src/Titanium.Web.Proxy/ExplicitClientHandler.cs @@ -141,26 +141,29 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect bool http2Supported = false; - var alpn = clientHelloInfo.GetAlpn(); - if (alpn != null && alpn.Contains(SslApplicationProtocol.Http2)) + if (EnableHttp2) { - // test server HTTP/2 support - try - { - // todo: this is a hack, because Titanium does not support HTTP protocol changing currently - var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs, - true, SslExtensions.Http2ProtocolAsList, - true, cancellationToken); - - http2Supported = connection.NegotiatedApplicationProtocol == - SslApplicationProtocol.Http2; - - // release connection back to pool instead of closing when connection pool is enabled. - await tcpConnectionFactory.Release(connection, true); - } - catch (Exception) + var alpn = clientHelloInfo.GetAlpn(); + if (alpn != null && alpn.Contains(SslApplicationProtocol.Http2)) { - // ignore + // test server HTTP/2 support + try + { + // todo: this is a hack, because Titanium does not support HTTP protocol changing currently + var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs, + true, SslExtensions.Http2ProtocolAsList, + true, cancellationToken); + + http2Supported = connection.NegotiatedApplicationProtocol == + SslApplicationProtocol.Http2; + + // release connection back to pool instead of closing when connection pool is enabled. + await tcpConnectionFactory.Release(connection, true); + } + catch (Exception) + { + // ignore + } } } @@ -274,7 +277,7 @@ private async Task handleClient(ExplicitProxyEndPoint endPoint, TcpClientConnect // If we detected that client tunnel CONNECTs without SSL by checking for empty client hello then // this connection should not be HTTPS. var connection = await tcpConnectionFactory.GetServerConnection(this, connectArgs, - true, SslExtensions.Http2ProtocolAsList, + true, null, true, cancellationToken); try diff --git a/src/Titanium.Web.Proxy/Helpers/HttpStream.cs b/src/Titanium.Web.Proxy/Helpers/HttpStream.cs index c600489af..82505dd85 100644 --- a/src/Titanium.Web.Proxy/Helpers/HttpStream.cs +++ b/src/Titanium.Web.Proxy/Helpers/HttpStream.cs @@ -33,7 +33,8 @@ internal class HttpStream : Stream, IHttpStreamWriter, IHttpStreamReader, IPeekS private bool disposed; - private bool closed; + private bool closedWrite; + private bool closedRead; private readonly IBufferPool bufferPool; @@ -43,7 +44,7 @@ internal class HttpStream : Stream, IHttpStreamWriter, IHttpStreamReader, IPeekS private Stream baseStream { get; } - public bool IsClosed => closed; + public bool IsClosed => closedRead; static HttpStream() { @@ -89,7 +90,21 @@ internal HttpStream(Stream baseStream, IBufferPool bufferPool, bool leaveOpen = /// public override void Flush() { - baseStream.Flush(); + if (closedWrite) + { + return; + } + + try + { + baseStream.Flush(); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } /// @@ -153,7 +168,22 @@ public override int Read(byte[] buffer, int offset, int count) public override void Write(byte[] buffer, int offset, int count) { OnDataWrite(buffer, offset, count); - baseStream.Write(buffer, offset, count); + + if (closedWrite) + { + return; + } + + try + { + baseStream.Write(buffer, offset, count); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } /// @@ -184,9 +214,23 @@ public override async Task CopyToAsync(Stream destination, int bufferSize, Cance /// /// A task that represents the asynchronous flush operation. /// - public override Task FlushAsync(CancellationToken cancellationToken) + public override async Task FlushAsync(CancellationToken cancellationToken) { - return baseStream.FlushAsync(cancellationToken); + if (closedWrite) + { + return; + } + + try + { + await baseStream.FlushAsync(cancellationToken); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } /// @@ -393,7 +437,22 @@ public byte ReadByteFromBuffer() public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { OnDataWrite(buffer, offset, count); - await baseStream.WriteAsync(buffer, offset, count, cancellationToken); + + if (closedWrite) + { + return; + } + + try + { + await baseStream.WriteAsync(buffer, offset, count, cancellationToken); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } /// @@ -402,6 +461,11 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc /// The byte to write to the stream. public override void WriteByte(byte value) { + if (closedWrite) + { + return; + } + var buffer = bufferPool.GetBuffer(); try { @@ -409,6 +473,12 @@ public override void WriteByte(byte value) OnDataWrite(buffer, 0, 1); baseStream.Write(buffer, 0, 1); } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } finally { bufferPool.ReturnBuffer(buffer); @@ -434,7 +504,8 @@ protected override void Dispose(bool disposing) if (!disposed) { disposed = true; - closed = true; + closedRead = true; + closedWrite = true; if (!leaveOpen) { baseStream.Dispose(); @@ -511,7 +582,7 @@ public override int WriteTimeout /// public bool FillBuffer() { - if (closed) + if (closedRead) { throw new Exception("Stream is already closed"); } @@ -536,11 +607,17 @@ public bool FillBuffer() bufferLength += readBytes; } } + catch + { + if (!swallowException) + throw; + } finally { if (!result) { - closed = true; + closedRead = true; + closedWrite = true; } } @@ -555,7 +632,7 @@ public bool FillBuffer() /// public async ValueTask FillBufferAsync(CancellationToken cancellationToken = default) { - if (closed) + if (closedRead) { throw new Exception("Stream is already closed"); } @@ -595,7 +672,8 @@ public async ValueTask FillBufferAsync(CancellationToken cancellationToken { if (!result) { - closed = true; + closedRead = true; + closedWrite = true; } } @@ -766,6 +844,11 @@ public ValueTask WriteLineAsync(CancellationToken cancellationToken = default) private async ValueTask writeAsyncInternal(string value, bool addNewLine, CancellationToken cancellationToken) { + if (closedWrite) + { + return; + } + int newLineChars = addNewLine ? newLine.Length : 0; int charCount = value.Length; if (charCount < bufferPool.BufferSize - newLineChars) @@ -782,6 +865,12 @@ private async ValueTask writeAsyncInternal(string value, bool addNewLine, Cancel await baseStream.WriteAsync(buffer, 0, idx, cancellationToken); } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } finally { bufferPool.ReturnBuffer(buffer); @@ -797,7 +886,16 @@ private async ValueTask writeAsyncInternal(string value, bool addNewLine, Cancel idx += newLineChars; } - await baseStream.WriteAsync(buffer, 0, idx, cancellationToken); + try + { + await baseStream.WriteAsync(buffer, 0, idx, cancellationToken); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } } @@ -826,20 +924,48 @@ internal async Task WriteHeadersAsync(HeaderBuilder headerBuilder, CancellationT /// The cancellation token. internal async ValueTask WriteAsync(byte[] data, bool flush = false, CancellationToken cancellationToken = default) { - await baseStream.WriteAsync(data, 0, data.Length, cancellationToken); - if (flush) + if (closedWrite) { - await baseStream.FlushAsync(cancellationToken); + return; + } + + try + { + await baseStream.WriteAsync(data, 0, data.Length, cancellationToken); + if (flush) + { + await baseStream.FlushAsync(cancellationToken); + } + } + catch + { + closedWrite = true; + if (!swallowException) + throw; } } internal async Task WriteAsync(byte[] data, int offset, int count, bool flush, CancellationToken cancellationToken = default) { - await baseStream.WriteAsync(data, offset, count, cancellationToken); - if (flush) + if (closedWrite) { - await baseStream.FlushAsync(cancellationToken); + return; + } + + try + { + await baseStream.WriteAsync(data, offset, count, cancellationToken); + if (flush) + { + await baseStream.FlushAsync(cancellationToken); + } + } + catch + { + closedWrite = true; + if (!swallowException) + throw; } } @@ -1056,9 +1182,23 @@ protected async ValueTask WriteAsync(RequestResponseBase requestResponse, Header /// The buffer to write data from. /// The token to monitor for cancellation requests. The default value is . /// A task that represents the asynchronous write operation. - public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) { - return baseStream.WriteAsync(buffer, cancellationToken); + if (closedWrite) + { + return; + } + + try + { + await baseStream.WriteAsync(buffer, cancellationToken); + } + catch + { + closedWrite = true; + if (!swallowException) + throw; + } } #else /// @@ -1067,11 +1207,19 @@ public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTo /// The buffer to write data from. /// The token to monitor for cancellation requests. The default value is . /// A task that represents the asynchronous write operation. - public Task WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) + public async Task WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) { var buf = ArrayPool.Shared.Rent(buffer.Length); buffer.CopyTo(buf); - return baseStream.WriteAsync(buf, 0, buf.Length, cancellationToken); + try + { + await baseStream.WriteAsync(buf, 0, buf.Length, cancellationToken); + } + catch + { + if (!swallowException) + throw; + } } #endif } diff --git a/src/Titanium.Web.Proxy/Helpers/NullWriter.cs b/src/Titanium.Web.Proxy/Helpers/NullWriter.cs index 503fb7b9e..d74161b69 100644 --- a/src/Titanium.Web.Proxy/Helpers/NullWriter.cs +++ b/src/Titanium.Web.Proxy/Helpers/NullWriter.cs @@ -2,33 +2,33 @@ using System.Threading.Tasks; using Titanium.Web.Proxy.StreamExtended.Network; -internal class NullWriter : IHttpStreamWriter +namespace Titanium.Web.Proxy.Helpers { - public static NullWriter Instance { get; } = new NullWriter(); - - public void Write(byte[] buffer, int offset, int count) + internal class NullWriter : IHttpStreamWriter { - } + public static NullWriter Instance { get; } = new NullWriter(); -#if NET45 - public async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - } + public void Write(byte[] buffer, int offset, int count) + { + } + public Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { +#if NET45 + return Net45Compatibility.CompletedTask; #else - public Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + return Task.CompletedTask; #endif + } - public ValueTask WriteLineAsync(CancellationToken cancellationToken = default) - { - throw new System.NotImplementedException(); - } + public ValueTask WriteLineAsync(CancellationToken cancellationToken = default) + { + throw new System.NotImplementedException(); + } - public ValueTask WriteLineAsync(string value, CancellationToken cancellationToken = default) - { - throw new System.NotImplementedException(); + public ValueTask WriteLineAsync(string value, CancellationToken cancellationToken = default) + { + throw new System.NotImplementedException(); + } } } diff --git a/src/Titanium.Web.Proxy/Net45Compatibility.cs b/src/Titanium.Web.Proxy/Net45Compatibility.cs index ffd9a4b91..10abc197e 100644 --- a/src/Titanium.Web.Proxy/Net45Compatibility.cs +++ b/src/Titanium.Web.Proxy/Net45Compatibility.cs @@ -10,6 +10,8 @@ namespace Titanium.Web.Proxy class Net45Compatibility { public static byte[] EmptyArray = new byte[0]; + + public static Task CompletedTask = new Task(() => { }); } } #endif