From 47380f307c17da2bc4eddc1aefb779ef164d01d8 Mon Sep 17 00:00:00 2001 From: "pavel.mash" Date: Mon, 5 Nov 2018 00:42:06 +0200 Subject: [PATCH 1/8] WIP: initial support for Unix Domain Socket --- SynCrtSock.pas | 79 +++++++++++++++++++++++++++++--------------------- SynFPCSock.pas | 7 +++++ 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index 0e0d3cdd9..e4fc2e239 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -431,6 +431,7 @@ TCrtSocket = class // - expects the port to be specified as Ansi string, e.g. '1234' // - you can optionally specify a server address to bind to, e.g. // '1.2.3.4:1234' + // - for unix domain socket use unix:/path/to/file constructor Bind(const aPort: SockString; aLayer: TCrtSocketLayer=cslTCP); /// low-level internal method called by Open() and Bind() constructors // - raise an ECrtSocket exception on error @@ -2000,6 +2001,9 @@ THttpServer = class(THttpServerGeneric) /// the resource address // - e.g. '/category/name/10?param=1' Address: SockString; + /// cslUnix for unix socket URI + // http://unix:/path/to/socket.sock:/url/path + Layer: TCrtSocketLayer; /// fill the members from a supplied URI function From(aURI: SockString; const DefaultPort: SockString=''): boolean; /// compute the whole normalized URI @@ -2469,7 +2473,8 @@ function Open(const aServer, aPort: SockString; aTLS: boolean=false): TCrtSocket /// create a THttpClientSocket, returning nil on error // - useful to easily catch socket error exception ECrtSocket -function OpenHttp(const aServer, aPort: SockString; aTLS: boolean=false): THttpClientSocket; overload; +function OpenHttp(const aServer, aPort: SockString; aTLS: boolean=false; + aLayer: TCrtSocketLayer = cslTCP): THttpClientSocket; overload; /// create a THttpClientSocket, returning nil on error // - useful to easily catch socket error exception ECrtSocket @@ -2479,7 +2484,8 @@ function OpenHttp(const aURI: SockString; aAddress: PSockString=nil): THttpClien // - this method will use a low-level THttpClientSock socket: if you want // something able to use your computer proxy, take a look at TWinINet.Get() function HttpGet(const server, port: SockString; const url: SockString; - const inHeaders: SockString; outHeaders: PSockString=nil): SockString; overload; + const inHeaders: SockString; outHeaders: PSockString=nil; + aLayer: TCrtSocketLayer = cslTCP): SockString; overload; /// retrieve the content of a web page, using the HTTP/1.1 protocol and GET method // - this method will use a low-level THttpClientSock socket for plain http URI, @@ -4310,10 +4316,12 @@ function GetMacAddressesText: SockString; const DEFAULT_PORT: array[boolean] of SockString = ('80','443'); + UNIX_LOW = ord('u')+ord('n')shl 8+ord('i')shl 16+ord('x')shl 24; procedure TURI.Clear; begin Https := false; + Layer := cslTCP; Finalize(self); end; @@ -4335,7 +4343,13 @@ function TURI.From(aURI: SockString; const DefaultPort: SockString): boolean; P := S+3; end; S := P; - while not (S^ in [#0,':','/']) do inc(S); + if (PInteger(S)^ = UNIX_LOW) and (S[4] = ':') then begin //http://unix:...:/path + Inc(S, 5); + Inc(P, 5); + Layer := cslUNIX; + while not (S^ in [#0,':']) do inc(S); + end else + while not (S^ in [#0,':','/']) do inc(S); SetString(Server,P,S-P); if S^=':' then begin inc(S); @@ -4450,9 +4464,9 @@ function CallServer(const Server, Port: SockString; doBind: boolean; aLayer: TCrtSocketLayer; ConnectTimeout: DWORD): TSocket; var sin: TVarSin; IP: SockString; - socktype, ipproto: integer; + socktype, ipproto, family: integer; {$ifndef MSWINDOWS} - serveraddr: TSockAddr; + //serveraddr: sockaddr_un; {$endif} begin result := -1; @@ -4468,36 +4482,25 @@ function CallServer(const Server, Port: SockString; doBind: boolean; cslUNIX: begin {$ifdef MSWINDOWS} exit; // not handled under Win32 - {$else} // special version for UNIX sockets + {$else} socktype := SOCK_STREAM; ipproto := 0; - result := socket(AF_UNIX,socktype,ipproto); - if result<0 then - exit; - if doBind then begin - fillchar(serveraddr,sizeof(serveraddr),0); - //http://publib.boulder.ibm.com/infocenter/iseries/v5r3/index.jsp?topic=/rzab6/rzab6uafunix.htm - {$ifdef KYLIX3} - if (Libc.bind(result,serveraddr,sizeof(serveraddr))<0) or - (Libc.listen(result,SOMAXCONN)<0) then begin - {$else} - if (fpbind(result,@serveraddr,sizeof(serveraddr))<0) or - (fplisten(result,SOMAXCONN)<0) then begin - {$endif} - result := -1; - end; - end; - exit; {$endif} end; else exit; end; if SameText(Server,'localhost') {$ifndef MSWINDOWS}or ((Server='') and not doBind){$endif} then - IP := cLocalHost else + IP := cLocalHost + else if (aLayer=cslUNIX) then + IP := Server + else IP := ResolveName(Server,AF_INET,ipproto,socktype); // use AF_INET instead of AF_UNSPEC: IP6 is buggy! - if SetVarSin(sin,IP,Port,AF_INET,ipproto,socktype,false)<>0 then + if (aLayer=cslUNIX) then + family := AF_UNIX else + family := AF_INET; + if SetVarSin(sin,IP,Port,family,ipproto,socktype,false)<>0 then exit; result := Socket(integer(sin.AddressFamily),socktype,ipproto); if result=-1 then @@ -4637,6 +4640,12 @@ constructor TCrtSocket.Bind(const aPort: SockString; aLayer: TCrtSocketLayer=csl s := '0.0.0.0'; p := aPort; end; + {$ifdef UNIX} + if s = 'unix' then begin + aLayer := cslUNIX; + s := p; p := ''; + end; + {$endif} OpenBind(s,p,true,-1,aLayer); // raise an ECrtSocket exception on error end; @@ -4687,7 +4696,7 @@ procedure TCrtSocket.SetInt32OptionByIndex(OptName, OptVal: integer); procedure TCrtSocket.AcceptRequest(aClientSock: TSocket; aRemoteIP: PSockString); begin CreateSockIn; // use SockIn by default if not already initialized: 2x faster - OpenBind('','',false,aClientSock); // set the ACCEPTed aClientSock + OpenBind('','',false,aClientSock, fSocketLayer); // set the ACCEPTed aClientSock Linger := 5; // should remain open for 5 seconds after a closesocket() call if aRemoteIP<>nil then aRemoteIP^ := GetRemoteIP(aClientSock); @@ -4701,7 +4710,7 @@ procedure TCrtSocket.OpenBind(const aServer, aPort: SockString; begin fSocketLayer := aLayer; if aSock<0 then begin - if aPort='' then + if (aPort='') and (aLayer<>cslUNIX) then fPort := DEFAULT_PORT[aTLS] else // default port is 80/443 (HTTP/S) fPort := aPort; fServer := aServer; @@ -5417,10 +5426,11 @@ function Open(const aServer, aPort: SockString; aTLS: boolean): TCrtSocket; end; end; -function OpenHttp(const aServer, aPort: SockString; aTLS: boolean): THttpClientSocket; +function OpenHttp(const aServer, aPort: SockString; aTLS: boolean; + aLayer: TCrtSocketLayer = cslTCP): THttpClientSocket; begin try - result := THttpClientSocket.Open(aServer,aPort,cslTCP,0,aTLS); // HTTP_DEFAULT_RECEIVETIMEOUT + result := THttpClientSocket.Open(aServer,aPort,aLayer,0,aTLS); // HTTP_DEFAULT_RECEIVETIMEOUT except on ECrtSocket do result := nil; @@ -5432,18 +5442,19 @@ function OpenHttp(const aURI: SockString; aAddress: PSockString): THttpClientSoc begin result := nil; if URI.From(aURI) then begin - result := OpenHttp(URI.Server,URI.Port,URI.Https); + result := OpenHttp(URI.Server,URI.Port,URI.Https,URI.Layer); if aAddress <> nil then aAddress^ := URI.Address; end; end; function HttpGet(const server, port: SockString; const url: SockString; - const inHeaders: SockString; outHeaders: PSockString): SockString; + const inHeaders: SockString; outHeaders: PSockString; + aLayer: TCrtSocketLayer = cslTCP): SockString; var Http: THttpClientSocket; begin result := ''; - Http := OpenHttp(server,port); + Http := OpenHttp(server,port,false,aLayer); if Http<>nil then try if Http.Get(url,0,inHeaders) in [STATUS_SUCCESS..STATUS_PARTIALCONTENT] then begin @@ -5475,7 +5486,7 @@ function HttpGet(const aURI: SockString; const inHeaders: SockString; outHeaders raise ECrtSocket.CreateFmt('https is not supported by HttpGet(%s)',[aURI]) else {$endif} {$endif USEWININET} - result := HttpGet(URI.Server,URI.Port,URI.Address,inHeaders,outHeaders) else + result := HttpGet(URI.Server,URI.Port,URI.Address,inHeaders,outHeaders,URI.Layer) else result := ''; {$ifdef LINUX_RAWDEBUGVOIDHTTPGET} if result='' then @@ -6548,6 +6559,7 @@ constructor THttpServerSocket.Create(aServer: THttpServer); fServer := aServer; fCompress := aServer.fCompress; fCompressAcceptEncoding := aServer.fCompressAcceptEncoding; + fSocketLayer:=aServer.Sock.SocketLayer; TCPPrefix := aServer.TCPPrefix; end; end; @@ -11447,6 +11459,7 @@ procedure TCurlHTTP.InternalCreateRequest(const method, aURL: SockString); var url: SockString; begin url := fRootURL+aURL; + //curl.easy_setopt(fHandle,coTCPNoDelay,0); // disable Nagle curl.easy_setopt(fHandle,coURL,pointer(url)); if fProxyName<>'' then curl.easy_setopt(fHandle,coProxy,pointer(fProxyName)); diff --git a/SynFPCSock.pas b/SynFPCSock.pas index cf75a6f6d..89402a71e 100644 --- a/SynFPCSock.pas +++ b/SynFPCSock.pas @@ -441,6 +441,7 @@ function fpsend(s:cint; msg:pointer; len:size_t; flags:cint): ssize_t; inline; sin6_flowinfo: longword; sin6_addr: TInAddr6; sin6_scope_id: longword); + AF_UNIX: (sun_path: array[0..{$ifdef SOCK_HAS_SINLEN}104{$else}107{$endif}] of Char); ); end; @@ -710,6 +711,7 @@ function SizeOfVarSin(sin: TVarSin): integer; case sin.sin_family of AF_INET: result := SizeOf(TSockAddrIn); AF_INET6: result := SizeOf(TSockAddrIn6); + AF_UNIX: result := SizeOf(sockaddr_un); else result := 0; end; end; @@ -1105,6 +1107,11 @@ function SetVarSin(var Sin: TVarSin; const IP,Port: string; begin result := 0; FillChar(Sin,Sizeof(Sin),0); + if (Family=AF_UNIX) then begin + Sin.AddressFamily := AF_UNIX; + Move(IP[1],Sin.sun_path,length(IP)); + exit; + end; Sin.sin_port := Resolveport(port,family,SockProtocol,SockType); TwoPass := false; if Family=AF_UNSPEC then begin From 4d12ff762fe8a272777baea236cae4e12589a9a2 Mon Sep 17 00:00:00 2001 From: "pavel.mash" Date: Mon, 5 Nov 2018 18:07:11 +0200 Subject: [PATCH 2/8] fix Windows build (AF_UNIX is not defined under Win) --- SynCrtSock.pas | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index e4fc2e239..565d5530e 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -4497,8 +4497,10 @@ function CallServer(const Server, Port: SockString; doBind: boolean; else IP := ResolveName(Server,AF_INET,ipproto,socktype); // use AF_INET instead of AF_UNSPEC: IP6 is buggy! + {$ifdef UNIX} if (aLayer=cslUNIX) then family := AF_UNIX else + {$endif} family := AF_INET; if SetVarSin(sin,IP,Port,family,ipproto,socktype,false)<>0 then exit; From 8dc60fce1661261df821eaf2e211fda186f67f88 Mon Sep 17 00:00:00 2001 From: "pavel.mash" Date: Mon, 5 Nov 2018 00:42:06 +0200 Subject: [PATCH 3/8] WIP: initial support for Unix Domain Socket --- SynCrtSock.pas | 79 +++++++++++++++++++++++++++++--------------------- SynFPCSock.pas | 7 +++++ 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index 2ae9fd991..5de03297a 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -431,6 +431,7 @@ TCrtSocket = class // - expects the port to be specified as Ansi string, e.g. '1234' // - you can optionally specify a server address to bind to, e.g. // '1.2.3.4:1234' + // - for unix domain socket use unix:/path/to/file constructor Bind(const aPort: SockString; aLayer: TCrtSocketLayer=cslTCP); /// low-level internal method called by Open() and Bind() constructors // - raise an ECrtSocket exception on error @@ -2000,6 +2001,9 @@ THttpServer = class(THttpServerGeneric) /// the resource address // - e.g. '/category/name/10?param=1' Address: SockString; + /// cslUnix for unix socket URI + // http://unix:/path/to/socket.sock:/url/path + Layer: TCrtSocketLayer; /// fill the members from a supplied URI function From(aURI: SockString; const DefaultPort: SockString=''): boolean; /// compute the whole normalized URI @@ -2469,7 +2473,8 @@ function Open(const aServer, aPort: SockString; aTLS: boolean=false): TCrtSocket /// create a THttpClientSocket, returning nil on error // - useful to easily catch socket error exception ECrtSocket -function OpenHttp(const aServer, aPort: SockString; aTLS: boolean=false): THttpClientSocket; overload; +function OpenHttp(const aServer, aPort: SockString; aTLS: boolean=false; + aLayer: TCrtSocketLayer = cslTCP): THttpClientSocket; overload; /// create a THttpClientSocket, returning nil on error // - useful to easily catch socket error exception ECrtSocket @@ -2479,7 +2484,8 @@ function OpenHttp(const aURI: SockString; aAddress: PSockString=nil): THttpClien // - this method will use a low-level THttpClientSock socket: if you want // something able to use your computer proxy, take a look at TWinINet.Get() function HttpGet(const server, port: SockString; const url: SockString; - const inHeaders: SockString; outHeaders: PSockString=nil): SockString; overload; + const inHeaders: SockString; outHeaders: PSockString=nil; + aLayer: TCrtSocketLayer = cslTCP): SockString; overload; /// retrieve the content of a web page, using the HTTP/1.1 protocol and GET method // - this method will use a low-level THttpClientSock socket for plain http URI, @@ -4310,10 +4316,12 @@ function GetMacAddressesText: SockString; const DEFAULT_PORT: array[boolean] of SockString = ('80','443'); + UNIX_LOW = ord('u')+ord('n')shl 8+ord('i')shl 16+ord('x')shl 24; procedure TURI.Clear; begin Https := false; + Layer := cslTCP; Finalize(self); end; @@ -4335,7 +4343,13 @@ function TURI.From(aURI: SockString; const DefaultPort: SockString): boolean; P := S+3; end; S := P; - while not (S^ in [#0,':','/']) do inc(S); + if (PInteger(S)^ = UNIX_LOW) and (S[4] = ':') then begin //http://unix:...:/path + Inc(S, 5); + Inc(P, 5); + Layer := cslUNIX; + while not (S^ in [#0,':']) do inc(S); + end else + while not (S^ in [#0,':','/']) do inc(S); SetString(Server,P,S-P); if S^=':' then begin inc(S); @@ -4450,9 +4464,9 @@ function CallServer(const Server, Port: SockString; doBind: boolean; aLayer: TCrtSocketLayer; ConnectTimeout: DWORD): TSocket; var sin: TVarSin; IP: SockString; - socktype, ipproto: integer; + socktype, ipproto, family: integer; {$ifndef MSWINDOWS} - serveraddr: TSockAddr; + //serveraddr: sockaddr_un; {$endif} begin result := -1; @@ -4468,36 +4482,25 @@ function CallServer(const Server, Port: SockString; doBind: boolean; cslUNIX: begin {$ifdef MSWINDOWS} exit; // not handled under Win32 - {$else} // special version for UNIX sockets + {$else} socktype := SOCK_STREAM; ipproto := 0; - result := socket(AF_UNIX,socktype,ipproto); - if result<0 then - exit; - if doBind then begin - fillchar(serveraddr,sizeof(serveraddr),0); - //http://publib.boulder.ibm.com/infocenter/iseries/v5r3/index.jsp?topic=/rzab6/rzab6uafunix.htm - {$ifdef KYLIX3} - if (Libc.bind(result,serveraddr,sizeof(serveraddr))<0) or - (Libc.listen(result,SOMAXCONN)<0) then begin - {$else} - if (fpbind(result,@serveraddr,sizeof(serveraddr))<0) or - (fplisten(result,SOMAXCONN)<0) then begin - {$endif} - result := -1; - end; - end; - exit; {$endif} end; else exit; end; if SameText(Server,'localhost') {$ifndef MSWINDOWS}or ((Server='') and not doBind){$endif} then - IP := cLocalHost else + IP := cLocalHost + else if (aLayer=cslUNIX) then + IP := Server + else IP := ResolveName(Server,AF_INET,ipproto,socktype); // use AF_INET instead of AF_UNSPEC: IP6 is buggy! - if SetVarSin(sin,IP,Port,AF_INET,ipproto,socktype,false)<>0 then + if (aLayer=cslUNIX) then + family := AF_UNIX else + family := AF_INET; + if SetVarSin(sin,IP,Port,family,ipproto,socktype,false)<>0 then exit; result := Socket(integer(sin.AddressFamily),socktype,ipproto); if result=-1 then @@ -4637,6 +4640,12 @@ constructor TCrtSocket.Bind(const aPort: SockString; aLayer: TCrtSocketLayer=csl s := '0.0.0.0'; p := aPort; end; + {$ifdef UNIX} + if s = 'unix' then begin + aLayer := cslUNIX; + s := p; p := ''; + end; + {$endif} OpenBind(s,p,true,-1,aLayer); // raise an ECrtSocket exception on error end; @@ -4687,7 +4696,7 @@ procedure TCrtSocket.SetInt32OptionByIndex(OptName, OptVal: integer); procedure TCrtSocket.AcceptRequest(aClientSock: TSocket; aRemoteIP: PSockString); begin CreateSockIn; // use SockIn by default if not already initialized: 2x faster - OpenBind('','',false,aClientSock); // set the ACCEPTed aClientSock + OpenBind('','',false,aClientSock, fSocketLayer); // set the ACCEPTed aClientSock Linger := 5; // should remain open for 5 seconds after a closesocket() call if aRemoteIP<>nil then aRemoteIP^ := GetRemoteIP(aClientSock); @@ -4701,7 +4710,7 @@ procedure TCrtSocket.OpenBind(const aServer, aPort: SockString; begin fSocketLayer := aLayer; if aSock<0 then begin - if aPort='' then + if (aPort='') and (aLayer<>cslUNIX) then fPort := DEFAULT_PORT[aTLS] else // default port is 80/443 (HTTP/S) fPort := aPort; fServer := aServer; @@ -5417,10 +5426,11 @@ function Open(const aServer, aPort: SockString; aTLS: boolean): TCrtSocket; end; end; -function OpenHttp(const aServer, aPort: SockString; aTLS: boolean): THttpClientSocket; +function OpenHttp(const aServer, aPort: SockString; aTLS: boolean; + aLayer: TCrtSocketLayer = cslTCP): THttpClientSocket; begin try - result := THttpClientSocket.Open(aServer,aPort,cslTCP,0,aTLS); // HTTP_DEFAULT_RECEIVETIMEOUT + result := THttpClientSocket.Open(aServer,aPort,aLayer,0,aTLS); // HTTP_DEFAULT_RECEIVETIMEOUT except on ECrtSocket do result := nil; @@ -5432,18 +5442,19 @@ function OpenHttp(const aURI: SockString; aAddress: PSockString): THttpClientSoc begin result := nil; if URI.From(aURI) then begin - result := OpenHttp(URI.Server,URI.Port,URI.Https); + result := OpenHttp(URI.Server,URI.Port,URI.Https,URI.Layer); if aAddress <> nil then aAddress^ := URI.Address; end; end; function HttpGet(const server, port: SockString; const url: SockString; - const inHeaders: SockString; outHeaders: PSockString): SockString; + const inHeaders: SockString; outHeaders: PSockString; + aLayer: TCrtSocketLayer = cslTCP): SockString; var Http: THttpClientSocket; begin result := ''; - Http := OpenHttp(server,port); + Http := OpenHttp(server,port,false,aLayer); if Http<>nil then try if Http.Get(url,0,inHeaders) in [STATUS_SUCCESS..STATUS_PARTIALCONTENT] then begin @@ -5475,7 +5486,7 @@ function HttpGet(const aURI: SockString; const inHeaders: SockString; outHeaders raise ECrtSocket.CreateFmt('https is not supported by HttpGet(%s)',[aURI]) else {$endif} {$endif USEWININET} - result := HttpGet(URI.Server,URI.Port,URI.Address,inHeaders,outHeaders) else + result := HttpGet(URI.Server,URI.Port,URI.Address,inHeaders,outHeaders,URI.Layer) else result := ''; {$ifdef LINUX_RAWDEBUGVOIDHTTPGET} if result='' then @@ -6548,6 +6559,7 @@ constructor THttpServerSocket.Create(aServer: THttpServer); fServer := aServer; fCompress := aServer.fCompress; fCompressAcceptEncoding := aServer.fCompressAcceptEncoding; + fSocketLayer:=aServer.Sock.SocketLayer; TCPPrefix := aServer.TCPPrefix; end; end; @@ -11447,6 +11459,7 @@ procedure TCurlHTTP.InternalCreateRequest(const method, aURL: SockString); var url: SockString; begin url := fRootURL+aURL; + //curl.easy_setopt(fHandle,coTCPNoDelay,0); // disable Nagle curl.easy_setopt(fHandle,coURL,pointer(url)); if fProxyName<>'' then curl.easy_setopt(fHandle,coProxy,pointer(fProxyName)); diff --git a/SynFPCSock.pas b/SynFPCSock.pas index cf75a6f6d..89402a71e 100644 --- a/SynFPCSock.pas +++ b/SynFPCSock.pas @@ -441,6 +441,7 @@ function fpsend(s:cint; msg:pointer; len:size_t; flags:cint): ssize_t; inline; sin6_flowinfo: longword; sin6_addr: TInAddr6; sin6_scope_id: longword); + AF_UNIX: (sun_path: array[0..{$ifdef SOCK_HAS_SINLEN}104{$else}107{$endif}] of Char); ); end; @@ -710,6 +711,7 @@ function SizeOfVarSin(sin: TVarSin): integer; case sin.sin_family of AF_INET: result := SizeOf(TSockAddrIn); AF_INET6: result := SizeOf(TSockAddrIn6); + AF_UNIX: result := SizeOf(sockaddr_un); else result := 0; end; end; @@ -1105,6 +1107,11 @@ function SetVarSin(var Sin: TVarSin; const IP,Port: string; begin result := 0; FillChar(Sin,Sizeof(Sin),0); + if (Family=AF_UNIX) then begin + Sin.AddressFamily := AF_UNIX; + Move(IP[1],Sin.sun_path,length(IP)); + exit; + end; Sin.sin_port := Resolveport(port,family,SockProtocol,SockType); TwoPass := false; if Family=AF_UNSPEC then begin From 76ce1e82b6ac7cb76c8c0628e709d1f17baecbae Mon Sep 17 00:00:00 2001 From: "pavel.mash" Date: Mon, 5 Nov 2018 18:07:11 +0200 Subject: [PATCH 4/8] fix Windows build (AF_UNIX is not defined under Win) --- SynCrtSock.pas | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index 5de03297a..cb0b2952c 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -4497,8 +4497,10 @@ function CallServer(const Server, Port: SockString; doBind: boolean; else IP := ResolveName(Server,AF_INET,ipproto,socktype); // use AF_INET instead of AF_UNSPEC: IP6 is buggy! + {$ifdef UNIX} if (aLayer=cslUNIX) then family := AF_UNIX else + {$endif} family := AF_INET; if SetVarSin(sin,IP,Port,family,ipproto,socktype,false)<>0 then exit; From 6da2a94a0a7f53c6a26e4c627342afca914f321d Mon Sep 17 00:00:00 2001 From: "pavel.mash" Date: Mon, 5 Nov 2018 23:18:39 +0200 Subject: [PATCH 5/8] Lazarus project for 36-RestBenchmark --- .../RESTBenchmark.lpi | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi diff --git a/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi new file mode 100644 index 000000000..8b2f3c05f --- /dev/null +++ b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + <UseAppBundle Value="False"/> + <ResourceType Value="res"/> + </General> + <BuildModes Count="1"> + <Item1 Name="Default" Default="True"/> + </BuildModes> + <PublishOptions> + <Version Value="2"/> + <UseFileFilters Value="True"/> + </PublishOptions> + <RunParams> + <FormatVersion Value="2"/> + <Modes Count="0"/> + </RunParams> + <Units Count="1"> + <Unit0> + <Filename Value="RESTBenchmark.dpr"/> + <IsPartOfProject Value="True"/> + </Unit0> + </Units> + </ProjectOptions> + <CompilerOptions> + <Version Value="11"/> + <Target> + <Filename Value="RESTBenchmark"/> + </Target> + <SearchPaths> + <IncludeFiles Value="../..;../../..;$(ProjOutDir;$(ProjOutDir)"/> + <OtherUnitFiles Value="../..;../../.."/> + <UnitOutputDirectory Value="lib/$(TargetCPU)-$(TargetOS)"/> + </SearchPaths> + </CompilerOptions> + <Debugging> + <Exceptions Count="1"> + <Item1> + <Name Value="Unknown"/> + <Enabled Value="False"/> + </Item1> + </Exceptions> + </Debugging> +</CONFIG> From aaf22a9f5759375c3a213992c7ca2a228861cca6 Mon Sep 17 00:00:00 2001 From: "pavel.mash" <pavel.mash@inbase.com.ua> Date: Sat, 8 Dec 2018 14:32:05 +0200 Subject: [PATCH 6/8] enhance Sample36 by UnixDomainSocket --- .../36 - Simple REST Benchmark/README.md | 64 +++++++++++++++++++ .../RESTBenchmark.dpr | 35 ++++++++-- .../RESTBenchmark.lpi | 1 - .../mormot-rest-nginx.conf | 59 +++++++++++++++++ 4 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 SQLite3/Samples/36 - Simple REST Benchmark/README.md create mode 100644 SQLite3/Samples/36 - Simple REST Benchmark/mormot-rest-nginx.conf diff --git a/SQLite3/Samples/36 - Simple REST Benchmark/README.md b/SQLite3/Samples/36 - Simple REST Benchmark/README.md new file mode 100644 index 000000000..c3e5baba6 --- /dev/null +++ b/SQLite3/Samples/36 - Simple REST Benchmark/README.md @@ -0,0 +1,64 @@ +# Simple mORMot server for REST benchmark + +## Socket based server + + - compile and run RESTBenchmark + - test it with browser: + - http://localhost:8888/root/abc + - http://localhost:8888/root/xyz + - test it with Apache Bench +``` +ab -n 10000 -c 1000 http://localhost:8888/root/abc +``` + +## Keep alive + By default mROMot HTTP server runs in KeepAlive mode. + + To disable KeepAlive run `RESTBenchmark` with secont parameter `false` +``` + ./RESTBenchmark 8888 false +``` + +Disabling KeepAlive make sence in case mORMotserver is behind the reverse proxy. +In this case reverse proxy cares about KeepAlive connection pool and mormot can +operate with fixed thread pool size. + +## Unix Domain Socket (Linux) + +### When to use +In case mORMot is behind a local reverse proxy on the environment with a +huge number of incoming connection it's make sence to use a UDS to minimize +unnecessary TCP handshakes between mORMot and reverse proxy. + +To emulate such environment on the syntetic test we can disable keep alive +in RESTBEnchmark by passing `false` to then second parameter +``` +./RESTBenchmark unix false +./RESTBenchmark 8888 false +``` + +### How to run + + - compile program and run with `unix` parameter +``` +./RESTBenchmark unix +``` + + Benchmark will listen on Unix Domain Socket `/tmp/rest-bench.socket` + + - test it with curl +``` +curl --unix-socket /tmp/rest-bench.socket http://localhost/root/abc +``` + + - setup nginx as a reverse proxy +``` +sudo ln -s "$(pwd)/mormot-rest-nginx.conf" /etc/nginx/sites-available +sudo ln -s /etc/nginx/sites-available/mormot-rest-nginx.conf /etc/nginx/sites-enabled +sudo nginx -s reload +``` + + - test it using ab (or better - wrk) +``` +wrk http://localhost:8888/root/abc +``` diff --git a/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.dpr b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.dpr index ce638701a..2b16e6d2c 100644 --- a/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.dpr +++ b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.dpr @@ -8,13 +8,15 @@ program RESTBenchmark; - ab -n 1000 -c 100 http://localhost:8888/root/xyz for bandwidth measure (returns some ORM query as 77KB of JSON) } - +{$ifndef UNIX} {$APPTYPE CONSOLE} +{$endif} uses {$I SynDprUses.inc} // use FastMM4 on older Delphi, or set FPC threads SysUtils, SynCommons, // framework core + SynCrtSock, // direct access to HTTP server SynLog, // logging features mORMot, // RESTful server & ORM mORMotSQLite3, // SQLite3 engine as ORM core @@ -99,7 +101,7 @@ begin ctxt.Returns(s); end; -procedure DoTest; +procedure DoTest(const url: AnsiString; keepAlive: boolean); var aRestServer: TSQLRestServerDB; aHttpServer: TSQLHttpServer; @@ -114,10 +116,15 @@ begin aServices := TMyServices.Create(aRestServer); try // serve aRestServer data over HTTP - aHttpServer := TSQLHttpServer.Create('8888',[aRestServer]); + aHttpServer := TSQLHttpServer.Create(url,[aRestServer]); + if not keepAlive and (aHttpServer.HttpServer is THttpServer) then + THttpServer(aHttpServer.HttpServer).ServerKeepAliveTimeOut := 0; try aHttpServer.AccessControlAllowOrigin := '*'; // allow cross-site AJAX queries - writeln('Background server is running.'#10); + write('Background server is running on ', url, ' keepAlive '); + if (keepAlive) then + writeLn('is enabled') else + writeLn('is disabled'); write('Press [Enter] to close the server.'); readln; finally @@ -131,12 +138,30 @@ begin end; end; +const + UNIX_SOCK_PATH = '/tmp/rest-bench.socket'; + +var + url: AnsiString; + keepAlive: boolean; begin // set logging abilities SQLite3Log.Family.Level := LOG_VERBOSE; //SQLite3Log.Family.EchoToConsole := LOG_VERBOSE; SQLite3Log.Family.PerThreadLog := ptIdentifiedInOnFile; - DoTest; + SQLite3Log.Family.NoFile := true; // do not create log files for benchmark + {$ifdef UNIX} + if (ParamCount>0) and (ParamStr(1)='unix') then begin + url := 'unix:' + UNIX_SOCK_PATH; + if FileExists(UNIX_SOCK_PATH) then + DeleteFile(UNIX_SOCK_PATH); // remove socket file + end else + {$endif} + url := '8888'; + if (ParamCount>1) and (ParamStr(2)='false') then + keepAlive := false else + keepAlive := true; + DoTest(url, keepAlive); end. diff --git a/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi index 8b2f3c05f..d828b6b41 100644 --- a/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi +++ b/SQLite3/Samples/36 - Simple REST Benchmark/RESTBenchmark.lpi @@ -7,7 +7,6 @@ <MainUnitHasCreateFormStatements Value="False"/> <MainUnitHasTitleStatement Value="False"/> <MainUnitHasScaledStatement Value="False"/> - <UseDefaultCompilerOptions Value="True"/> </Flags> <SessionStorage Value="InProjectDir"/> <MainUnit Value="0"/> diff --git a/SQLite3/Samples/36 - Simple REST Benchmark/mormot-rest-nginx.conf b/SQLite3/Samples/36 - Simple REST Benchmark/mormot-rest-nginx.conf new file mode 100644 index 000000000..a17d3d5b6 --- /dev/null +++ b/SQLite3/Samples/36 - Simple REST Benchmark/mormot-rest-nginx.conf @@ -0,0 +1,59 @@ +upstream mormot-uds { + server unix:/tmp/rest-bench.socket; + keepalive 32; +} + +upstream mormot-sock { + server localhost:8888; + keepalive 32; +} + +server { + listen 8889; + + server_name localhost; + + access_log /dev/null; + # prevent nginx version exposing in Server header + server_tokens off; + + proxy_set_header Host $host; + # Tell upstream real IP address of client + proxy_set_header X-Real-IP $realip_remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Do not rewrite a URL while pass it to upstream + proxy_redirect off; + # Let's upstream handle errors + proxy_intercept_errors on; + tcp_nodelay on; + + # proxy all requests to the beckend + location / { + proxy_pass http://mormot-sock; + } +} + +server { + listen 8887; + + server_name localhost; + + access_log /dev/null; + # prevent nginx version exposing in Server header + server_tokens off; + + proxy_set_header Host $host; + # Tell upstream real IP address of client + proxy_set_header X-Real-IP $realip_remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Do not rewrite a URL while pass it to upstream + proxy_redirect off; + # Let's upstream handle errors + proxy_intercept_errors on; + tcp_nodelay on; + + # proxy all requests to the beckend + location / { + proxy_pass http://mormot-uds; + } +} \ No newline at end of file From 9a6d1e0ab4bb6010828b74da5511b56faf8839c6 Mon Sep 17 00:00:00 2001 From: "pavel.mash" <pavel.mash@inbase.com.ua> Date: Sat, 8 Dec 2018 16:58:14 +0200 Subject: [PATCH 7/8] UnixDomainSocket support for TCurlHTTP --- SynCrtSock.pas | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index cb0b2952c..826b6c680 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -1902,8 +1902,8 @@ THttpServer = class(THttpServerGeneric) // - expects the port to be specified as string, e.g. '1234'; you can // optionally specify a server address to bind to, e.g. '1.2.3.4:1234' // - you can specify a number of threads to be initialized to handle - // incoming connections (default is 32, which may be sufficient for most - // cases, maximum is 256) - if you set 0, the thread pool will be disabled + // incoming connections. Default is 32, which may be sufficient for most + // cases, maximum is 256. If you set 0, the thread pool will be disabled // and one thread will be created for any incoming connection // - you can also tune (or disable with 0) HTTP/1.1 keep alive delay constructor Create(const aPort: SockString; OnStart,OnStop: TNotifyThreadEvent; @@ -2048,6 +2048,7 @@ THttpRequest = class fProxyByPass: SockString; fPort: cardinal; fHttps: boolean; + fLayer: TCrtSocketLayer; fKeepAlive: cardinal; fUserAgent: SockString; fExtendedOptions: THttpRequestExtendedOptions; @@ -2088,7 +2089,8 @@ THttpRequest = class // - *TimeOut parameters are currently ignored by TCurlHttp constructor Create(const aServer, aPort: SockString; aHttps: boolean; const aProxyName: SockString=''; const aProxyByPass: SockString=''; - ConnectionTimeOut: DWORD=0; SendTimeout: DWORD=0; ReceiveTimeout: DWORD=0); overload; virtual; + ConnectionTimeOut: DWORD=0; SendTimeout: DWORD=0; ReceiveTimeout: DWORD=0; + aLayer: TCrtSocketLayer=cslTCP); overload; virtual; /// connect to the supplied URI // - is just a wrapper around TURI and the overloaded Create() constructor constructor Create(const aURI: SockString; @@ -10242,13 +10244,17 @@ function THttpRequest.RegisterCompress(aFunction: THttpSocketCompress; constructor THttpRequest.Create(const aServer, aPort: SockString; aHttps: boolean; const aProxyName,aProxyByPass: SockString; - ConnectionTimeOut,SendTimeout,ReceiveTimeout: DWORD); + ConnectionTimeOut,SendTimeout,ReceiveTimeout: DWORD; + aLayer: TCrtSocketLayer); begin - fPort := GetCardinal(pointer(aPort)); - if fPort=0 then - if aHttps then - fPort := 443 else - fPort := 80; + fLayer := aLayer; + if (fLayer <> cslUNIX) then begin + fPort := GetCardinal(pointer(aPort)); + if fPort=0 then + if aHttps then + fPort := 443 else + fPort := 80; + end; fServer := aServer; fHttps := aHttps; fProxyName := aProxyName; @@ -10271,7 +10277,7 @@ constructor THttpRequest.Create(const aURI, aProxyName,aProxyByPass: SockString; raise ECrtSocket.CreateFmt('%.Create: invalid aURI=%', [ClassName, aURI]); IgnoreSSLCertificateErrors := aIgnoreSSLCertificateErrors; Create(URI.Server,URI.Port,URI.Https,aProxyName,aProxyByPass, - ConnectionTimeOut,SendTimeout,ReceiveTimeout); + ConnectionTimeOut,SendTimeout,ReceiveTimeout,URI.Layer); end; class function THttpRequest.InternalREST(const url,method,data,header: SockString; @@ -10283,7 +10289,7 @@ class function THttpRequest.InternalREST(const url,method,data,header: SockStrin with URI do if From(url) then try - with self.Create(Server,Port,Https,'','') do + with self.Create(Server,Port,Https,'','',0,0,0,Layer) do try IgnoreSSLCertificateErrors := aIgnoreSSLCertificateErrors; Request(Address,method,0,header,data,'',oh,result); @@ -11233,6 +11239,7 @@ function TWinHTTPWebSocketClient.Send(aBufferType: WINHTTP_WEB_SOCKET_BUFFER_TYP coSourceQuote = 10133, coFTPAccount = 10134, coCookieList = 10135, + coUnixSocketPath = 10231, coWriteFunction = 20011, coReadFunction = 20012, coProgressFunction = 20056, @@ -11437,7 +11444,9 @@ procedure TCurlHTTP.InternalConnect(ConnectionTimeOut,SendTimeout,ReceiveTimeout if not IsAvailable then raise ECrtSocket.CreateFmt('No available %s',[LIBCURL_DLL]); fHandle := curl.easy_init; - fRootURL := AnsiString(Format('http%s://%s:%d',[HTTPS[fHttps],fServer,fPort])); + if (fLayer = cslUNIX) then + fRootURL := 'http://localhost' else // see CURLOPT_UNIX_SOCKET_PATH doc + fRootURL := AnsiString(Format('http%s://%s:%d',[HTTPS[fHttps],fServer,fPort])); end; destructor TCurlHTTP.Destroy; @@ -11462,6 +11471,8 @@ procedure TCurlHTTP.InternalCreateRequest(const method, aURL: SockString); begin url := fRootURL+aURL; //curl.easy_setopt(fHandle,coTCPNoDelay,0); // disable Nagle + if fLayer=cslUNIX then + curl.easy_setopt(fHandle,coUnixSocketPath, pointer(fServer)); curl.easy_setopt(fHandle,coURL,pointer(url)); if fProxyName<>'' then curl.easy_setopt(fHandle,coProxy,pointer(fProxyName)); From de4aefac3dcfd40ae0f587e8d190dc65a1ecfc2f Mon Sep 17 00:00:00 2001 From: "pavel.mash" <pavel.mash@inbase.com.ua> Date: Sun, 9 Dec 2018 21:02:30 +0200 Subject: [PATCH 8/8] fix Windows build --- SynCrtSock.pas | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SynCrtSock.pas b/SynCrtSock.pas index 826b6c680..4755fcede 100644 --- a/SynCrtSock.pas +++ b/SynCrtSock.pas @@ -2343,7 +2343,7 @@ TWinHTTPUpgradeable = class(TWinHTTP) constructor Create(const aServer, aPort: SockString; aHttps: boolean; const aProxyName: SockString=''; const aProxyByPass: SockString=''; ConnectionTimeOut: DWORD=0; SendTimeout: DWORD=0; - ReceiveTimeout: DWORD=0); override; + ReceiveTimeout: DWORD=0; aLayer: TCrtSocketLayer=cslTCP); override; end; /// WebSocket client implementation @@ -11023,7 +11023,8 @@ procedure TWinHTTP.InternalSendRequest(const aData: SockString); constructor TWinHTTPUpgradeable.Create(const aServer, aPort: SockString; aHttps: boolean; const aProxyName, aProxyByPass: SockString; - ConnectionTimeOut, SendTimeout, ReceiveTimeout: DWORD); + ConnectionTimeOut, SendTimeout, ReceiveTimeout: DWORD; + aLayer: TCrtSocketLayer=cslTCP); begin inherited; fSocket := nil;