diff --git a/docs/docs.polserver.com/pol100/corechanges.xml b/docs/docs.polserver.com/pol100/corechanges.xml index 0a397bdb1c..dbfe7fdb88 100644 --- a/docs/docs.polserver.com/pol100/corechanges.xml +++ b/docs/docs.polserver.com/pol100/corechanges.xml @@ -2,9 +2,14 @@
Latest Core Changes - 08-21-2019 + 08-26-2019
+ + 08-26-2019 + Nando: + Improved how the debug, www and aux servers handle connections. They should be faster and more stable now. Minor issues fixed. + 08-21-2019 DevGIB: diff --git a/pol-core/clib/network/sckutil.cpp b/pol-core/clib/network/sckutil.cpp index 66744898a9..ab7c227f03 100644 --- a/pol-core/clib/network/sckutil.cpp +++ b/pol-core/clib/network/sckutil.cpp @@ -14,77 +14,11 @@ namespace Pol { namespace Clib { -bool readline( Socket& sck, std::string& s, bool* timeout_exit, unsigned int timeout_secs, - unsigned maxlen ) -{ - if ( timeout_exit ) - *timeout_exit = false; - s = ""; - unsigned char ch; - unsigned timeouts_left = timeout_secs / 2; - while ( !exit_signalled && sck.connected() ) - { - if ( sck.recvbyte( &ch, 2000 ) ) - { - timeouts_left = timeout_secs / 2; - if ( isprint( ch ) ) - { - s.append( 1, ch ); - if ( maxlen && s.length() > maxlen ) - { - sck.close(); - return false; - } - } - else - { - if ( ch == '\n' ) - return true; - } - } - else - { - if ( timeout_secs && !--timeouts_left ) - { - if ( timeout_exit ) - *timeout_exit = true; - return false; - } - } - } - return false; -} - void writeline( Socket& sck, const std::string& s ) { sck.send( (void*)s.c_str(), static_cast( s.length() ) ); sck.send( "\r\n", 2 ); } -bool readstring( Socket& sck, std::string& s, unsigned int interchar_secs, unsigned length ) -{ - s = ""; - unsigned char ch; - unsigned timeouts_left = interchar_secs / 2; - while ( !exit_signalled && sck.connected() ) - { - if ( sck.recvbyte( &ch, 2000 ) ) - { - timeouts_left = interchar_secs / 2; - - s.append( 1, ch ); - if ( s.length() == length ) - { - return true; - } - } - else - { - if ( !--timeouts_left ) - return false; - } - } - return false; -} } } diff --git a/pol-core/clib/network/sckutil.h b/pol-core/clib/network/sckutil.h index 48fa4b7c3d..ba5805e3aa 100644 --- a/pol-core/clib/network/sckutil.h +++ b/pol-core/clib/network/sckutil.h @@ -14,12 +14,10 @@ namespace Clib { class Socket; -bool readline( Socket& sck, std::string& s, bool* timeout_exit = 0, - unsigned int interchar_timeout_secs = 0, unsigned maxlen = 0 ); - +// TODO: move this function into the Socket class +// TODO: get rid of http_writeline, it's doing exactly the same as this function void writeline( Socket& sck, const std::string& s ); -bool readstring( Socket& sck, std::string& s, unsigned int interchar_secs, unsigned length ); } } #endif diff --git a/pol-core/clib/network/singlepollers/pollingwithpoll.h b/pol-core/clib/network/singlepollers/pollingwithpoll.h index 93023a33f7..0c613f9614 100644 --- a/pol-core/clib/network/singlepollers/pollingwithpoll.h +++ b/pol-core/clib/network/singlepollers/pollingwithpoll.h @@ -8,7 +8,7 @@ #ifdef _WIN32 // compatibility wrapper for windows -int poll( struct pollfd* fds, ULONG nfds, int timeout ) +inline int poll( struct pollfd* fds, ULONG nfds, int timeout ) { return WSAPoll( fds, nfds, timeout ); }; diff --git a/pol-core/clib/network/socketsvc.cpp b/pol-core/clib/network/socketsvc.cpp index 31e9942e97..b6a2883fe7 100644 --- a/pol-core/clib/network/socketsvc.cpp +++ b/pol-core/clib/network/socketsvc.cpp @@ -36,9 +36,9 @@ SocketListener::SocketListener( unsigned short port, Socket::option opt ) : _lis } bool SocketListener::GetConnection( Socket* newsck, unsigned int timeout_sec, - unsigned int timeout_usec ) + unsigned int timeout_msec ) { - if ( _listen_sck.select( timeout_sec, timeout_usec ) ) + if ( _listen_sck.has_incoming_data( timeout_sec * 1000 + timeout_msec ) ) return _listen_sck.accept( newsck ); return false; } diff --git a/pol-core/clib/network/socketsvc.h b/pol-core/clib/network/socketsvc.h index 2278f43ff5..50e43eee43 100644 --- a/pol-core/clib/network/socketsvc.h +++ b/pol-core/clib/network/socketsvc.h @@ -18,7 +18,7 @@ class SocketListener public: explicit SocketListener( unsigned short port ); SocketListener( unsigned short port, Socket::option opt ); - bool GetConnection( Socket* newsck, unsigned int timeout_sec, unsigned int timeout_usec = 0 ); + bool GetConnection( Socket* newsck, unsigned int timeout_sec, unsigned int timeout_msec = 0 ); friend class SocketClientThread; diff --git a/pol-core/clib/network/wnsckt.cpp b/pol-core/clib/network/wnsckt.cpp index 10c5bfb27f..1350b0ea0c 100644 --- a/pol-core/clib/network/wnsckt.cpp +++ b/pol-core/clib/network/wnsckt.cpp @@ -6,6 +6,7 @@ #include "wnsckt.h" +#include #include #include @@ -14,6 +15,8 @@ #include "passert.h" #include "strutil.h" +#include "singlepoller.h" + #if defined( WINDOWS ) #define SOCKET_ERRNO( x ) WSA##x #define socket_errno WSAGetLastError() @@ -103,7 +106,7 @@ std::string Socket::getpeername() const { struct sockaddr client_addr; // inet_addr socklen_t addrlen = sizeof client_addr; - if ( ::getpeername( _sck, &client_addr, &addrlen ) == 0 ) + if (::getpeername( _sck, &client_addr, &addrlen ) == 0 ) { struct sockaddr_in* in_addr = (struct sockaddr_in*)&client_addr; if ( client_addr.sa_family == AF_INET ) @@ -249,7 +252,7 @@ bool Socket::listen( unsigned short port ) HandleError(); return false; } - if ( ::listen( _sck, SOMAXCONN ) == -1 ) + if (::listen( _sck, SOMAXCONN ) == -1 ) { HandleError(); return false; @@ -257,29 +260,28 @@ bool Socket::listen( unsigned short port ) return true; } -bool Socket::select( unsigned int seconds, unsigned int useconds ) +bool Socket::has_incoming_data( unsigned int waitms, int* result ) { - fd_set fd; - struct timeval timeout = {0, 0}; - int nfds = 0; - FD_ZERO( &fd ); - FD_SET( _sck, &fd ); -#ifndef _WIN32 - passert_r( _sck < FD_SETSIZE, - "Select() implementation in Linux cant handle this many sockets at the same time." ) - nfds = _sck + 1; -#endif + SinglePoller poller( _sck ); + poller.set_timeout( 0, waitms * 1000 ); - int res; + if ( !poller.prepare( false ) ) + { + if ( result ) + *result = -1; + throw std::runtime_error( "Unable to poll socket=" + tostring( _sck ) ); + } + int res = -1; do { - timeout.tv_sec = seconds; - timeout.tv_usec = useconds; - res = ::select( nfds, &fd, nullptr, nullptr, &timeout ); + res = poller.wait_for_events(); } while ( res < 0 && !exit_signalled && socket_errno == SOCKET_ERRNO( EINTR ) ); - return ( res > 0 && FD_ISSET( _sck, &fd ) ); + if ( result ) + *result = res; + + return poller.incoming(); } bool Socket::accept( SOCKET* s, unsigned int /*mstimeout*/ ) @@ -360,42 +362,31 @@ void Socket::HandleError() bool Socket::recvbyte( unsigned char* ch, unsigned int waitms ) { - fd_set fd; - if ( !connected() ) return false; #if SCK_WATCH INFO_PRINT << "{L;1}\n"; #endif - FD_ZERO( &fd ); - FD_SET( _sck, &fd ); - int nfds = 0; -#ifndef _WIN32 - nfds = _sck + 1; -#endif - struct timeval tv; - int res; - do - { - tv.tv_sec = 0; - tv.tv_usec = waitms * 1000; - res = ::select( nfds, &fd, nullptr, nullptr, &tv ); - } while ( res < 0 && exit_signalled && socket_errno == SOCKET_ERRNO( EINTR ) ); - - if ( res == 0 ) + int res; + if ( !has_incoming_data( waitms, &res ) ) { + if ( res == -1 ) + { + HandleError(); + close(); + } + else if ( res == 0 ) + { #if SCK_WATCH - INFO_PRINT << "{TO}\n"; + INFO_PRINT << "{TO}\n"; #endif + } + return false; } - else if ( res == -1 ) - { - HandleError(); - close(); /* FIXME: very likely unrecoverable */ - } + res = recv( _sck, (char*)ch, 1, 0 ); if ( res == 1 ) @@ -421,11 +412,46 @@ bool Socket::recvbyte( unsigned char* ch, unsigned int waitms ) } } +bool Socket::recvdata_nowait( char* pdest, unsigned len, int* bytes_read ) +{ + if ( bytes_read ) + *bytes_read = -1; + + if ( !connected() ) + return false; + +#if SCK_WATCH + INFO_PRINT << "{L:" << len << "}\n"; +#endif + + int res; + res = ::recv( _sck, pdest, len, 0 ); + + if ( bytes_read ) + *bytes_read = res; + + if ( res < 0 ) + { + /* Can't time out here this is an ERROR! */ + HandleError(); + return false; + } + + if ( res == 0 ) + { +#if SCK_WATCH + INFO_PRINT << "{CLOSE}\n"; +#endif + close(); + return false; + } + + return true; +} + bool Socket::recvdata( void* vdest, unsigned len, unsigned int waitms ) { - fd_set fd; char* pdest = (char*)vdest; - int nfds = 0; while ( len ) { @@ -435,33 +461,23 @@ bool Socket::recvdata( void* vdest, unsigned len, unsigned int waitms ) #if SCK_WATCH INFO_PRINT << "{L:" << len << "}\n"; #endif - FD_ZERO( &fd ); - FD_SET( _sck, &fd ); -#ifndef _WIN32 - nfds = _sck + 1; -#endif - - struct timeval tv; int res; - do - { - tv.tv_sec = 0; - tv.tv_usec = waitms * 1000; - res = ::select( nfds, &fd, nullptr, nullptr, &tv ); - } while ( res < 0 && exit_signalled && socket_errno == SOCKET_ERRNO( EINTR ) ); - - if ( res == 0 ) + if ( !has_incoming_data( waitms, &res ) ) { + if ( res == -1 ) + { + HandleError(); + close(); + } + else if ( res == 0 ) + { #if SCK_WATCH - INFO_PRINT << "{TO}\n"; + INFO_PRINT << "{TO}\n"; #endif + } + return false; } - else if ( res == -1 ) - { - HandleError(); - close(); /* FIXME: very likely unrecoverable */ - } res = ::recv( _sck, pdest, len, 0 ); @@ -499,39 +515,28 @@ bool Socket::recvdata( void* vdest, unsigned len, unsigned int waitms ) unsigned Socket::peek( void* vdest, unsigned len, unsigned int wait_sec ) { - fd_set fd; + ; char* pdest = (char*)vdest; - int nfds = 0; #if SCK_WATCH INFO_PRINT << "{L:" << len << "}\n"; #endif - FD_ZERO( &fd ); - FD_SET( _sck, &fd ); -#ifndef _WIN32 - nfds = _sck + 1; -#endif - struct timeval tv; - int res; - - do - { - tv.tv_sec = wait_sec; - tv.tv_usec = 0; - res = ::select( nfds, &fd, nullptr, nullptr, &tv ); - } while ( res < 0 && exit_signalled && socket_errno == SOCKET_ERRNO( EINTR ) ); - if ( res == 0 ) + int res; + if ( !has_incoming_data( 1000 * wait_sec, &res ) ) { + if ( res == -1 ) + { + HandleError(); + close(); + } + else if ( res == 0 ) + { #if SCK_WATCH - INFO_PRINT << "{TO}\n"; + INFO_PRINT << "{TO}\n"; #endif - return 0; - } - else if ( res == -1 ) - { - HandleError(); - close(); /* FIXME: very likely unrecoverable */ + } + return 0; } @@ -647,5 +652,128 @@ bool Socket::is_local() const std::string s = getpeername(); return ( s == "127.0.0.1" ); } + +bool is_invalid_readline_char( unsigned char c ) +{ + return !isprint( c ) && c != '\n' && c != '\r'; +} + +bool SocketLineReader::try_readline( std::string& out, bool* timed_out ) +{ + if ( timed_out ) + *timed_out = false; + + // check if there is already a line in the buffer + auto pos_newline = _currentLine.find_first_of( "\r\n" ); + + // If not, try to read more data + if ( pos_newline == std::string::npos ) + { + std::array buffer; + + int res = -1; + if ( !_socket.has_incoming_data( _waitms, &res ) ) + { + if ( timed_out ) + *timed_out = true; + return false; + } + int bytes_read = -1; + if ( !_socket.recvdata_nowait( buffer.data(), buffer.size(), &bytes_read ) ) + return false; + + // store current line size so we don't need to search from the beginning again + size_t oldSize = _currentLine.size(); + + // remove invalid characters from buffer + auto buffer_end = + std::remove_if( buffer.begin(), buffer.begin() + bytes_read, is_invalid_readline_char ); + + auto valid_char_count = std::distance( buffer.begin(), buffer_end ); + + // nothing gained from these bytes + if ( !valid_char_count ) + return false; + + // append only the valid characters to the buffer + _currentLine.append( buffer.data(), valid_char_count ); + + // update position + pos_newline = _currentLine.find_first_of( "\r\n", oldSize ); + } + + // note that std::string::npos is larger than any other number, so the conditon below will be + // false only if there is a newline before the maximum line length or if the current line is still + // small. + if ( _maxLinelength > 0 && pos_newline > _maxLinelength && _currentLine.size() > _maxLinelength ) + { + out = _currentLine; + _currentLine.clear(); + return false; + } + + // Haven't found it yet + if ( pos_newline == std::string::npos ) + return false; + + auto end_newline = pos_newline + 1; + if ( _currentLine[pos_newline] == '\r' ) + end_newline++; + + out = _currentLine.substr( 0, pos_newline ); + _currentLine.erase( 0, end_newline ); + + return true; +} + +// Blocks until a whole line is received, waitms are over or maxlen is reached +bool SocketLineReader::readline( std::string& out, bool* timed_out ) +{ + out = ""; + if ( timed_out ) + *timed_out = false; + + const int max_timeouts = ( _timeout_secs * 1000 ) / _waitms; + bool single_timed_out = false; + + int timeout_left = max_timeouts; + while ( !Clib::exit_signalled && _socket.connected() ) + { + if ( try_readline( out, &single_timed_out ) ) + { + return true; + } + + // if try_readline() is false, string "out" should be empty unless the maxlen was reached. + if ( !out.empty() ) + { + _socket.close(); + return false; + } + + if ( _timeout_secs > 0 && single_timed_out ) + { + timeout_left--; + if ( timeout_left <= 0 ) + { + if ( _disconnect_on_timeout || !timed_out ) + _socket.close(); + + if ( timed_out ) + *timed_out = true; + + return false; + } + } + else + { + // refresh timeout counter + timeout_left = max_timeouts; + } + } + return false; +} + + } // namespace Clib } // namespace Pol diff --git a/pol-core/clib/network/wnsckt.h b/pol-core/clib/network/wnsckt.h index 2319d8222e..fd6b78fcd8 100644 --- a/pol-core/clib/network/wnsckt.h +++ b/pol-core/clib/network/wnsckt.h @@ -39,10 +39,11 @@ class Socket bool open( const char* ipaddr, unsigned short port ); bool listen( unsigned short port ); - bool select( unsigned int seconds, unsigned int useconds ); + bool has_incoming_data( unsigned int waitms, int* result = nullptr ); bool accept( SOCKET* s, unsigned int mstimeout ); bool accept( Socket* newsocket ); bool recvbyte( unsigned char* byte, unsigned int waitms ); + bool recvdata_nowait( char* vdest, unsigned len, int* bytes_read ); bool recvdata( void* vdest, unsigned len, unsigned int waitms ); unsigned peek( void* vdest, unsigned len, unsigned int waitms ); void send( const void* data, unsigned length ); @@ -78,6 +79,39 @@ class Socket int _options; struct sockaddr _peer; }; + +class SocketLineReader +{ +public: + SocketLineReader( Socket& socket, unsigned int timeout_secs = 0, unsigned int max_linelength = 0, + bool disconnect_on_timeout = true ) + : _socket( socket ), + _waitms( 500 ), + _timeout_secs( timeout_secs ), + _maxLinelength( max_linelength ), + _disconnect_on_timeout( disconnect_on_timeout ) + { + } + bool try_readline( std::string& out, bool* timed_out = nullptr ); + bool readline( std::string& out, bool* timed_out = nullptr ); + + void set_max_linelength( unsigned int max_linelength ) { _maxLinelength = max_linelength; } + void set_wait( unsigned int waitms ) { _waitms = waitms; } + void set_timeout( unsigned int timeout_secs ) { _timeout_secs = timeout_secs; } + + void set_disconnect_on_timeout( bool disconnect ) { _disconnect_on_timeout = disconnect; } + +private: + Socket& _socket; + std::string _currentLine; + + unsigned int _waitms; + unsigned int _timeout_secs; + unsigned int _maxLinelength; + + bool _disconnect_on_timeout; +}; + } // namespace Clib } // namespace Pol #endif // CLIB_WNSCKT_H diff --git a/pol-core/doc/core-changes.txt b/pol-core/doc/core-changes.txt index bee36a57e5..3b449ffe34 100644 --- a/pol-core/doc/core-changes.txt +++ b/pol-core/doc/core-changes.txt @@ -1,4 +1,6 @@ -- POL100 -- +08-26-2019 Nando: + Changed: Improved how the debug, www and aux servers handle connections. They should be faster and more stable now. Minor issues fixed. 08-21-2019 DevGIB: Fixed: Missing default on GetProcess( pid ), this is now GetProcess( pid:= 0 ) to align with docs and functionallity. Please update your os.em file and recompile. 07-12-2019 DevGIB: diff --git a/pol-core/pol/network/auxclient.cpp b/pol-core/pol/network/auxclient.cpp index 204034bba1..80f4026e10 100644 --- a/pol-core/pol/network/auxclient.cpp +++ b/pol-core/pol/network/auxclient.cpp @@ -182,9 +182,10 @@ void AuxClientThread::run() std::string tmp; bool result, timeout_exit; + Clib::SocketLineReader linereader(_sck, 5); for ( ;; ) { - result = readline( _sck, tmp, &timeout_exit, 5 ); + result = linereader.readline( tmp, &timeout_exit ); if ( !result && !timeout_exit ) break; diff --git a/pol-core/pol/poldbg.cpp b/pol-core/pol/poldbg.cpp index 413de327a5..175680cfa7 100644 --- a/pol-core/pol/poldbg.cpp +++ b/pol-core/pol/poldbg.cpp @@ -1358,10 +1358,13 @@ void DebugClientThread::run() DebugContext dctx; std::string cmdline; std::vector results; + + Clib::SocketLineReader linereader(_sck); + while ( !dctx.done() ) { Clib::writeline( _sck, dctx.prompt() ); - if ( !readline( _sck, cmdline ) ) + if ( !linereader.readline( cmdline ) ) break; bool ret = dctx.process( cmdline, results ); diff --git a/pol-core/pol/polwww.cpp b/pol-core/pol/polwww.cpp index fed21450f1..ad38a8215e 100644 --- a/pol-core/pol/polwww.cpp +++ b/pol-core/pol/polwww.cpp @@ -33,8 +33,11 @@ #include "../clib/stlutil.h" #include "../clib/strutil.h" #include "../clib/threadhelp.h" +#include "../clib/timer.h" + #include "../plib/pkg.h" #include "../plib/systemstate.h" + #include "globals/uvars.h" #include "module/httpmod.h" #include "module/uomod.h" @@ -126,41 +129,10 @@ void config_web_server() load_mime_config(); } -// TODO: The http server is susceptible to DOS attacks -// TODO: limit access to localhost by default, probably - -bool http_readline( Clib::Socket& sck, std::string& s ) -{ - bool res = false; - s = ""; - unsigned char ch; - while ( sck.connected() && sck.recvbyte( &ch, 10000 ) ) - { - if ( isprint( ch ) ) - { - s.append( 1, ch ); - if ( s.length() > 3000 ) - { - sck.close(); - break; // return false; - } - } - else - { - if ( ch == '\n' ) - { - res = true; - break; - // return true; - } - } - } - return res; -} void http_writeline( Clib::Socket& sck, const std::string& s ) { sck.send( (void*)s.c_str(), static_cast( s.length() ) ); - sck.send( "\n", 1 ); + sck.send( "\r\n", 2 ); } void http_forbidden( Clib::Socket& sck ) @@ -197,6 +169,17 @@ void http_not_authorized( Clib::Socket& sck, const std::string& /*filename*/ ) http_writeline( sck, "" ); } +void http_internal_error( Clib::Socket& sck, const std::string& filename ) +{ + http_writeline( sck, "HTTP/1.1 500 Internal Sever Error" ); + http_writeline( sck, "Content-Type: text/html" ); + http_writeline( sck, "" ); + http_writeline( sck, "500 Internal Server Error" ); + http_writeline( sck, "

Internal Server Error

" ); + http_writeline( sck, "The requested URL " + filename + " caused an internal server error." ); + http_writeline( sck, "" ); +} + void http_not_found( Clib::Socket& sck, const std::string& filename ) { http_writeline( sck, "HTTP/1.1 404 Not Found" ); @@ -618,21 +601,23 @@ void send_binary( Clib::Socket& sck, const std::string& page, const std::string& void http_func( SOCKET client_socket ) { Clib::Socket sck( client_socket ); + Clib::SocketLineReader lineReader( sck, 5, 3000, + false ); // we take care of disconnecting at timeout + std::string get; std::string auth; std::string tmpstr; std::string host; - if ( Plib::systemstate.config.web_server_local_only ) + if ( Plib::systemstate.config.web_server_local_only && !sck.is_local() ) { - if ( !sck.is_local() ) - { - http_forbidden( sck ); - return; - } + http_forbidden( sck ); + return; } - while ( sck.connected() && http_readline( sck, tmpstr ) ) + bool timed_out = false; + Tools::HighPerfTimer requestTimer; + while ( sck.connected() && lineReader.readline( tmpstr, &timed_out ) ) { if ( Plib::systemstate.config.web_server_debug ) INFO_PRINT << "http(" << sck.handle() << "): '" << tmpstr << "'\n"; @@ -645,9 +630,22 @@ void http_func( SOCKET client_socket ) if ( strncmp( tmpstr.c_str(), "Host: ", 5 ) == 0 ) host = tmpstr.substr( 6 ); } + + if ( timed_out ) + { + INFO_PRINT << "HTTP connection " << sck.getpeername() << " timed out\n"; + sck.close(); + } + if ( !sck.connected() ) return; + if ( Plib::systemstate.config.web_server_debug ) + { + INFO_PRINT << "[" << double( requestTimer.ellapsed().count() / 1000.0 ) + << " msec] finished reading header\n"; + } + ISTRINGSTREAM is( get ); std::string cmd; // GET, POST (we only handle GET) @@ -761,6 +759,7 @@ void http_func( SOCKET client_socket ) else { POLLOG_INFO << "HTTP server: I can't handle pagetype '" << pagetype << "'\n"; + http_internal_error( sck, page ); } } } diff --git a/pol-core/pol/uolisten.cpp b/pol-core/pol/uolisten.cpp index 5bb7f3a04a..a732582547 100644 --- a/pol-core/pol/uolisten.cpp +++ b/pol-core/pol/uolisten.cpp @@ -107,14 +107,14 @@ void uo_client_listener_thread( void* arg ) while ( !Clib::exit_signalled ) { unsigned int timeout = 2; - unsigned int utimeout = 0; + unsigned int mstimeout = 0; if ( !ls->login_clients.empty() ) { timeout = 0; - utimeout = 200000; + mstimeout = 200; } Clib::Socket newsck; - if ( SL.GetConnection( &newsck, timeout, utimeout ) && newsck.connected() ) + if ( SL.GetConnection( &newsck, timeout, mstimeout ) && newsck.connected() ) { // create an appropriate Client object if ( Plib::systemstate.config.use_single_thread_login )