ClickHouse · alexey-milovidov · Apr 27, 2017 · Apr 26, 2017 · Apr 26, 2017 · Apr 26, 2017
diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake
@@ -1,6 +1,6 @@
 #This strings autochanged from release_lib.sh :
-set(VERSION_DESCRIBE v1.1.54229-testing)
-set(VERSION_REVISION 54229)
+set(VERSION_DESCRIBE v1.1.54230-testing)
+set(VERSION_REVISION 54230)
 #===end of autochange
 
 set (VERSION_MAJOR 1)

diff --git a/dbms/src/DataStreams/ODBCDriverBlockOutputStream.h b/dbms/src/DataStreams/ODBCDriverBlockOutputStream.h
@@ -2,7 +2,7 @@
 
 #include <string>
 #include <DataStreams/IBlockOutputStream.h>
-
+#include <Core/Block.h>
 
 namespace DB
 {

diff --git a/dbms/src/IO/MySQLxxHelpers.h b/dbms/src/IO/MySQLxxHelpers.h
@@ -3,12 +3,11 @@
 #include <mysqlxx/Row.h>
 #include <mysqlxx/Null.h>
 #include <mysqlxx/Manip.h>
-
+#include <common/MetrikaTypes.h>
 #include <Core/Field.h>
 #include <Core/FieldVisitors.h>
 #include <IO/WriteHelpers.h>
 
-
 /// This is for Yandex.Metrica code.
 
 namespace mysqlxx

diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h
@@ -80,8 +80,9 @@ class SystemLog : private boost::noncopyable
       */
     void add(const LogElement & element)
     {
-        /// We could lock here in case of queue overflow. Maybe better to throw an exception or even don't do logging in that case.
-        queue.push({false, element});
+        /// Without try we could block here in case of queue overflow.
+        if (!queue.tryPush({false, element}))
+            LOG_ERROR(log, "SystemLog queue is full");
     }
 
 private:
@@ -215,14 +216,18 @@ void SystemLog<LogElement>::flush()
 {
     try
     {
-        LOG_TRACE(log, "Flushing query log");
+        LOG_TRACE(log, "Flushing system log");
 
         if (!is_prepared)    /// BTW, flush method is called from single thread.
             prepareTable();
 
         Block block = LogElement::createBlock();
         for (const LogElement & elem : data)
             elem.appendToBlock(block);
+
+        /// Clear queue early, because insertion to the table could lead to generation of more log entrites
+        ///  and pushing them to already full queue will lead to deadlock.
+        data.clear();
 
         /// We write to table indirectly, using InterpreterInsertQuery.
         /// This is needed to support DEFAULT-columns in table.
@@ -242,10 +247,9 @@ void SystemLog<LogElement>::flush()
     catch (...)
     {
         tryLogCurrentException(__PRETTY_FUNCTION__);
+        /// In case of exception, also clean accumulated data - to avoid locking.
+        data.clear();
     }
-
-    /// In case of exception, also clean accumulated data - to avoid locking.
-    data.clear();
 }
 
 

diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
@@ -443,10 +443,12 @@ int Server::main(const std::vector<std::string> & args)
             listen_hosts.emplace_back(config().getString(key));
         }
 
+        bool try_listen = false;
         if (listen_hosts.empty())
         {
             listen_hosts.emplace_back("::1");
             listen_hosts.emplace_back("127.0.0.1");
+            try_listen = true;
         }
 
         auto make_socket_address = [&](const std::string & host, std::uint16_t port) {
@@ -479,76 +481,88 @@ int Server::main(const std::vector<std::string> & args)
         for (const auto & listen_host : listen_hosts)
         {
             /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file.
-
-            /// HTTP
-            if (config().has("http_port"))
+            try 
             {
-                Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("http_port"));
-                Poco::Net::ServerSocket http_socket(http_socket_address);
-                http_socket.setReceiveTimeout(settings.receive_timeout);
-                http_socket.setSendTimeout(settings.send_timeout);
+                /// HTTP
+                if (config().has("http_port"))
+                {
+                    Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("http_port"));
+                    Poco::Net::ServerSocket http_socket(http_socket_address);
+                    http_socket.setReceiveTimeout(settings.receive_timeout);
+                    http_socket.setSendTimeout(settings.send_timeout);
 
-                servers.emplace_back(new Poco::Net::HTTPServer(
-                    new HTTPRequestHandlerFactory<HTTPHandler>(*this, "HTTPHandler-factory"), server_pool, http_socket, http_params));
+                    servers.emplace_back(new Poco::Net::HTTPServer(
+                        new HTTPRequestHandlerFactory<HTTPHandler>(*this, "HTTPHandler-factory"), server_pool, http_socket, http_params));
 
-                LOG_INFO(log, "Listening http://" + http_socket_address.toString());
-            }
-
-            /// HTTPS
-            if (config().has("https_port"))
-            {
-#if Poco_NetSSL_FOUND
-                std::call_once(ssl_init_once, SSLInit);
-                Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("https_port"));
-                Poco::Net::SecureServerSocket http_socket(http_socket_address);
-                http_socket.setReceiveTimeout(settings.receive_timeout);
-                http_socket.setSendTimeout(settings.send_timeout);
-
-                servers.emplace_back(new Poco::Net::HTTPServer(
-                    new HTTPRequestHandlerFactory<HTTPHandler>(*this, "HTTPHandler-factory"), server_pool, http_socket, http_params));
-
-                LOG_INFO(log, "Listening https://" + http_socket_address.toString());
-#else
-                throw Exception{"https protocol disabled because poco library built without NetSSL support.",
-                    ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-            }
+                    LOG_INFO(log, "Listening http://" + http_socket_address.toString());
+                }
 
-            /// TCP
-            if (config().has("tcp_port"))
-            {
-                Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port"));
-                Poco::Net::ServerSocket tcp_socket(tcp_address);
-                tcp_socket.setReceiveTimeout(settings.receive_timeout);
-                tcp_socket.setSendTimeout(settings.send_timeout);
-                servers.emplace_back(
-                    new Poco::Net::TCPServer(new TCPConnectionFactory(*this), server_pool, tcp_socket, new Poco::Net::TCPServerParams));
-
-                LOG_INFO(log, "Listening tcp: " + tcp_address.toString());
-            }
+                /// HTTPS
+                if (config().has("https_port"))
+                {
+                #if Poco_NetSSL_FOUND
+                    std::call_once(ssl_init_once, SSLInit);
+                    Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("https_port"));
+                    Poco::Net::SecureServerSocket http_socket(http_socket_address);
+                    http_socket.setReceiveTimeout(settings.receive_timeout);
+                    http_socket.setSendTimeout(settings.send_timeout);
+
+                    servers.emplace_back(new Poco::Net::HTTPServer(
+                        new HTTPRequestHandlerFactory<HTTPHandler>(*this, "HTTPHandler-factory"), server_pool, http_socket, http_params));
+
+                    LOG_INFO(log, "Listening https://" + http_socket_address.toString());
+                #else
+                    throw Exception{"https protocol disabled because poco library built without NetSSL support.",
+                        ErrorCodes::SUPPORT_IS_DISABLED};
+                #endif
+                }
 
+                /// TCP
+                if (config().has("tcp_port"))
+                {
+                    Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port"));
+                    Poco::Net::ServerSocket tcp_socket(tcp_address);
+                    tcp_socket.setReceiveTimeout(settings.receive_timeout);
+                    tcp_socket.setSendTimeout(settings.send_timeout);
+                    servers.emplace_back(
+                        new Poco::Net::TCPServer(new TCPConnectionFactory(*this), server_pool, tcp_socket, new Poco::Net::TCPServerParams));
+
+                    LOG_INFO(log, "Listening tcp: " + tcp_address.toString());
+                }
 
-            /// At least one of TCP and HTTP servers must be created.
-            if (servers.empty())
-                throw Exception("No 'tcp_port' and 'http_port' is specified in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
+                /// At least one of TCP and HTTP servers must be created.
+                if (servers.empty())
+                    throw Exception("No 'tcp_port' and 'http_port' is specified in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
 
-            /// Interserver IO HTTP
-            if (config().has("interserver_http_port"))
+                /// Interserver IO HTTP
+                if (config().has("interserver_http_port"))
+                {
+                    Poco::Net::SocketAddress interserver_address = make_socket_address(listen_host, config().getInt("interserver_http_port"));
+                    Poco::Net::ServerSocket interserver_io_http_socket(interserver_address);
+                    interserver_io_http_socket.setReceiveTimeout(settings.receive_timeout);
+                    interserver_io_http_socket.setSendTimeout(settings.send_timeout);
+                    servers.emplace_back(new Poco::Net::HTTPServer(
+                        new HTTPRequestHandlerFactory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
+                        server_pool,
+                        interserver_io_http_socket,
+                        http_params));
+
+                    LOG_INFO(log, "Listening interserver: " + interserver_address.toString());
+                }
+            }
+            catch (const Poco::Net::NetException & e)
             {
-                Poco::Net::SocketAddress interserver_address = make_socket_address(listen_host, config().getInt("interserver_http_port"));
-                Poco::Net::ServerSocket interserver_io_http_socket(interserver_address);
-                interserver_io_http_socket.setReceiveTimeout(settings.receive_timeout);
-                interserver_io_http_socket.setSendTimeout(settings.send_timeout);
-                servers.emplace_back(new Poco::Net::HTTPServer(
-                    new HTTPRequestHandlerFactory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
-                    server_pool,
-                    interserver_io_http_socket,
-                    http_params));
-
-                LOG_INFO(log, "Listening interserver: " + interserver_address.toString());
+                if (try_listen && e.code() == POCO_EPROTONOSUPPORT)
+                    LOG_ERROR(log, "Listen [" << listen_host << "]: " << e.what() << ": " << e.message());
+                else
+                    throw;
             }
+
         }
 
+        if (servers.empty())
+             throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
+
         for (auto & server : servers)
             server->start();
 

diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h
@@ -3,6 +3,7 @@
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Core/Defines.h>
 #include <Core/Types.h>
+#include <Common/Exception.h>
 
 
 namespace DB

diff --git a/libs/libdaemon/CMakeLists.txt b/libs/libdaemon/CMakeLists.txt
@@ -4,6 +4,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
 add_library (daemon
     src/BaseDaemon.cpp
     src/GraphiteWriter.cpp
+    src/OwnPatternFormatter.cpp
 
     include/daemon/BaseDaemon.h
     include/daemon/GraphiteWriter.h

diff --git a/libs/libdaemon/include/daemon/OwnPatternFormatter.h b/libs/libdaemon/include/daemon/OwnPatternFormatter.h
@@ -2,13 +2,6 @@
 
 
 #include <Poco/PatternFormatter.h>
-#include <Poco/Ext/ThreadNumber.h>
-#include <IO/WriteBufferFromString.h>
-#include <IO/WriteHelpers.h>
-#include <daemon/BaseDaemon.h>
-
-#include <experimental/optional>
-#include <functional>
 
 
 /** Форматирует по своему.
@@ -24,6 +17,9 @@
   *
   * Также сделан чуть более эффективным (что имеет мало значения).
   */
+
+class BaseDaemon;
+
 class OwnPatternFormatter : public Poco::PatternFormatter
 {
 public:
@@ -35,47 +31,7 @@ class OwnPatternFormatter : public Poco::PatternFormatter
 
     OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING) : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {}
 
-    void format(const Poco::Message & msg, std::string & text) override
-    {
-        DB::WriteBufferFromString wb(text);
-
-        /// For syslog: tag must be before message and first whitespace.
-        if (options & ADD_LAYER_TAG && daemon)
-        {
-            auto layer = daemon->getLayer();
-            if (layer)
-            {
-                writeCString("layer[", wb);
-                DB::writeIntText(*layer, wb);
-                writeCString("]: ", wb);
-            }
-        }
-
-        /// Output time with microsecond resolution.
-        timeval tv;
-        if (0 != gettimeofday(&tv, nullptr))
-            DB::throwFromErrno("Cannot gettimeofday");
-
-        /// Change delimiters in date for compatibility with old logs.
-        DB::writeDateTimeText<'.', ':'>(tv.tv_sec, wb);
-
-        DB::writeChar('.', wb);
-        DB::writeChar('0' + ((tv.tv_usec / 100000) % 10), wb);
-        DB::writeChar('0' + ((tv.tv_usec / 10000) % 10), wb);
-        DB::writeChar('0' + ((tv.tv_usec / 1000) % 10), wb);
-        DB::writeChar('0' + ((tv.tv_usec / 100) % 10), wb);
-        DB::writeChar('0' + ((tv.tv_usec / 10) % 10), wb);
-        DB::writeChar('0' + ((tv.tv_usec / 1) % 10), wb);
-
-        writeCString(" [ ", wb);
-        DB::writeIntText(Poco::ThreadNumber::get(), wb);
-        writeCString(" ] <", wb);
-        DB::writeString(getPriorityName(static_cast<int>(msg.getPriority())), wb);
-        writeCString("> ", wb);
-        DB::writeString(msg.getSource(), wb);
-        writeCString(": ", wb);
-        DB::writeString(msg.getText(), wb);
-    }
+    void format(const Poco::Message & msg, std::string & text) override;
 
 private:
     const BaseDaemon * daemon;

diff --git a/libs/libdaemon/src/OwnPatternFormatter.cpp b/libs/libdaemon/src/OwnPatternFormatter.cpp
@@ -0,0 +1,52 @@
+#include <daemon/OwnPatternFormatter.h>
+
+#include <functional>
+#include <IO/WriteBufferFromString.h>
+#include <IO/WriteHelpers.h>
+#include <experimental/optional>
+#include <sys/time.h>
+#include <Poco/Ext/ThreadNumber.h>
+#include <daemon/BaseDaemon.h>
+
+
+void OwnPatternFormatter::format(const Poco::Message & msg, std::string & text)
+{
+    DB::WriteBufferFromString wb(text);
+
+    /// For syslog: tag must be before message and first whitespace.
+    if (options & ADD_LAYER_TAG && daemon)
+    {
+        auto layer = daemon->getLayer();
+        if (layer)
+        {
+            writeCString("layer[", wb);
+            DB::writeIntText(*layer, wb);
+            writeCString("]: ", wb);
+        }
+    }
+
+    /// Output time with microsecond resolution.
+    timeval tv;
+    if (0 != gettimeofday(&tv, nullptr))
+        DB::throwFromErrno("Cannot gettimeofday");
+
+    /// Change delimiters in date for compatibility with old logs.
+    DB::writeDateTimeText<'.', ':'>(tv.tv_sec, wb);
+
+    DB::writeChar('.', wb);
+    DB::writeChar('0' + ((tv.tv_usec / 100000) % 10), wb);
+    DB::writeChar('0' + ((tv.tv_usec / 10000) % 10), wb);
+    DB::writeChar('0' + ((tv.tv_usec / 1000) % 10), wb);
+    DB::writeChar('0' + ((tv.tv_usec / 100) % 10), wb);
+    DB::writeChar('0' + ((tv.tv_usec / 10) % 10), wb);
+    DB::writeChar('0' + ((tv.tv_usec / 1) % 10), wb);
+
+    writeCString(" [ ", wb);
+    DB::writeIntText(Poco::ThreadNumber::get(), wb);
+    writeCString(" ] <", wb);
+    DB::writeString(getPriorityName(static_cast<int>(msg.getPriority())), wb);
+    writeCString("> ", wb);
+    DB::writeString(msg.getSource(), wb);
+    writeCString(": ", wb);
+    DB::writeString(msg.getText(), wb);
+}
diff --git a/utils/check_include.sh b/utils/check_include.sh
@@ -3,7 +3,7 @@
 # Finds missing #include <...>
 # prints compile time, number of includes, use with sort: ./check_include.sh 2>&1 | sort -rk3
 pwd=`pwd`
-inc="-I. -I./contrib/libdivide -I./contrib/libre2 -I./build/contrib/libre2 -I./contrib/libfarmhash -I./contrib/libmetrohash/src -I./contrib/libdouble-conversion -I./contrib/libcityhash/include -I./contrib/libzookeeper/include -I./contrib/libtcmalloc/include -I./build/contrib/libzlib-ng -I./contrib/libzlib-ng -I./contrib/libpoco/MongoDB/include -I./contrib/libpoco/XML/include -I./contrib/libpoco/Crypto/include -I./contrib/libpoco/Data/ODBC/include -I./contrib/libpoco/Data/include -I./contrib/libpoco/Net/include -I./contrib/libpoco/Util/include -I./contrib/libpoco/Foundation/include -I./contrib/libboost/boost_1_62_0 -I/usr/include/x86_64-linux-gnu -I./libs/libmysqlxx/include -I./libs/libcommon/include -I./dbms/src -I./build/libs/libcommon/include -I./libs/libpocoext/include -I./libs/libzkutil/include -I./libs/libdaemon/include"
+inc="-I. -I./contrib/libdivide -I./contrib/libre2 -I./build/contrib/libre2 -I./contrib/libfarmhash -I./contrib/libmetrohash/src -I./contrib/libdouble-conversion -I./contrib/libcityhash/include -I./contrib/libzookeeper/include -I./contrib/libtcmalloc/include -I./build/contrib/libzlib-ng -I./contrib/libzlib-ng -I./contrib/libpoco/MongoDB/include -I./contrib/libpoco/XML/include -I./contrib/libpoco/Crypto/include -I./contrib/libpoco/Data/ODBC/include -I./contrib/libpoco/Data/include -I./contrib/libpoco/Net/include -I./contrib/libpoco/Util/include -I./contrib/libpoco/Foundation/include -I./contrib/libboost/boost_1_62_0 -I./libs/libmysqlxx/include -I./libs/libcommon/include -I./build/libs/libcommon/include -I./dbms/src -I./build/dbms/src -I./libs/libpocoext/include -I./libs/libzkutil/include -I./libs/libdaemon/include"
 if [ -z $1 ]; then
     cd ..
     find dbms libs utils -name *.h -exec sh $pwd/$0 {} \; ;

diff --git a/utils/test-data-generator/MarkovModel.h b/utils/test-data-generator/MarkovModel.h
@@ -4,6 +4,7 @@
 #include <Common/HashTable/HashMap.h>
 #include <Common/Arena.h>
 #include <ext/bit_cast.hpp>
+#include <Core/StringRef.h>
 
 
 namespace DB