From d28a97a8d9cbb3545c5150837e8a6cab643effa4 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Wed, 18 Jun 2014 10:18:22 -0700 Subject: [PATCH 1/8] DRILL-998: Limit memory usage of the C++ Client --- .../native/client/example/querySubmitter.cpp | 1 + .../client/src/clientlib/CMakeLists.txt | 1 + .../client/src/clientlib/drillClient.cpp | 11 +-- .../client/src/clientlib/drillClientImpl.cpp | 50 ++++++++------ .../client/src/clientlib/drillClientImpl.hpp | 12 ++-- .../client/src/clientlib/recordBatch.cpp | 2 +- contrib/native/client/src/clientlib/utils.cpp | 68 +++++++++++++++++++ contrib/native/client/src/clientlib/utils.hpp | 31 ++++++--- .../client/src/include/drill/common.hpp | 6 ++ .../client/src/include/drill/recordBatch.hpp | 8 +-- 10 files changed, 147 insertions(+), 43 deletions(-) create mode 100644 contrib/native/client/src/clientlib/utils.cpp diff --git a/contrib/native/client/example/querySubmitter.cpp b/contrib/native/client/example/querySubmitter.cpp index 8e806580ba2..17bec3c75f0 100644 --- a/contrib/native/client/example/querySubmitter.cpp +++ b/contrib/native/client/example/querySubmitter.cpp @@ -280,6 +280,7 @@ int main(int argc, char* argv[]) { //DrillClient::initLogging("/var/log/drill/", l); // To log to stderr Drill::DrillClient::initLogging(NULL, l); + Drill::DrillClientConfig::setBufferLimit(2*1024*1024); // 2MB. Allows us to hold at least two record batches. if(client.connect(connectStr.c_str(), schema.c_str())!=Drill::CONN_SUCCESS){ std::cerr<< "Failed to connect with error: "<< client.getError() << " (Using:"<m_pQueryResult->waitForData(); - if(m_pQueryResult->hasError()){ - return m_pQueryResult->getErrorStatus(); - } this->m_currentRecord++; if(!this->m_pQueryResult->isCancelled()){ @@ -169,8 +165,13 @@ status_t RecordIterator::next(){ if(this->m_pCurrentRecordBatch !=NULL){ DRILL_LOG(LOG_TRACE) << "Deleted old Record batch " << (void*) m_pCurrentRecordBatch << std::endl; delete this->m_pCurrentRecordBatch; //free the previous record batch + this->m_pCurrentRecordBatch=NULL; } this->m_currentRecord=0; + this->m_pQueryResult->waitForData(); + if(m_pQueryResult->hasError()){ + return m_pQueryResult->getErrorStatus(); + } this->m_pCurrentRecordBatch=this->m_pQueryResult->getNext(); if(this->m_pCurrentRecordBatch != NULL){ DRILL_LOG(LOG_TRACE) << "Fetched new Record batch " << std::endl; diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp index 54dcdd0f1dc..f2627b0681f 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.cpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp @@ -175,7 +175,7 @@ connectionStatus_t DrillClientImpl::recvHandshake(){ DRILL_LOG(LOG_DEBUG) << "Sent handshake read request to server" << std::endl; m_io_service.run(); if(m_rbuf!=NULL){ - Utils::freeBuffer(m_rbuf); m_rbuf=NULL; + Utils::freeBuffer(m_rbuf, MAX_SOCK_RD_BUFSIZE); m_rbuf=NULL; } return CONN_SUCCESS; } @@ -332,6 +332,13 @@ void DrillClientImpl::getNextResult(){ // This call is always made from within a function where the mutex has already been acquired //boost::lock_guard lock(this->m_dcMutex); + { + boost::unique_lock memLock(AllocatedBuffer::s_memCVMutex); + DRILL_LOG(LOG_TRACE) << "Read blocked waiting for memory." << std::endl; + while(AllocatedBuffer::s_isBufferLimitReached){ + AllocatedBuffer::s_memCV.wait(memLock); + } + } //use free, not delete to free ByteBuf_t readBuf = Utils::allocateBuffer(LEN_PREFIX_BUFLEN); @@ -362,10 +369,13 @@ void DrillClientImpl::waitForResults(){ delete this->m_pListenerThread; this->m_pListenerThread=NULL; } -status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, InBoundRpcMessage& msg, boost::system::error_code& error){ +status_t DrillClientImpl::readMsg(ByteBuf_t _buf, + AllocatedBufferPtr* allocatedBuffer, + InBoundRpcMessage& msg, + boost::system::error_code& error){ size_t leftover=0; uint32_t rmsgLen; - ByteBuf_t currentBuffer; + AllocatedBufferPtr currentBuffer; *allocatedBuffer=NULL; { // We need to protect the readLength and read buffer, and the pending requests counter, @@ -379,18 +389,18 @@ status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, In leftover = LEN_PREFIX_BUFLEN - bytes_read; // Allocate a buffer DRILL_LOG(LOG_TRACE) << "Allocated and locked buffer." << std::endl; - currentBuffer=Utils::allocateBuffer(rmsgLen); + currentBuffer=new AllocatedBuffer(rmsgLen); if(currentBuffer==NULL){ - Utils::freeBuffer(_buf); + Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN); return handleQryError(QRY_CLIENT_OUTOFMEM, getMessage(ERR_QRY_OUTOFMEM), NULL); } *allocatedBuffer=currentBuffer; if(leftover){ - memcpy(currentBuffer, _buf + bytes_read, leftover); + memcpy(currentBuffer->m_pBuffer, _buf + bytes_read, leftover); } DRILL_LOG(LOG_TRACE) << "reading data (rmsgLen - leftover) : " << (rmsgLen - leftover) << std::endl; - ByteBuf_t b=currentBuffer + leftover; + ByteBuf_t b=currentBuffer->m_pBuffer + leftover; size_t bytesToRead=rmsgLen - leftover; while(1){ size_t dataBytesRead=this->m_socket.read_some( @@ -404,24 +414,24 @@ status_t DrillClientImpl::readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, In } if(!error){ // read data successfully - DrillClientImpl::s_decoder.Decode(currentBuffer, rmsgLen, msg); + DrillClientImpl::s_decoder.Decode(currentBuffer->m_pBuffer, rmsgLen, msg); DRILL_LOG(LOG_TRACE) << "Done decoding chunk. Coordination id: " <query_state() == exec::shared::QueryResult_QueryState_FAILED){ status_t ret=handleQryError(QRY_FAILURE, qr->error(0), pDrillClientQueryResult); - Utils::freeBuffer(allocatedBuffer); + delete allocatedBuffer; delete qr; return ret; } //Validate the RPC message std::string valErr; if( (ret=validateMessage(msg, *qr, valErr)) != QRY_SUCCESS){ - Utils::freeBuffer(allocatedBuffer); + delete allocatedBuffer; delete qr; return handleQryError(ret, getMessage(ERR_QRY_INVRPC, valErr.c_str()), pDrillClientQueryResult); } @@ -521,7 +531,7 @@ status_t DrillClientImpl::processQueryResult(ByteBuf_t allocatedBuffer, InBoundR return ret; } -status_t DrillClientImpl::processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg ){ +status_t DrillClientImpl::processQueryId(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg ){ DrillClientQueryResult* pDrillClientQueryResult=NULL; DRILL_LOG(LOG_DEBUG) << "Processing Query Handle with coordination id:" << msg.m_coord_id << std::endl; status_t ret=QRY_SUCCESS; @@ -539,10 +549,10 @@ status_t DrillClientImpl::processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMe //save queryId allocated here so we can free it later pDrillClientQueryResult->setQueryId(qid); }else{ - Utils::freeBuffer(allocatedBuffer); + delete allocatedBuffer; return handleQryError(QRY_INTERNAL_ERROR, getMessage(ERR_QRY_INVQUERYID), NULL); } - Utils::freeBuffer(allocatedBuffer); + delete allocatedBuffer; return ret; } @@ -580,7 +590,7 @@ void DrillClientImpl::handleRead(ByteBuf_t _buf, InBoundRpcMessage msg; DRILL_LOG(LOG_TRACE) << "Getting new message" << std::endl; - ByteBuf_t allocatedBuffer=NULL; + AllocatedBufferPtr allocatedBuffer=NULL; if(readMsg(_buf, &allocatedBuffer, msg, error)!=QRY_SUCCESS){ if(m_pendingRequests!=0){ @@ -628,7 +638,7 @@ void DrillClientImpl::handleRead(ByteBuf_t _buf, } }else{ // boost error - Utils::freeBuffer(_buf); + Utils::freeBuffer(_buf, LEN_PREFIX_BUFLEN); boost::lock_guard lock(this->m_dcMutex); handleQryError(QRY_COMM_ERROR, getMessage(ERR_QRY_COMMERR, error.message().c_str()), NULL); return; @@ -828,7 +838,7 @@ status_t DrillClientQueryResult::defaultQueryResultsListener(void* ctx, return QRY_SUCCESS; } -RecordBatch* DrillClientQueryResult::peekNext() { +RecordBatch* DrillClientQueryResult::peekNext(){ RecordBatch* pRecordBatch=NULL; //if no more data, return NULL; if(!m_bIsQueryPending) return NULL; diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp index e40b2147cc7..42abbfa95cb 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.hpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp @@ -198,7 +198,7 @@ class DrillClientImpl{ m_socket.shutdown(boost::asio::ip::tcp::socket::shutdown_send, ignorederr); m_socket.close(); if(m_rbuf!=NULL){ - Utils::freeBuffer(m_rbuf); m_rbuf=NULL; + Utils::freeBuffer(m_rbuf, MAX_SOCK_RD_BUFSIZE); m_rbuf=NULL; } if(m_pError!=NULL){ delete m_pError; m_pError=NULL; @@ -244,9 +244,13 @@ class DrillClientImpl{ void handleHShakeReadTimeout(const boost::system::error_code & err); // Query results void getNextResult(); - status_t readMsg(ByteBuf_t _buf, ByteBuf_t* allocatedBuffer, InBoundRpcMessage& msg, boost::system::error_code& error); - status_t processQueryResult(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg); - status_t processQueryId(ByteBuf_t allocatedBuffer, InBoundRpcMessage& msg ); + status_t readMsg( + ByteBuf_t _buf, + AllocatedBufferPtr* allocatedBuffer, + InBoundRpcMessage& msg, + boost::system::error_code& error); + status_t processQueryResult(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg); + status_t processQueryId(AllocatedBufferPtr allocatedBuffer, InBoundRpcMessage& msg ); void handleReadTimeout(const boost::system::error_code & err); void handleRead(ByteBuf_t _buf, const boost::system::error_code & err, size_t bytes_transferred) ; status_t validateMessage(InBoundRpcMessage& msg, exec::shared::QueryResult& qr, std::string& valError); diff --git a/contrib/native/client/src/clientlib/recordBatch.cpp b/contrib/native/client/src/clientlib/recordBatch.cpp index 17073bd6284..4c55f048270 100644 --- a/contrib/native/client/src/clientlib/recordBatch.cpp +++ b/contrib/native/client/src/clientlib/recordBatch.cpp @@ -312,7 +312,7 @@ RecordBatch::~RecordBatch(){ } m_fieldDefs->clear(); delete m_pQueryResult; - Utils::freeBuffer(m_allocatedBuffer); + delete m_allocatedBuffer; } ret_t RecordBatch::build(){ diff --git a/contrib/native/client/src/clientlib/utils.cpp b/contrib/native/client/src/clientlib/utils.cpp new file mode 100644 index 00000000000..999ee9ff9d1 --- /dev/null +++ b/contrib/native/client/src/clientlib/utils.cpp @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "utils.hpp" +#include "drill/common.hpp" + +namespace Drill{ + + +boost::mutex AllocatedBuffer::s_memCVMutex; +boost::condition_variable AllocatedBuffer::s_memCV; +size_t AllocatedBuffer::s_allocatedMem=0; +bool AllocatedBuffer::s_isBufferLimitReached=false; + +ByteBuf_t Utils::allocateBuffer(size_t len){ + boost::lock_guard memLock(AllocatedBuffer::s_memCVMutex); + AllocatedBuffer::s_allocatedMem+=len; + //http://stackoverflow.com/questions/2688466/why-mallocmemset-is-slower-than-calloc + ByteBuf_t b = (ByteBuf_t)calloc(len, sizeof(Byte_t)); + size_t safeSize= DrillClientConfig::getBufferLimit()-MEM_CHUNK_SIZE; + if(b!=NULL && AllocatedBuffer::s_allocatedMem >= safeSize){ + AllocatedBuffer::s_isBufferLimitReached=true; + } + return b; +} + +void Utils::freeBuffer(ByteBuf_t b, size_t len){ + boost::lock_guard memLock(AllocatedBuffer::s_memCVMutex); + AllocatedBuffer::s_allocatedMem-=len; + free(b); + size_t safeSize= DrillClientConfig::getBufferLimit()-MEM_CHUNK_SIZE; + if(b!=NULL && AllocatedBuffer::s_allocatedMem < safeSize){ + AllocatedBuffer::s_isBufferLimitReached=false; + //signal any waiting threads + AllocatedBuffer::s_memCV.notify_one(); + } +} + + +AllocatedBuffer::AllocatedBuffer(size_t l){ + m_pBuffer=NULL; + m_pBuffer=Utils::allocateBuffer(l); + m_bufSize=m_pBuffer!=NULL?l:0; +} + +AllocatedBuffer::~AllocatedBuffer(){ + Utils::freeBuffer(m_pBuffer, m_bufSize); + m_pBuffer=NULL; + m_bufSize=0; +} + +} // namespace diff --git a/contrib/native/client/src/clientlib/utils.hpp b/contrib/native/client/src/clientlib/utils.hpp index 9def9b4da30..e8e997460ed 100644 --- a/contrib/native/client/src/clientlib/utils.hpp +++ b/contrib/native/client/src/clientlib/utils.hpp @@ -23,24 +23,37 @@ #include #include #include -#include +#include #include "drill/common.hpp" +#include "drill/drillClient.hpp" namespace Drill{ -class Utils{ +// Wrapper Class to keep track of allocated memory +class AllocatedBuffer{ public: + AllocatedBuffer(size_t l); + ~AllocatedBuffer(); - //allocate memory for Record Batches - static ByteBuf_t allocateBuffer(size_t len){ - //http://stackoverflow.com/questions/2688466/why-mallocmemset-is-slower-than-calloc - ByteBuf_t b = (ByteBuf_t)calloc(len, sizeof(Byte_t)); return b; - } - static void freeBuffer(ByteBuf_t b){ free(b); } + ByteBuf_t m_pBuffer; + size_t m_bufSize; -}; // Utils + // keep track of allocated memory. The client lib blocks + // if we have allocated up to a limit (defined in drillClientConfig). + static boost::mutex s_memCVMutex; + static boost::condition_variable s_memCV; + static size_t s_allocatedMem; + static bool s_isBufferLimitReached; + +}; +class Utils{ + public: + //allocate memory for Record Batches + static ByteBuf_t allocateBuffer(size_t len); + static void freeBuffer(ByteBuf_t b, size_t len); +}; // Utils } // namespace Drill diff --git a/contrib/native/client/src/include/drill/common.hpp b/contrib/native/client/src/include/drill/common.hpp index 2113ce5e14d..74f81b60952 100644 --- a/contrib/native/client/src/include/drill/common.hpp +++ b/contrib/native/client/src/include/drill/common.hpp @@ -33,6 +33,9 @@ #define MAX_CONNECT_STR 4096 #define MAX_SOCK_RD_BUFSIZE 1024 +#define MEM_CHUNK_SIZE 64*1024; // 64K +#define MAX_MEM_ALLOC_SIZE 256*1024*1024; // 256 MB + #ifdef _DEBUG #define EXTRA_DEBUGGING #define CODER_DEBUGGING @@ -48,6 +51,9 @@ typedef Byte_t * ByteBuf_t; class FieldMetadata; typedef boost::shared_ptr< std::vector > FieldDefPtr; +class AllocatedBuffer; +typedef AllocatedBuffer* AllocatedBufferPtr; + typedef enum{ QRY_SUCCESS=0, QRY_FAILURE=1, diff --git a/contrib/native/client/src/include/drill/recordBatch.hpp b/contrib/native/client/src/include/drill/recordBatch.hpp index e9298bf1790..9a3df2b6fff 100644 --- a/contrib/native/client/src/include/drill/recordBatch.hpp +++ b/contrib/native/client/src/include/drill/recordBatch.hpp @@ -836,10 +836,10 @@ class ValueVectorFactory{ class DECLSPEC_DRILL_CLIENT RecordBatch{ public: - //m_allocatedBuffer is the memory block allocated to hold the incoming RPC message. Record BAtches operate on - //slices of the allcoated buffer. The first slice (the first Field Batch), begins at m_buffer. Data in the + //m_allocatedBuffer is the memory block allocated to hold the incoming RPC message. Record Batches operate on + //slices of the allocated buffer. The first slice (the first Field Batch), begins at m_buffer. Data in the //allocated buffer before m_buffer is mostly the RPC header, and the QueryResult object. - RecordBatch(exec::shared::QueryResult* pResult, ByteBuf_t r, ByteBuf_t b) + RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b) :m_fieldDefs(new(std::vector)){ m_pQueryResult=pResult; m_pRecordBatchDef=&pResult->def(); @@ -892,7 +892,7 @@ class DECLSPEC_DRILL_CLIENT RecordBatch{ private: const exec::shared::QueryResult* m_pQueryResult; const exec::shared::RecordBatchDef* m_pRecordBatchDef; - ByteBuf_t m_allocatedBuffer; + AllocatedBufferPtr m_allocatedBuffer; ByteBuf_t m_buffer; //build the current schema out of the field metadata FieldDefPtr m_fieldDefs; From 1576ddd989a8d978f1cd1428db54ba0e5b7ee33b Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Wed, 11 Jun 2014 18:10:34 -0700 Subject: [PATCH 2/8] DRILL-1021: Windows build --- contrib/native/client/CMakeLists.txt | 25 ++- .../client/cmakeModules/FindZookeeper.cmake | 28 ++- .../client/patches/zookeeper-3.4.6-x64.patch | 163 ++++++++++++++++ contrib/native/client/readme.win.txt | 179 ++++++++++++++++++ .../client/src/clientlib/CMakeLists.txt | 5 + .../client/src/clientlib/drillClient.cpp | 1 + .../client/src/clientlib/drillClientImpl.cpp | 6 + .../client/src/clientlib/drillClientImpl.hpp | 5 + .../client/src/include/drill/common.hpp | 8 + .../native/client/src/protobuf/CMakeLists.txt | 4 + 10 files changed, 417 insertions(+), 7 deletions(-) create mode 100644 contrib/native/client/patches/zookeeper-3.4.6-x64.patch create mode 100644 contrib/native/client/readme.win.txt diff --git a/contrib/native/client/CMakeLists.txt b/contrib/native/client/CMakeLists.txt index 31ac47278eb..603586d0e51 100644 --- a/contrib/native/client/CMakeLists.txt +++ b/contrib/native/client/CMakeLists.txt @@ -26,9 +26,16 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmakeModules/") # Find Boost -set(Boost_USE_STATIC_LIBS OFF) -set(Boost_USE_MULTITHREADED ON) -set(Boost_USE_STATIC_RUNTIME OFF) +if(MSVC) + set(Boost_USE_STATIC_LIBS ON) + set(Boost_USE_MULTITHREADED ON) + set(Boost_USE_STATIC_RUNTIME OFF) +else() + set(Boost_USE_STATIC_LIBS OFF) + set(Boost_USE_MULTITHREADED ON) + set(Boost_USE_STATIC_RUNTIME OFF) +endif() + find_package(Boost 1.53.0 REQUIRED COMPONENTS regex system date_time chrono thread ) include_directories(${Boost_INCLUDE_DIRS}) @@ -36,8 +43,18 @@ if(CMAKE_COMPILER_IS_GNUCXX) set(CMAKE_EXE_LINKER_FLAGS "-lrt -lpthread") set(CMAKE_CXX_FLAGS "-fPIC") endif() +if(MSVC) + set(CMAKE_CXX_FLAGS "/EHsc") +endif() -add_definitions(-DBOOST_ALL_DYN_LINK) +if(MSVC) + # ask boost to not to auto link any libraries + add_definitions(-DBOOST_ALL_NO_LIB) + # use static libs with zookeeper + add_definitions(-DUSE_STATIC_LIB) +else() + add_definitions(-DBOOST_ALL_DYN_LINK) +endif() # Find Protobufs find_package(Protobuf REQUIRED ) diff --git a/contrib/native/client/cmakeModules/FindZookeeper.cmake b/contrib/native/client/cmakeModules/FindZookeeper.cmake index c727ad26c46..fd8247f28cf 100644 --- a/contrib/native/client/cmakeModules/FindZookeeper.cmake +++ b/contrib/native/client/cmakeModules/FindZookeeper.cmake @@ -27,10 +27,32 @@ #pkg_check_modules(PC_LIBXML QUIET libxml-2.0) #set(Zookeeper_DEFINITIONS ${PC_LIBXML_CFLAGS_OTHER}) -find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include) +if (MSVC) + if(${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(ZK_BuildOutputDir "Debug") + else() + set(ZK_BuildOutputDir "Release") + endif() + if("${ZOOKEEPER_HOME}_" MATCHES "^_$") + message(" ") + message("- Please set the cache variable ZOOKEEPER_HOME to point to the directory with the zookeeper source.") + message("- CMAKE will look for zookeeper include files in $ZOOKEEPER_HOME/src/c/include.") + message("- CMAKE will look for zookeeper library files in $ZOOKEEPER_HOME/src/c/Debug or $ZOOKEEPER_HOME/src/c/Release.") + else() + FILE(TO_CMAKE_PATH ${ZOOKEEPER_HOME} Zookeeper_HomePath) + set(Zookeeper_LIB_PATHS ${Zookeeper_HomePath}/src/c/${ZK_BuildOutputDir}) + + find_path(ZK_INCLUDE_DIR zookeeper.h ${Zookeeper_HomePath}/src/c/include) + find_path(ZK_INCLUDE_DIR_GEN zookeeper.jute.h ${Zookeeper_HomePath}/src/c/generated) + set(Zookeeper_INCLUDE_DIR zookeeper.h ${ZK_INCLUDE_DIR} ${ZK_INCLUDE_DIR_GEN} ) + find_library(Zookeeper_LIBRARY NAMES zookeeper PATHS ${Zookeeper_LIB_PATHS}) + endif() +else() + set(Zookeeper_LIB_PATHS /usr/local/lib /opt/local/lib) + find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include) + find_library(Zookeeper_LIBRARY NAMES zookeeper_mt PATHS ${Zookeeper_LIB_PATHS}) +endif() -set(Zookeeper_LIB_PATHS /usr/local/lib /opt/local/lib) -find_library(Zookeeper_LIBRARY NAMES zookeeper_mt PATHS ${Zookeeper_LIB_PATHS}) set(Zookeeper_LIBRARIES ${Zookeeper_LIBRARY} ) set(Zookeeper_INCLUDE_DIRS ${Zookeeper_INCLUDE_DIR} ) diff --git a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch new file mode 100644 index 00000000000..96f2d103252 --- /dev/null +++ b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch @@ -0,0 +1,163 @@ +From 64697ddd8a90f29d1693658f04e975e435e3c869 Mon Sep 17 00:00:00 2001 +From: unknown +Date: Thu, 5 Jun 2014 16:40:48 -0700 +Subject: [PATCH] Allow zookeeper to build in x64 + +--- + src/c/include/winstdint.h | 4 ++++ + src/c/src/mt_adaptor.c | 54 +++++++++++++++++++++++------------------------ + 2 files changed, 30 insertions(+), 28 deletions(-) + +diff --git a/src/c/include/winstdint.h b/src/c/include/winstdint.h +index d02608a..df405f7 100644 +--- a/src/c/include/winstdint.h ++++ b/src/c/include/winstdint.h +@@ -40,6 +40,9 @@ + #pragma once + #endif + ++#if (_MSC_VER > 1500) // Visual Studio 2010 and Beyond ++#include ++#else + #include + + // For Visual Studio 6 in C++ mode and for many Visual Studio versions when +@@ -244,4 +247,5 @@ typedef uint64_t uintmax_t; + #endif // __STDC_CONSTANT_MACROS ] + + ++#endif + #endif // _MSC_STDINT_H_ ] +diff --git a/src/c/src/mt_adaptor.c b/src/c/src/mt_adaptor.c +index 974063f..5ce0fd9 100644 +--- a/src/c/src/mt_adaptor.c ++++ b/src/c/src/mt_adaptor.c +@@ -114,7 +114,7 @@ int process_async(int outstanding_sync) + unsigned __stdcall do_io( void * ); + unsigned __stdcall do_completion( void * ); + +-int handle_error(SOCKET sock, char* message) ++int handle_error(zhandle_t* zh, SOCKET sock, char* message) + { + LOG_ERROR(("%s. %d",message, WSAGetLastError())); + closesocket (sock); +@@ -122,7 +122,7 @@ int handle_error(SOCKET sock, char* message) + } + + //--create socket pair for interupting selects. +-int create_socket_pair(SOCKET fds[2]) ++int create_socket_pair(zhandle_t* zh, SOCKET fds[2]) + { + struct sockaddr_in inaddr; + struct sockaddr addr; +@@ -141,23 +141,23 @@ int create_socket_pair(SOCKET fds[2]) + inaddr.sin_port = 0; //--system assigns the port + + if ( setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)) == SOCKET_ERROR ) { +- return handle_error(lst,"Error trying to set socket option."); ++ return handle_error(zh, lst,"Error trying to set socket option."); + } + if (bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)) == SOCKET_ERROR){ +- return handle_error(lst,"Error trying to bind socket."); ++ return handle_error(zh, lst,"Error trying to bind socket."); + } + if (listen(lst,1) == SOCKET_ERROR){ +- return handle_error(lst,"Error trying to listen on socket."); ++ return handle_error(zh, lst,"Error trying to listen on socket."); + } + len=sizeof(inaddr); + getsockname(lst, &addr,&len); + fds[0]=socket(AF_INET, SOCK_STREAM,0); + if (connect(fds[0],&addr,len) == SOCKET_ERROR){ +- return handle_error(lst, "Error while connecting to socket."); ++ return handle_error(zh, lst, "Error while connecting to socket."); + } + if ((fds[1]=accept(lst,0,0)) == INVALID_SOCKET){ + closesocket(fds[0]); +- return handle_error(lst, "Error while accepting socket connection."); ++ return handle_error(zh, lst, "Error while accepting socket connection."); + } + closesocket(lst); + return 0; +@@ -238,11 +238,11 @@ int adaptor_init(zhandle_t *zh) + + /* We use a pipe for interrupting select() in unix/sol and socketpair in windows. */ + #ifdef WIN32 +- if (create_socket_pair(adaptor_threads->self_pipe) == -1){ ++ if (create_socket_pair(zh, adaptor_threads->self_pipe) == -1){ + LOG_ERROR(("Can't make a socket.")); + #else + if(pipe(adaptor_threads->self_pipe)==-1) { +- LOG_ERROR(("Can't make a pipe %d",errno)); ++ LOG_ERROR(LOGCALLBACK(zh), "Can't make a pipe %d",errno); + #endif + free(adaptor_threads); + return -1; +@@ -255,6 +255,7 @@ int adaptor_init(zhandle_t *zh) + zh->adaptor_priv = adaptor_threads; + pthread_mutex_init(&zh->to_process.lock,0); + pthread_mutex_init(&adaptor_threads->zh_lock,0); ++ pthread_mutex_init(&adaptor_threads->zh_lock,0); + // to_send must be recursive mutex + pthread_mutexattr_init(&recursive_mx_attr); + pthread_mutexattr_settype(&recursive_mx_attr, PTHREAD_MUTEX_RECURSIVE); +@@ -364,7 +365,7 @@ void *do_io(void *v) + + api_prolog(zh); + notify_thread_ready(zh); +- LOG_DEBUG(("started IO thread")); ++ LOG_DEBUG(LOGCALLBACK(zh), "started IO thread"); + fds[0].fd=adaptor_threads->self_pipe[0]; + fds[0].events=POLLIN; + while(!zh->close_requested) { +@@ -483,25 +484,9 @@ int32_t inc_ref_counter(zhandle_t* zh,int i) + int32_t fetch_and_add(volatile int32_t* operand, int incr) + { + #ifndef WIN32 +- int32_t result; +- asm __volatile__( +- "lock xaddl %0,%1\n" +- : "=r"(result), "=m"(*(int *)operand) +- : "0"(incr) +- : "memory"); +- return result; ++ return __sync_fetch_and_add(operand, incr); + #else +- volatile int32_t result; +- _asm +- { +- mov eax, operand; //eax = v; +- mov ebx, incr; // ebx = i; +- mov ecx, 0x0; // ecx = 0; +- lock xadd dword ptr [eax], ecx; +- lock xadd dword ptr [eax], ebx; +- mov result, ecx; // result = ebx; +- } +- return result; ++ return InterlockedExchangeAdd(operand, incr); + #endif + } + +@@ -515,6 +500,19 @@ __attribute__((constructor)) int32_t get_xid() + return fetch_and_add(&xid,1); + } + ++void lock_reconfig(struct _zhandle *zh) ++{ ++ struct adaptor_threads *adaptor = zh->adaptor_priv; ++ if(adaptor) ++ pthread_mutex_lock(&adaptor->zh_lock); ++} ++void unlock_reconfig(struct _zhandle *zh) ++{ ++ struct adaptor_threads *adaptor = zh->adaptor_priv; ++ if(adaptor) ++ pthread_mutex_lock(&adaptor->zh_lock); ++} ++ + void enter_critical(zhandle_t* zh) + { + struct adaptor_threads *adaptor = zh->adaptor_priv; +-- +1.9.2.msysgit.0 + diff --git a/contrib/native/client/readme.win.txt b/contrib/native/client/readme.win.txt new file mode 100644 index 00000000000..df6ee644353 --- /dev/null +++ b/contrib/native/client/readme.win.txt @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +Building the Drill C++ Client on Windows + +This document lists the steps to build the Drill C++ Client on Windows. The +steps and examples are for Windows 7, using Visual Studio 2010 Express. Newer +Windows platforms should be more or less similar. + +1 Tools and software + +1.1 The following git tools would be useful to get source code and apply patches - + MSysGit and GitForWindows - http://msysgit.github.io/ + Tortoise SVN - http://tortoisesvn.net/ + +1.2 Windows SDK + The Windows SDK is required to enable 64 bit options for Visual Studio 2010 + Express. You can get it here: + http://www.microsoft.com/en-us/download/details.aspx?id=8279 + Note: For 64 bit builds, change the platform toolset to Windows SDK for your + project. + (Root node, i.e. the projectname) Properties ->Configuration Properties->General->Platform Toolset = Windows7.1SDK + +1.3 [Optional] Windows Driver kit + The Windows Driver kit is required to get the 64 bit assembler ml64. The + 32 bit assembler masm is included in VS2010. The assembler is required to build + boost and can be ignored for Drill + + ML64(the 64 bit assembler) can be downloaded from here: + http://www.microsoft.com/en-us/download/details.aspx?id=11800 + MASM(the 32 bit assembler) should be installed in VC/bin already. If not it + can be downloaded from here: + http://www.microsoft.com/en-us/download/details.aspx?id=12654 + + Add the paths to the assemblers in your path environment variable. ml64 can be + found in (or the appropriate path to the WinDDK installation on your system): + C:\WinDDK\7600.16385.1\bin\x86\amd64 + +2 Dependencies + +2.0 + a) The Drill client library requires the following third party libraries - + boost + zookeeper C API + protobufs + The Drill client is linked with STATIC versions of these libraries. The + libraries are themselves linked with the DYNAMIC C Runtime DLLs. It is + important that the libraries all have the same linkage model, otherwise the + Drill Client library will not link successfully. + b) The build assumes that Zookeeper is not availble as a + binary installer and is available in source form only for windows. The + location of the header files, as a result is different from Unix like + systems. This will be important later with Cmake. + c) The document below will refer to the following as the directories where + the thirdparty library source is installed. + BOOST_HOME - Directory where boost source is installed. + ZOOKEEPER_HOME - Directory where Zookeeper source is installed. Note that + this is the directory for the full Zookeeper source not just the + source for the C library. + PROTOBUF_HOME - Directory where Protobuf source is installed. + +2.1 Boost (version 1.55) + a) Download Boost from: + http://www.boost.org/users/history/version_1_55_0.html + b) i) Boost 32 bit build - + Open a Visual Studio command prompt from the Visual Studio IDE + ii) Boost 64 bit build - + Open a command prompt from the Windows SDK menu + Start->All Programs->Windows SDK 7.1->Start Command Prompt + c) In the command prompt window - + C:> cd + C:> .\bootstrap.bat + d) Choose the build type (64 bit, 32 bit) and the variant type (debug, release) + and build the libraries. Boost build will write the libraries to + /stage/lib. Copy them to an appropriately named directory + + C:> .\b2 variant=debug link=static threading=multi address-model=64 toolset=msvc runtime-link=shared + C:> mkdir Debug64 + C:> copy stage\lib\* Debug64 + + C:> .\b2 variant=release link=static threading=multi address-model=64 toolset=msvc runtime-link=shared + C:> mkdir Release64 + C:> copy stage\lib\* Release64 + + C:> .\b2 variant=debug link=static threading=multi address-model=32 toolset=msvc runtime-link=shared + C:> mkdir Debug32 + C:> copy stage\lib\* Debug32 + + C:> .\b2 variant=release link=static threading=multi address-model=32 toolset=msvc runtime-link=shared + C:> mkdir Release32 + C:> copy stage\lib\* Release32 + e) Notes: + i) For more information on Boost build + http://www.boost.org/doc/libs/1_55_0/more/getting_started/windows.html#the-boost-distribution + ii) Detail options for b2 - + http://www.boost.org/boost-build2/doc/html/bbv2/overview/invocation.html + iii) If you do not have the 64 bit assembler installed, boost-context does not + build. It is safe to ignore it as boost-context is not needed for Drill + +2.2 Protobuf (2.5.0) + a) Protobuf builds static libraries + b) In Visual Studio, open /vsprojects/protobuf.sln. The IDE may + update the solution file. This should go thru successfully. + c) If build for 64 bit, add a 64 bit project configuration for each project. (Make sure the + platform toolset is set to Windows7.1SDK) + d) Build + +2.3 Zookeeper (3.4.6) + a) Set the ZOOKEEPER_HOME environment variable + b) The 3.4.6 release of Zookeeper does not build correctly on 64 bit windows. To + fix that for the 64 bit build, apply patch zookeeper-3.4.6-x64.patch + For example in Msysgit + $ cd && git apply /contrib/native/client/patches/zookeeper-3.4.6-x64.patch + c) InVisual Studio 2010 Express open /src/c/zookeeper.sln + i) Add a 64 bit project configuration for each project. (Make sure the + platform toolset is set to Windows7.1SDK) + ii) Change the output type for the zookeeper project to a static lib + Properties->Configuration Properties->General->Configuration Type = Static Library + iii) In the cli project add the preprocessor define USE_STATIC_LIB + iiii) Build. Build zookeeper lib first, then build cli + +3 Building Drill Clientlib +3.1 SET the following environment variables + set BOOST_LIBRARYDIR=\BUILD_TYPE + set BOOST_INCLUDEDIR= + +3.2 Generate the Visual Studio Solutions file + C:> cd /contrib/native/client + C:> mkdir build + C:> cd build + + a) For the 32 bit build : + C:> cmake -G "Visual Studio 10" -D ZOOKEEPER_HOME= -D PROTOBUF_SRC_ROOT_FOLDER= -D CMAKE_BUILD_TYPE=Debug .. + + b) For the 64 bit build : + C:> cmake -G "Visual Studio 10 Win64 " -D ZOOKEEPER_HOME= -D PROTOBUF_SRC_ROOT_FOLDER= -D CMAKE_BUILD_TYPE=Debug .. + +3.3 Open the generated /contrib/native/client/build/drillclient.sln + file in Visual Studio. + a) If doing a Debug build, check the link libraries for the clientlib + project. You might see the protobuf library being linked from the Release + build. Correct that to pick up the library from the Debug build. (See 4.2 + below) + +3.4 Select the ALL_BUILD project and build. + +4 Common Problems +4.1 In the 64 bit build, If you get: " error LNK1112: module machine type 'x64' conflicts with target machine type 'X86'" + a) Some library is a 32 bit library and you're linking it into a 64 bit Drill + Client library, or vice versa. You can check the type of the libraryi with + dumpbin. + dumpbin library_or_dll /headers + b) Check the linker settings for the project. + Properties->Configuration Properties->Linker->Command Line->Additional Options. + If you're doing a 64 bit build and you see "/machine:X86", remove it and rebuild + +4.2) error LNK2038: mismatch detected for '_ITERATOR_DEBUG_LEVEL': value '0' doesn't match value '2' in decimalUtils.obj + a) This happens if you are doing a Debug build and linking a Release build + version of some library, or vice versa. Check the link libraries. + Properties->Configuration Properties->Linker->Input->Additional Dependencies. + Check the libraries are all the same as your configuration (all debug, or all + release). + In particular, for debug builds, check the path of the protobuf library. + diff --git a/contrib/native/client/src/clientlib/CMakeLists.txt b/contrib/native/client/src/clientlib/CMakeLists.txt index dc8f032679c..a2e705273e9 100644 --- a/contrib/native/client/src/clientlib/CMakeLists.txt +++ b/contrib/native/client/src/clientlib/CMakeLists.txt @@ -41,5 +41,10 @@ set_property( PROPERTY COMPILE_DEFINITIONS_DEBUG DEBUG DEBUG=1 THREADED ) +if(MSVC) + set(CMAKE_CXX_FLAGS "/EHsc") + add_definitions(-DDRILL_CLIENT_EXPORTS) +endif() + add_library(drillClient SHARED ${CLIENTLIB_SRC_FILES} ) target_link_libraries(drillClient ${Boost_LIBRARIES} ${PROTOBUF_LIBRARY} ${Zookeeper_LIBRARIES} protomsgs y2038) diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp index 6611332b0a6..a7aafaa06e2 100644 --- a/contrib/native/client/src/clientlib/drillClient.cpp +++ b/contrib/native/client/src/clientlib/drillClient.cpp @@ -17,6 +17,7 @@ */ +#include "drill/common.hpp" #include "drill/drillClient.hpp" #include "drill/recordBatch.hpp" #include "drillClientImpl.hpp" diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp index f2627b0681f..ff9e419adef 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.cpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp @@ -16,6 +16,8 @@ * limitations under the License. */ + +#include "drill/common.hpp" #include #include #include @@ -23,7 +25,11 @@ #include #include #include +#ifdef _WIN32 +#include +#else #include +#endif #include "drill/drillClient.hpp" #include "drill/recordBatch.hpp" diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp index 42abbfa95cb..22574374bd5 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.hpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp @@ -28,6 +28,7 @@ //#define BOOST_ASIO_DISABLE_IOCP //#endif // _WIN32 +#include "drill/common.hpp" #include #include #include @@ -35,7 +36,11 @@ #include #include #include +#ifdef _WIN32 +#include +#else #include +#endif #include "drill/common.hpp" #include "drill/drillClient.hpp" diff --git a/contrib/native/client/src/include/drill/common.hpp b/contrib/native/client/src/include/drill/common.hpp index 74f81b60952..423faeda901 100644 --- a/contrib/native/client/src/include/drill/common.hpp +++ b/contrib/native/client/src/include/drill/common.hpp @@ -20,6 +20,14 @@ #ifndef _COMMON_H_ #define _COMMON_H_ +#ifdef _WIN32 +// The order of inclusion is important. Including winsock2 before everything else +// ensures that the correct typedefs are defined and that the older typedefs defined +// in winsock and windows.h are not picked up. +#include +#include +#endif + #include #include #include diff --git a/contrib/native/client/src/protobuf/CMakeLists.txt b/contrib/native/client/src/protobuf/CMakeLists.txt index 154138d02ea..1d0b3af1b09 100644 --- a/contrib/native/client/src/protobuf/CMakeLists.txt +++ b/contrib/native/client/src/protobuf/CMakeLists.txt @@ -103,6 +103,10 @@ add_custom_target(cpProtobufs #message("ProtoHeaders = ${ProtoHeaders}" ) #message("ProtoIncludes = ${ProtoIncludes}" ) +if(MSVC) + set(CMAKE_CXX_FLAGS "/EHsc") +endif() + add_library(protomsgs STATIC ${ProtoSources} ${ProtoHeaders} ${ProtoIncludes} ) #set linker properties. The first time around, the protobufs generated files may not exist # and CMAKE will not be able to determine the linker type. From 0f7648fb2efd2bb4814712111efc5414c0ada056 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Thu, 7 Aug 2014 16:32:00 -0700 Subject: [PATCH 3/8] preparing for build under windows --- .../client/patches/zookeeper-3.4.6-x64.patch | 23 ++++++++----------- contrib/native/client/readme.win.txt | 12 ++++++++-- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch index 96f2d103252..e2b9fee3d60 100644 --- a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch +++ b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch @@ -1,13 +1,3 @@ -From 64697ddd8a90f29d1693658f04e975e435e3c869 Mon Sep 17 00:00:00 2001 -From: unknown -Date: Thu, 5 Jun 2014 16:40:48 -0700 -Subject: [PATCH] Allow zookeeper to build in x64 - ---- - src/c/include/winstdint.h | 4 ++++ - src/c/src/mt_adaptor.c | 54 +++++++++++++++++++++++------------------------ - 2 files changed, 30 insertions(+), 28 deletions(-) - diff --git a/src/c/include/winstdint.h b/src/c/include/winstdint.h index d02608a..df405f7 100644 --- a/src/c/include/winstdint.h @@ -158,6 +148,13 @@ index 974063f..5ce0fd9 100644 void enter_critical(zhandle_t* zh) { struct adaptor_threads *adaptor = zh->adaptor_priv; --- -1.9.2.msysgit.0 - +diff --git a/src/c/zookeeper.vcproj b/src/c/zookeeper.vcproj +index dc3ab43..ffc6f90 100644 +--- a/src/c/zookeeper.vcproj ++++ b/src/c/zookeeper.vcproj +@@ -1,4 +1,4 @@ +-??? ++ + Configuration Properties->General->Platform Toolset = Windows7.1SDK + If your are running Windows 7 and having problem follow the instructions here http://stackoverflow.com/questions/19366006/error-when-installing-windows-sdk-7-1 1.3 [Optional] Windows Driver kit The Windows Driver kit is required to get the 64 bit assembler ml64. The @@ -76,7 +77,9 @@ Windows platforms should be more or less similar. 2.1 Boost (version 1.55) a) Download Boost from: - http://www.boost.org/users/history/version_1_55_0.html + i) http://www.boost.org/users/history/version_1_55_0.html + ii) open boost_1_55_0\boost/archive/iterators/transform_width.hpp and add the following to the include statements: #include + iii) Yes somehow this header was not included and is missing! See here for more info: https://svn.boost.org/trac/boost/ticket/8757 b) i) Boost 32 bit build - Open a Visual Studio command prompt from the Visual Studio IDE ii) Boost 64 bit build - @@ -113,12 +116,17 @@ Windows platforms should be more or less similar. build. It is safe to ignore it as boost-context is not needed for Drill 2.2 Protobuf (2.5.0) + Get protobuf from here: svn checkout http://protobuf.googlecode.com/svn/trunk/ protobuf-read-only + apply the patch as instructed in https://code.google.com/p/protobuf/source/diff?spec=svn480&r=480&format=side&path=/tags/2.5.0rc2/gtest/msvc/gtest.vcproj + + a) Protobuf builds static libraries b) In Visual Studio, open /vsprojects/protobuf.sln. The IDE may update the solution file. This should go thru successfully. c) If build for 64 bit, add a 64 bit project configuration for each project. (Make sure the platform toolset is set to Windows7.1SDK) - d) Build + d) Build the protobuf project first (not the solution) + e) Build the solution! 2.3 Zookeeper (3.4.6) a) Set the ZOOKEEPER_HOME environment variable From 261b689860f6bc2dfbdb2c27d3c526de00427177 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Fri, 8 Aug 2014 15:56:04 -0700 Subject: [PATCH 4/8] updated download link --- contrib/native/client/readme.win.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/native/client/readme.win.txt b/contrib/native/client/readme.win.txt index 2ecafdabeff..8e84bc5d7cc 100644 --- a/contrib/native/client/readme.win.txt +++ b/contrib/native/client/readme.win.txt @@ -116,8 +116,7 @@ Windows platforms should be more or less similar. build. It is safe to ignore it as boost-context is not needed for Drill 2.2 Protobuf (2.5.0) - Get protobuf from here: svn checkout http://protobuf.googlecode.com/svn/trunk/ protobuf-read-only - apply the patch as instructed in https://code.google.com/p/protobuf/source/diff?spec=svn480&r=480&format=side&path=/tags/2.5.0rc2/gtest/msvc/gtest.vcproj + Get protobuf from here: https://protobuf.googlecode.com/files/protobuf-2.5.0.zip a) Protobuf builds static libraries From 8ea65e3abe23ce10e77ccf2fe095958849021da7 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Mon, 11 Aug 2014 13:05:54 -0700 Subject: [PATCH 5/8] built under windows --- contrib/native/client/readme.win.txt | 17 +- .../client/src/clientlib/drillClientImpl.cpp | 5 - .../client/src/clientlib/drillClientImpl.hpp | 2 + .../src/include/drill/protobuf/Types.pb.h | 5 +- .../client/src/protobuf/BitControl.pb.cc | 305 +++++++++++++++++- .../client/src/protobuf/BitControl.pb.h | 192 ++++++++++- .../native/client/src/protobuf/Types.pb.cc | 10 +- .../client/src/protobuf/UserBitShared.pb.cc | 28 +- .../client/src/protobuf/UserBitShared.pb.h | 7 +- 9 files changed, 521 insertions(+), 50 deletions(-) diff --git a/contrib/native/client/readme.win.txt b/contrib/native/client/readme.win.txt index 8e84bc5d7cc..671c45bb4bb 100644 --- a/contrib/native/client/readme.win.txt +++ b/contrib/native/client/readme.win.txt @@ -35,7 +35,7 @@ Windows platforms should be more or less similar. Note: For 64 bit builds, change the platform toolset to Windows SDK for your project. (Root node, i.e. the projectname) Properties ->Configuration Properties->General->Platform Toolset = Windows7.1SDK - If your are running Windows 7 and having problem follow the instructions here http://stackoverflow.com/questions/19366006/error-when-installing-windows-sdk-7-1 + If your are running Windows 7 and having problem isntalling windows SDK follow the instructions here http://stackoverflow.com/questions/19366006/error-when-installing-windows-sdk-7-1 1.3 [Optional] Windows Driver kit The Windows Driver kit is required to get the 64 bit assembler ml64. The @@ -79,7 +79,7 @@ Windows platforms should be more or less similar. a) Download Boost from: i) http://www.boost.org/users/history/version_1_55_0.html ii) open boost_1_55_0\boost/archive/iterators/transform_width.hpp and add the following to the include statements: #include - iii) Yes somehow this header was not included and is missing! See here for more info: https://svn.boost.org/trac/boost/ticket/8757 + iii) Yes somehow this header was not included and has been missed! See here for more info: https://svn.boost.org/trac/boost/ticket/8757 b) i) Boost 32 bit build - Open a Visual Studio command prompt from the Visual Studio IDE ii) Boost 64 bit build - @@ -92,19 +92,19 @@ Windows platforms should be more or less similar. and build the libraries. Boost build will write the libraries to /stage/lib. Copy them to an appropriately named directory - C:> .\b2 variant=debug link=static threading=multi address-model=64 toolset=msvc runtime-link=shared + C:> .\b2 variant=debug link=static threading=multi address-model=64 toolset=msvc-10.0 runtime-link=shared C:> mkdir Debug64 C:> copy stage\lib\* Debug64 - C:> .\b2 variant=release link=static threading=multi address-model=64 toolset=msvc runtime-link=shared + C:> .\b2 variant=release link=static threading=multi address-model=64 toolset=msvc-10.0 runtime-link=shared C:> mkdir Release64 C:> copy stage\lib\* Release64 - C:> .\b2 variant=debug link=static threading=multi address-model=32 toolset=msvc runtime-link=shared + C:> .\b2 variant=debug link=static threading=multi address-model=32 toolset=msvc-10.0 runtime-link=shared C:> mkdir Debug32 C:> copy stage\lib\* Debug32 - C:> .\b2 variant=release link=static threading=multi address-model=32 toolset=msvc runtime-link=shared + C:> .\b2 variant=release link=static threading=multi address-model=32 toolset=msvc-10.0 runtime-link=shared C:> mkdir Release32 C:> copy stage\lib\* Release32 e) Notes: @@ -126,6 +126,7 @@ Windows platforms should be more or less similar. platform toolset is set to Windows7.1SDK) d) Build the protobuf project first (not the solution) e) Build the solution! + f) If building the solution failed build once more and it will probabley build for the second time!! 2.3 Zookeeper (3.4.6) a) Set the ZOOKEEPER_HOME environment variable @@ -139,7 +140,9 @@ Windows platforms should be more or less similar. ii) Change the output type for the zookeeper project to a static lib Properties->Configuration Properties->General->Configuration Type = Static Library iii) In the cli project add the preprocessor define USE_STATIC_LIB - iiii) Build. Build zookeeper lib first, then build cli + iv) Build. Build zookeeper lib first, then build cli + v)If building the solution failed build once more and it will probabley build for the second time!! + 3 Building Drill Clientlib 3.1 SET the following environment variables diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp index ff9e419adef..67123e3a338 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.cpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp @@ -25,11 +25,6 @@ #include #include #include -#ifdef _WIN32 -#include -#else -#include -#endif #include "drill/drillClient.hpp" #include "drill/recordBatch.hpp" diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp index 22574374bd5..69c5212f4ed 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.hpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp @@ -37,7 +37,9 @@ #include #include #ifdef _WIN32 +extern "C"{ #include +} #else #include #endif diff --git a/contrib/native/client/src/include/drill/protobuf/Types.pb.h b/contrib/native/client/src/include/drill/protobuf/Types.pb.h index 853415317e0..b26177c44ba 100644 --- a/contrib/native/client/src/include/drill/protobuf/Types.pb.h +++ b/contrib/native/client/src/include/drill/protobuf/Types.pb.h @@ -72,11 +72,12 @@ enum MinorType { DM_UNKNOWN = 37, INTERVALYEAR = 38, INTERVALDAY = 39, - LIST = 40 + LIST = 40, + GENERIC_OBJECT = 41 }; bool MinorType_IsValid(int value); const MinorType MinorType_MIN = LATE; -const MinorType MinorType_MAX = LIST; +const MinorType MinorType_MAX = GENERIC_OBJECT; const int MinorType_ARRAYSIZE = MinorType_MAX + 1; const ::google::protobuf::EnumDescriptor* MinorType_descriptor(); diff --git a/contrib/native/client/src/protobuf/BitControl.pb.cc b/contrib/native/client/src/protobuf/BitControl.pb.cc index 4f8749cee2c..a7623861e2a 100644 --- a/contrib/native/client/src/protobuf/BitControl.pb.cc +++ b/contrib/native/client/src/protobuf/BitControl.pb.cc @@ -37,6 +37,9 @@ const ::google::protobuf::internal::GeneratedMessageReflection* const ::google::protobuf::Descriptor* WorkQueueStatus_descriptor_ = NULL; const ::google::protobuf::internal::GeneratedMessageReflection* WorkQueueStatus_reflection_ = NULL; +const ::google::protobuf::Descriptor* FinishedReceiver_descriptor_ = NULL; +const ::google::protobuf::internal::GeneratedMessageReflection* + FinishedReceiver_reflection_ = NULL; const ::google::protobuf::EnumDescriptor* RpcType_descriptor_ = NULL; } // namespace @@ -142,6 +145,22 @@ void protobuf_AssignDesc_BitControl_2eproto() { ::google::protobuf::DescriptorPool::generated_pool(), ::google::protobuf::MessageFactory::generated_factory(), sizeof(WorkQueueStatus)); + FinishedReceiver_descriptor_ = file->message_type(5); + static const int FinishedReceiver_offsets_[2] = { + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, receiver_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, sender_), + }; + FinishedReceiver_reflection_ = + new ::google::protobuf::internal::GeneratedMessageReflection( + FinishedReceiver_descriptor_, + FinishedReceiver::default_instance_, + FinishedReceiver_offsets_, + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _has_bits_[0]), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _unknown_fields_), + -1, + ::google::protobuf::DescriptorPool::generated_pool(), + ::google::protobuf::MessageFactory::generated_factory(), + sizeof(FinishedReceiver)); RpcType_descriptor_ = file->enum_type(0); } @@ -165,6 +184,8 @@ void protobuf_RegisterTypes(const ::std::string&) { PlanFragment_descriptor_, &PlanFragment::default_instance()); ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( WorkQueueStatus_descriptor_, &WorkQueueStatus::default_instance()); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + FinishedReceiver_descriptor_, &FinishedReceiver::default_instance()); } } // namespace @@ -180,6 +201,8 @@ void protobuf_ShutdownFile_BitControl_2eproto() { delete PlanFragment_reflection_; delete WorkQueueStatus::default_instance_; delete WorkQueueStatus_reflection_; + delete FinishedReceiver::default_instance_; + delete FinishedReceiver_reflection_; } void protobuf_AddDesc_BitControl_2eproto() { @@ -216,15 +239,18 @@ void protobuf_AddDesc_BitControl_2eproto() { "e_zone\030\020 \001(\005\022\024\n\014options_json\030\021 \001(\t\"f\n\017Wo" "rkQueueStatus\022(\n\010endpoint\030\001 \001(\0132\026.exec.D" "rillbitEndpoint\022\024\n\014queue_length\030\002 \001(\005\022\023\n" - "\013report_time\030\003 \001(\003*\207\002\n\007RpcType\022\r\n\tHANDSH" - "AKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIAT" - "ILIZE_FRAGMENT\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020" - "\006\022\027\n\023REQ_FRAGMENT_STATUS\020\007\022\022\n\016REQ_BIT_ST" - "ATUS\020\010\022\024\n\020REQ_QUERY_STATUS\020\t\022\030\n\024RESP_FRA" - "GMENT_HANDLE\020\n\022\030\n\024RESP_FRAGMENT_STATUS\020\013" - "\022\023\n\017RESP_BIT_STATUS\020\014\022\025\n\021RESP_QUERY_STAT" - "US\020\rB+\n\033org.apache.drill.exec.protoB\nBit" - "ControlH\001", 1289); + "\013report_time\030\003 \001(\003\"h\n\020FinishedReceiver\022*" + "\n\010receiver\030\001 \001(\0132\030.exec.bit.FragmentHand" + "le\022(\n\006sender\030\002 \001(\0132\030.exec.bit.FragmentHa" + "ndle*\242\002\n\007RpcType\022\r\n\tHANDSHAKE\020\000\022\007\n\003ACK\020\001" + "\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIATILIZE_FRAGMENT" + "\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020\006\022\031\n\025REQ_RECEI" + "VER_FINISHED\020\007\022\027\n\023REQ_FRAGMENT_STATUS\020\010\022" + "\022\n\016REQ_BIT_STATUS\020\t\022\024\n\020REQ_QUERY_STATUS\020" + "\n\022\030\n\024RESP_FRAGMENT_HANDLE\020\013\022\030\n\024RESP_FRAG" + "MENT_STATUS\020\014\022\023\n\017RESP_BIT_STATUS\020\r\022\025\n\021RE" + "SP_QUERY_STATUS\020\016B+\n\033org.apache.drill.ex" + "ec.protoB\nBitControlH\001", 1422); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "BitControl.proto", &protobuf_RegisterTypes); BitControlHandshake::default_instance_ = new BitControlHandshake(); @@ -232,11 +258,13 @@ void protobuf_AddDesc_BitControl_2eproto() { FragmentStatus::default_instance_ = new FragmentStatus(); PlanFragment::default_instance_ = new PlanFragment(); WorkQueueStatus::default_instance_ = new WorkQueueStatus(); + FinishedReceiver::default_instance_ = new FinishedReceiver(); BitControlHandshake::default_instance_->InitAsDefaultInstance(); BitStatus::default_instance_->InitAsDefaultInstance(); FragmentStatus::default_instance_->InitAsDefaultInstance(); PlanFragment::default_instance_->InitAsDefaultInstance(); WorkQueueStatus::default_instance_->InitAsDefaultInstance(); + FinishedReceiver::default_instance_->InitAsDefaultInstance(); ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_BitControl_2eproto); } @@ -264,6 +292,7 @@ bool RpcType_IsValid(int value) { case 11: case 12: case 13: + case 14: return true; default: return false; @@ -2142,6 +2171,264 @@ ::google::protobuf::Metadata WorkQueueStatus::GetMetadata() const { } +// =================================================================== + +#ifndef _MSC_VER +const int FinishedReceiver::kReceiverFieldNumber; +const int FinishedReceiver::kSenderFieldNumber; +#endif // !_MSC_VER + +FinishedReceiver::FinishedReceiver() + : ::google::protobuf::Message() { + SharedCtor(); +} + +void FinishedReceiver::InitAsDefaultInstance() { + receiver_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance()); + sender_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance()); +} + +FinishedReceiver::FinishedReceiver(const FinishedReceiver& from) + : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); +} + +void FinishedReceiver::SharedCtor() { + _cached_size_ = 0; + receiver_ = NULL; + sender_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FinishedReceiver::~FinishedReceiver() { + SharedDtor(); +} + +void FinishedReceiver::SharedDtor() { + if (this != default_instance_) { + delete receiver_; + delete sender_; + } +} + +void FinishedReceiver::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ::google::protobuf::Descriptor* FinishedReceiver::descriptor() { + protobuf_AssignDescriptorsOnce(); + return FinishedReceiver_descriptor_; +} + +const FinishedReceiver& FinishedReceiver::default_instance() { + if (default_instance_ == NULL) protobuf_AddDesc_BitControl_2eproto(); + return *default_instance_; +} + +FinishedReceiver* FinishedReceiver::default_instance_ = NULL; + +FinishedReceiver* FinishedReceiver::New() const { + return new FinishedReceiver; +} + +void FinishedReceiver::Clear() { + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (has_receiver()) { + if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear(); + } + if (has_sender()) { + if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear(); + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->Clear(); +} + +bool FinishedReceiver::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) return false + ::google::protobuf::uint32 tag; + while ((tag = input->ReadTag()) != 0) { + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .exec.bit.FragmentHandle receiver = 1; + case 1: { + if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_receiver())); + } else { + goto handle_uninterpreted; + } + if (input->ExpectTag(18)) goto parse_sender; + break; + } + + // optional .exec.bit.FragmentHandle sender = 2; + case 2: { + if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + parse_sender: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_sender())); + } else { + goto handle_uninterpreted; + } + if (input->ExpectAtEnd()) return true; + break; + } + + default: { + handle_uninterpreted: + if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + return true; + } + DO_(::google::protobuf::internal::WireFormat::SkipField( + input, tag, mutable_unknown_fields())); + break; + } + } + } + return true; +#undef DO_ +} + +void FinishedReceiver::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // optional .exec.bit.FragmentHandle receiver = 1; + if (has_receiver()) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 1, this->receiver(), output); + } + + // optional .exec.bit.FragmentHandle sender = 2; + if (has_sender()) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 2, this->sender(), output); + } + + if (!unknown_fields().empty()) { + ::google::protobuf::internal::WireFormat::SerializeUnknownFields( + unknown_fields(), output); + } +} + +::google::protobuf::uint8* FinishedReceiver::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + // optional .exec.bit.FragmentHandle receiver = 1; + if (has_receiver()) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteMessageNoVirtualToArray( + 1, this->receiver(), target); + } + + // optional .exec.bit.FragmentHandle sender = 2; + if (has_sender()) { + target = ::google::protobuf::internal::WireFormatLite:: + WriteMessageNoVirtualToArray( + 2, this->sender(), target); + } + + if (!unknown_fields().empty()) { + target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( + unknown_fields(), target); + } + return target; +} + +int FinishedReceiver::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .exec.bit.FragmentHandle receiver = 1; + if (has_receiver()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->receiver()); + } + + // optional .exec.bit.FragmentHandle sender = 2; + if (has_sender()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->sender()); + } + + } + if (!unknown_fields().empty()) { + total_size += + ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( + unknown_fields()); + } + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FinishedReceiver::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const FinishedReceiver* source = + ::google::protobuf::internal::dynamic_cast_if_available( + &from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void FinishedReceiver::MergeFrom(const FinishedReceiver& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_receiver()) { + mutable_receiver()->::exec::bit::FragmentHandle::MergeFrom(from.receiver()); + } + if (from.has_sender()) { + mutable_sender()->::exec::bit::FragmentHandle::MergeFrom(from.sender()); + } + } + mutable_unknown_fields()->MergeFrom(from.unknown_fields()); +} + +void FinishedReceiver::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void FinishedReceiver::CopyFrom(const FinishedReceiver& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FinishedReceiver::IsInitialized() const { + + return true; +} + +void FinishedReceiver::Swap(FinishedReceiver* other) { + if (other != this) { + std::swap(receiver_, other->receiver_); + std::swap(sender_, other->sender_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.Swap(&other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata FinishedReceiver::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = FinishedReceiver_descriptor_; + metadata.reflection = FinishedReceiver_reflection_; + return metadata; +} + + // @@protoc_insertion_point(namespace_scope) } // namespace control diff --git a/contrib/native/client/src/protobuf/BitControl.pb.h b/contrib/native/client/src/protobuf/BitControl.pb.h index 06af929d8f4..ba171a67faf 100644 --- a/contrib/native/client/src/protobuf/BitControl.pb.h +++ b/contrib/native/client/src/protobuf/BitControl.pb.h @@ -44,6 +44,7 @@ class BitStatus; class FragmentStatus; class PlanFragment; class WorkQueueStatus; +class FinishedReceiver; enum RpcType { HANDSHAKE = 0, @@ -51,13 +52,14 @@ enum RpcType { GOODBYE = 2, REQ_INIATILIZE_FRAGMENT = 3, REQ_CANCEL_FRAGMENT = 6, - REQ_FRAGMENT_STATUS = 7, - REQ_BIT_STATUS = 8, - REQ_QUERY_STATUS = 9, - RESP_FRAGMENT_HANDLE = 10, - RESP_FRAGMENT_STATUS = 11, - RESP_BIT_STATUS = 12, - RESP_QUERY_STATUS = 13 + REQ_RECEIVER_FINISHED = 7, + REQ_FRAGMENT_STATUS = 8, + REQ_BIT_STATUS = 9, + REQ_QUERY_STATUS = 10, + RESP_FRAGMENT_HANDLE = 11, + RESP_FRAGMENT_STATUS = 12, + RESP_BIT_STATUS = 13, + RESP_QUERY_STATUS = 14 }; bool RpcType_IsValid(int value); const RpcType RpcType_MIN = HANDSHAKE; @@ -703,6 +705,102 @@ class WorkQueueStatus : public ::google::protobuf::Message { void InitAsDefaultInstance(); static WorkQueueStatus* default_instance_; }; +// ------------------------------------------------------------------- + +class FinishedReceiver : public ::google::protobuf::Message { + public: + FinishedReceiver(); + virtual ~FinishedReceiver(); + + FinishedReceiver(const FinishedReceiver& from); + + inline FinishedReceiver& operator=(const FinishedReceiver& from) { + CopyFrom(from); + return *this; + } + + inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { + return _unknown_fields_; + } + + inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const FinishedReceiver& default_instance(); + + void Swap(FinishedReceiver* other); + + // implements Message ---------------------------------------------- + + FinishedReceiver* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const FinishedReceiver& from); + void MergeFrom(const FinishedReceiver& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + + ::google::protobuf::Metadata GetMetadata() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .exec.bit.FragmentHandle receiver = 1; + inline bool has_receiver() const; + inline void clear_receiver(); + static const int kReceiverFieldNumber = 1; + inline const ::exec::bit::FragmentHandle& receiver() const; + inline ::exec::bit::FragmentHandle* mutable_receiver(); + inline ::exec::bit::FragmentHandle* release_receiver(); + inline void set_allocated_receiver(::exec::bit::FragmentHandle* receiver); + + // optional .exec.bit.FragmentHandle sender = 2; + inline bool has_sender() const; + inline void clear_sender(); + static const int kSenderFieldNumber = 2; + inline const ::exec::bit::FragmentHandle& sender() const; + inline ::exec::bit::FragmentHandle* mutable_sender(); + inline ::exec::bit::FragmentHandle* release_sender(); + inline void set_allocated_sender(::exec::bit::FragmentHandle* sender); + + // @@protoc_insertion_point(class_scope:exec.bit.control.FinishedReceiver) + private: + inline void set_has_receiver(); + inline void clear_has_receiver(); + inline void set_has_sender(); + inline void clear_has_sender(); + + ::google::protobuf::UnknownFieldSet _unknown_fields_; + + ::exec::bit::FragmentHandle* receiver_; + ::exec::bit::FragmentHandle* sender_; + + mutable int _cached_size_; + ::google::protobuf::uint32 _has_bits_[(2 + 31) / 32]; + + friend void protobuf_AddDesc_BitControl_2eproto(); + friend void protobuf_AssignDesc_BitControl_2eproto(); + friend void protobuf_ShutdownFile_BitControl_2eproto(); + + void InitAsDefaultInstance(); + static FinishedReceiver* default_instance_; +}; // =================================================================== @@ -1482,6 +1580,86 @@ inline void WorkQueueStatus::set_report_time(::google::protobuf::int64 value) { report_time_ = value; } +// ------------------------------------------------------------------- + +// FinishedReceiver + +// optional .exec.bit.FragmentHandle receiver = 1; +inline bool FinishedReceiver::has_receiver() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FinishedReceiver::set_has_receiver() { + _has_bits_[0] |= 0x00000001u; +} +inline void FinishedReceiver::clear_has_receiver() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FinishedReceiver::clear_receiver() { + if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear(); + clear_has_receiver(); +} +inline const ::exec::bit::FragmentHandle& FinishedReceiver::receiver() const { + return receiver_ != NULL ? *receiver_ : *default_instance_->receiver_; +} +inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_receiver() { + set_has_receiver(); + if (receiver_ == NULL) receiver_ = new ::exec::bit::FragmentHandle; + return receiver_; +} +inline ::exec::bit::FragmentHandle* FinishedReceiver::release_receiver() { + clear_has_receiver(); + ::exec::bit::FragmentHandle* temp = receiver_; + receiver_ = NULL; + return temp; +} +inline void FinishedReceiver::set_allocated_receiver(::exec::bit::FragmentHandle* receiver) { + delete receiver_; + receiver_ = receiver; + if (receiver) { + set_has_receiver(); + } else { + clear_has_receiver(); + } +} + +// optional .exec.bit.FragmentHandle sender = 2; +inline bool FinishedReceiver::has_sender() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FinishedReceiver::set_has_sender() { + _has_bits_[0] |= 0x00000002u; +} +inline void FinishedReceiver::clear_has_sender() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FinishedReceiver::clear_sender() { + if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear(); + clear_has_sender(); +} +inline const ::exec::bit::FragmentHandle& FinishedReceiver::sender() const { + return sender_ != NULL ? *sender_ : *default_instance_->sender_; +} +inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_sender() { + set_has_sender(); + if (sender_ == NULL) sender_ = new ::exec::bit::FragmentHandle; + return sender_; +} +inline ::exec::bit::FragmentHandle* FinishedReceiver::release_sender() { + clear_has_sender(); + ::exec::bit::FragmentHandle* temp = sender_; + sender_ = NULL; + return temp; +} +inline void FinishedReceiver::set_allocated_sender(::exec::bit::FragmentHandle* sender) { + delete sender_; + sender_ = sender; + if (sender) { + set_has_sender(); + } else { + clear_has_sender(); + } +} + // @@protoc_insertion_point(namespace_scope) diff --git a/contrib/native/client/src/protobuf/Types.pb.cc b/contrib/native/client/src/protobuf/Types.pb.cc index c7fa2eaaff8..a14f40453fb 100644 --- a/contrib/native/client/src/protobuf/Types.pb.cc +++ b/contrib/native/client/src/protobuf/Types.pb.cc @@ -91,7 +91,7 @@ void protobuf_AddDesc_Types_2eproto() { "inor_type\030\001 \001(\0162\021.common.MinorType\022\036\n\004mo" "de\030\002 \001(\0162\020.common.DataMode\022\r\n\005width\030\003 \001(" "\005\022\021\n\tprecision\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010t" - "imeZone\030\006 \001(\005*\374\003\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" + "imeZone\030\006 \001(\005*\220\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" "\003MAP\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT" "\020\005\022\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL1" "8\020\010\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPA" @@ -104,9 +104,10 @@ void protobuf_AddDesc_Types_2eproto() { "\036\022\t\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENS" "E\020!\022\022\n\016DECIMAL38DENSE\020\"\022\016\n\nDM_UNKNOWN\020%\022" "\020\n\014INTERVALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LI" - "ST\020(*=\n\010DataMode\022\017\n\013DM_OPTIONAL\020\000\022\017\n\013DM_" - "REQUIRED\020\001\022\017\n\013DM_REPEATED\020\002B-\n\035org.apach" - "e.drill.common.typesB\nTypeProtosH\001", 794); + "ST\020(\022\022\n\016GENERIC_OBJECT\020)*=\n\010DataMode\022\017\n\013" + "DM_OPTIONAL\020\000\022\017\n\013DM_REQUIRED\020\001\022\017\n\013DM_REP" + "EATED\020\002B-\n\035org.apache.drill.common.types" + "B\nTypeProtosH\001", 814); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "Types.proto", &protobuf_RegisterTypes); MajorType::default_instance_ = new MajorType(); @@ -162,6 +163,7 @@ bool MinorType_IsValid(int value) { case 38: case 39: case 40: + case 41: return true; default: return false; diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc index 88507e5a303..0e7649f2717 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc @@ -532,24 +532,25 @@ void protobuf_AddDesc_UserBitShared_2eproto() { "\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen" "tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI" "ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE" - "LLED\020\004\022\n\n\006FAILED\020\005*\372\004\n\020CoreOperatorType\022" + "LLED\020\004\022\n\n\006FAILED\020\005*\224\005\n\020CoreOperatorType\022" "\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001" "\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH" "_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO" "N_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVE" "R\020\010\022\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013\n\007PRO" - "JECT\020\n\022\023\n\017RANDOM_RECEIVER\020\013\022\020\n\014RANGE_SEN" - "DER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_RE" - "MOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_" - "N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t" - "\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_G" - "ROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM" - "_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQ" - "UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX" - "T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU" - "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017CO" - "MPLEX_TO_JSON\020\037B.\n\033org.apache.drill.exec" - ".protoB\rUserBitSharedH\001", 3543); + "JECT\020\n\022\026\n\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_" + "SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR" + "_REMOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nT" + "OP_N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020" + "\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_RO" + "W_GROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYS" + "TEM_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016P" + "ARQUET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013" + "TEXT_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON" + "_SUB_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n" + "\017COMPLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020" + " B.\n\033org.apache.drill.exec.protoB\rUserBi" + "tSharedH\001", 3569); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "UserBitShared.proto", &protobuf_RegisterTypes); UserCredentials::default_instance_ = new UserCredentials(); @@ -677,6 +678,7 @@ bool CoreOperatorType_IsValid(int value) { case 29: case 30: case 31: + case 32: return true; default: return false; diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h index 6ef17c38f27..e7d40b19d1f 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.h +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h @@ -171,7 +171,7 @@ enum CoreOperatorType { MERGING_RECEIVER = 8, ORDERED_PARTITION_SENDER = 9, PROJECT = 10, - RANDOM_RECEIVER = 11, + UNORDERED_RECEIVER = 11, RANGE_SENDER = 12, SCREEN = 13, SELECTION_VECTOR_REMOVER = 14, @@ -191,11 +191,12 @@ enum CoreOperatorType { TEXT_SUB_SCAN = 28, JSON_SUB_SCAN = 29, INFO_SCHEMA_SUB_SCAN = 30, - COMPLEX_TO_JSON = 31 + COMPLEX_TO_JSON = 31, + PRODUCER_CONSUMER = 32 }; bool CoreOperatorType_IsValid(int value); const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER; -const CoreOperatorType CoreOperatorType_MAX = COMPLEX_TO_JSON; +const CoreOperatorType CoreOperatorType_MAX = PRODUCER_CONSUMER; const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1; const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor(); From bf061403233756f8a1600dc2a8fae10c2ead1f57 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Fri, 15 Aug 2014 18:25:39 -0700 Subject: [PATCH 6/8] Cleaning public API from Protobuf dependencies --- .../native/client/example/querySubmitter.cpp | 17 +++++---- .../client/src/clientlib/drillClient.cpp | 11 +++--- .../client/src/clientlib/recordBatch.cpp | 35 ++++++++++++++++++ .../client/src/include/drill/drillClient.hpp | 25 ++++++++++--- .../client/src/include/drill/drillc.hpp | 1 - .../client/src/include/drill/recordBatch.hpp | 36 ++++++++----------- .../{include/drill => }/protobuf/User.pb.h | 0 7 files changed, 87 insertions(+), 38 deletions(-) rename contrib/native/client/src/{include/drill => }/protobuf/User.pb.h (100%) diff --git a/contrib/native/client/example/querySubmitter.cpp b/contrib/native/client/example/querySubmitter.cpp index 17bec3c75f0..2b3ca81f346 100644 --- a/contrib/native/client/example/querySubmitter.cpp +++ b/contrib/native/client/example/querySubmitter.cpp @@ -21,6 +21,9 @@ #include #include #include "drill/drillc.hpp" +//#include "../protobuf/User.pb.h" + + Drill::status_t SchemaListener(void* ctx, Drill::FieldDefPtr fields, Drill::DrillClientError* err){ if(!err){ @@ -229,6 +232,8 @@ Drill::logLevel_t getLogLevel(const char *s){ return Drill::LOG_ERROR; } + + int main(int argc, char* argv[]) { try { @@ -244,7 +249,7 @@ int main(int argc, char* argv[]) { std::string type_str=qsOptionValues["type"]; std::string logLevel=qsOptionValues["logLevel"]; - exec::shared::QueryType type; + Drill::QueryType type; if(!validate(type_str, queryList, planList)){ exit(1); @@ -255,16 +260,16 @@ int main(int argc, char* argv[]) { std::vector queryInputs; if(type_str=="sql" ){ readQueries(queryList, queryInputs); - type=exec::shared::SQL; + type=Drill::SQL; }else if(type_str=="physical" ){ readPlans(planList, queryInputs); - type=exec::shared::PHYSICAL; + type=Drill::PHYSICAL; }else if(type_str == "logical"){ readPlans(planList, queryInputs); - type=exec::shared::LOGICAL; + type=Drill::LOGICAL; }else{ readQueries(queryList, queryInputs); - type=exec::shared::SQL; + type=Drill::SQL; } std::vector::iterator queryInpIter; @@ -329,7 +334,7 @@ int main(int argc, char* argv[]) { }else{ for(queryInpIter = queryInputs.begin(); queryInpIter != queryInputs.end(); queryInpIter++) { Drill::QueryHandle_t* qryHandle = new Drill::QueryHandle_t; - client.submitQuery(type, *queryInpIter, QueryResultsListener, NULL, qryHandle); + //client.submitQuery(type, *queryInpIter, QueryResultsListener, NULL, qryHandle); client.registerSchemaChangeListener(qryHandle, SchemaListener); queryHandles.push_back(qryHandle); } diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp index a7aafaa06e2..4c8738bc7fa 100644 --- a/contrib/native/client/src/clientlib/drillClient.cpp +++ b/contrib/native/client/src/clientlib/drillClient.cpp @@ -276,15 +276,18 @@ void DrillClient::close() { this->m_pImpl->Close(); } -status_t DrillClient::submitQuery(::exec::shared::QueryType t, const std::string& plan, pfnQueryResultsListener listener, void* listenerCtx, QueryHandle_t* qHandle){ - DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(t, plan, listener, listenerCtx); +status_t DrillClient::submitQuery(Drill::QueryType t, const std::string& plan, pfnQueryResultsListener listener, void* listenerCtx, QueryHandle_t* qHandle){ + + ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); + DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(castedType, plan, listener, listenerCtx); *qHandle=(QueryHandle_t)pResult; return QRY_SUCCESS; } -RecordIterator* DrillClient::submitQuery(::exec::shared::QueryType t, const std::string& plan, DrillClientError* err){ +RecordIterator* DrillClient::submitQuery(Drill::QueryType t, const std::string& plan, DrillClientError* err){ RecordIterator* pIter=NULL; - DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(t, plan, NULL, NULL); + ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); + DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(castedType, plan, NULL, NULL); if(pResult){ pIter=new RecordIterator(pResult); } diff --git a/contrib/native/client/src/clientlib/recordBatch.cpp b/contrib/native/client/src/clientlib/recordBatch.cpp index 4c55f048270..b4bc4724c62 100644 --- a/contrib/native/client/src/clientlib/recordBatch.cpp +++ b/contrib/native/client/src/clientlib/recordBatch.cpp @@ -19,6 +19,7 @@ #include "drill/common.hpp" #include "drill/recordBatch.hpp" #include "utils.hpp" +#include "../protobuf/User.pb.h" const int32_t YEARS_TO_MONTHS=12; const int32_t DAYS_TO_MILLIS=24*60*60*1000; @@ -300,6 +301,18 @@ ret_t FieldBatch::load(){ return RET_SUCCESS; } +RecordBatch::RecordBatch(exec::shared::QueryResult* pResult, ByteBuf_t r, ByteBuf_t b) + :m_fieldDefs(new(std::vector)){ + m_pQueryResult=pResult; + m_pRecordBatchDef=&pResult->def(); + m_numRecords=pResult->row_count(); + m_allocatedBuffer=r; + m_buffer=b; + m_numFields=pResult->def().field_size(); + m_bHasSchemaChanged=false; +} + + RecordBatch::~RecordBatch(){ m_buffer=NULL; //free memory allocated for FieldBatch objects saved in m_fields; @@ -369,6 +382,28 @@ void RecordBatch::print(std::ostream& s, size_t num){ s<field_size(); +} + +bool RecordBatch::isLastChunk() +{ + return m_pQueryResult->is_last_chunk(); +} + + + +void FieldMetadata::set(const exec::shared::SerializedField& f){ + m_name=f.name_part().name(); + m_minorType=f.major_type().minor_type(); + m_dataMode=f.major_type().mode(); + m_valueCount=f.value_count(); + m_scale=f.major_type().scale(); + m_precision=f.major_type().precision(); + m_bufferLength=f.buffer_length(); +} + void DateHolder::load(){ m_year=1970; diff --git a/contrib/native/client/src/include/drill/drillClient.hpp b/contrib/native/client/src/include/drill/drillClient.hpp index d03f88dad0f..6be44581bd3 100644 --- a/contrib/native/client/src/include/drill/drillClient.hpp +++ b/contrib/native/client/src/include/drill/drillClient.hpp @@ -23,7 +23,7 @@ #include #include #include "drill/common.hpp" -#include "drill/protobuf/User.pb.h" +#include "drill/protobuf/types.pb.h" #if defined _WIN32 || defined __CYGWIN__ @@ -44,6 +44,15 @@ #endif #endif +namespace exec +{ + namespace shared + { + class DrillPBError; + enum QueryType; + }; +}; + namespace Drill { //struct UserServerEndPoint; @@ -53,6 +62,12 @@ class FieldMetadata; class RecordBatch; class SchemaDef; + enum QueryType { + SQL = 1, + LOGICAL = 2, + PHYSICAL = 3 + }; + class DECLSPEC_DRILL_CLIENT DrillClientError{ public: static const uint32_t CONN_ERROR_START = 100; @@ -222,16 +237,16 @@ class DECLSPEC_DRILL_CLIENT DrillClient{ void close() ; /* - * Submit a query asynchronously and wait for results to be returned thru a callback. A query context handle is passed + * Submit a query asynchronously and wait for results to be returned through a callback. A query context handle is passed * back. The listener callback will return the handle in the ctx parameter. */ - status_t submitQuery(::exec::shared::QueryType t, const std::string& plan, pfnQueryResultsListener listener, void* listenerCtx, QueryHandle_t* qHandle); + status_t submitQuery(Drill::QueryType t, const std::string& plan, pfnQueryResultsListener listener, void* listenerCtx, QueryHandle_t* qHandle); /* - * Submit a query asynchronously and wait for results to be returned thru an iterator that returns + * Submit a query asynchronously and wait for results to be returned through an iterator that returns * results synchronously. The client app needs to call delete on the iterator when done. */ - RecordIterator* submitQuery(::exec::shared::QueryType t, const std::string& plan, DrillClientError* err); + RecordIterator* submitQuery(Drill::QueryType t, const std::string& plan, DrillClientError* err); /* * The client application should call this function to wait for results if it has registered a diff --git a/contrib/native/client/src/include/drill/drillc.hpp b/contrib/native/client/src/include/drill/drillc.hpp index 93a6b79d321..3697ee8cd15 100644 --- a/contrib/native/client/src/include/drill/drillc.hpp +++ b/contrib/native/client/src/include/drill/drillc.hpp @@ -23,7 +23,6 @@ #include "drill/drillClient.hpp" #include "drill/recordBatch.hpp" #include "drill/protobuf/Types.pb.h" -#include "drill/protobuf/User.pb.h" #endif diff --git a/contrib/native/client/src/include/drill/recordBatch.hpp b/contrib/native/client/src/include/drill/recordBatch.hpp index 9a3df2b6fff..6b6546c8f03 100644 --- a/contrib/native/client/src/include/drill/recordBatch.hpp +++ b/contrib/native/client/src/include/drill/recordBatch.hpp @@ -30,7 +30,8 @@ #include #include "drill/common.hpp" #include "drill/decimalUtils.hpp" -#include "drill/protobuf/User.pb.h" +#include "drill/protobuf/Types.pb.h" + #if defined _WIN32 || defined __CYGWIN__ #ifdef DRILL_CLIENT_EXPORTS @@ -50,6 +51,13 @@ #endif #endif +namespace exec{ + namespace shared { + class SerializedField; + class RecordBatchDef; + class QueryResult; + + };}; namespace Drill { @@ -759,16 +767,9 @@ typedef NullableValueVectorTyped N class DECLSPEC_DRILL_CLIENT FieldMetadata{ public: + FieldMetadata(){}; - void set(const exec::shared::SerializedField& f){ - m_name=f.name_part().name(); - m_minorType=f.major_type().minor_type(); - m_dataMode=f.major_type().mode(); - m_valueCount=f.value_count(); - m_scale=f.major_type().scale(); - m_precision=f.major_type().precision(); - m_bufferLength=f.buffer_length(); - } + void set(const exec::shared::SerializedField& f); const std::string& getName() const{ return m_name;} common::MinorType getMinorType() const{ return m_minorType;} common::DataMode getDataMode() const{return m_dataMode;} @@ -839,16 +840,7 @@ class DECLSPEC_DRILL_CLIENT RecordBatch{ //m_allocatedBuffer is the memory block allocated to hold the incoming RPC message. Record Batches operate on //slices of the allocated buffer. The first slice (the first Field Batch), begins at m_buffer. Data in the //allocated buffer before m_buffer is mostly the RPC header, and the QueryResult object. - RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b) - :m_fieldDefs(new(std::vector)){ - m_pQueryResult=pResult; - m_pRecordBatchDef=&pResult->def(); - m_numRecords=pResult->row_count(); - m_allocatedBuffer=r; - m_buffer=b; - m_numFields=pResult->def().field_size(); - m_bHasSchemaChanged=false; - } + RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b); ~RecordBatch(); @@ -860,8 +852,8 @@ class DECLSPEC_DRILL_CLIENT RecordBatch{ size_t getNumRecords(){ return m_numRecords;} std::vector& getFields(){ return m_fields;} - size_t getNumFields() { return m_pRecordBatchDef->field_size(); } - bool isLastChunk() { return m_pQueryResult->is_last_chunk(); } + size_t getNumFields(); + bool isLastChunk(); boost::shared_ptr > getColumnDefs(){ return m_fieldDefs;} diff --git a/contrib/native/client/src/include/drill/protobuf/User.pb.h b/contrib/native/client/src/protobuf/User.pb.h similarity index 100% rename from contrib/native/client/src/include/drill/protobuf/User.pb.h rename to contrib/native/client/src/protobuf/User.pb.h From 35e29953ab3c7afbc7243acf0a30820e0df6e065 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Tue, 26 Aug 2014 17:48:13 -0700 Subject: [PATCH 7/8] Fixing Protobuf makie file and RecordBatch constructor definition --- contrib/native/client/src/clientlib/recordBatch.cpp | 2 +- contrib/native/client/src/protobuf/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/native/client/src/clientlib/recordBatch.cpp b/contrib/native/client/src/clientlib/recordBatch.cpp index b4bc4724c62..c20f3d6e34c 100644 --- a/contrib/native/client/src/clientlib/recordBatch.cpp +++ b/contrib/native/client/src/clientlib/recordBatch.cpp @@ -301,7 +301,7 @@ ret_t FieldBatch::load(){ return RET_SUCCESS; } -RecordBatch::RecordBatch(exec::shared::QueryResult* pResult, ByteBuf_t r, ByteBuf_t b) +RecordBatch::RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b) :m_fieldDefs(new(std::vector)){ m_pQueryResult=pResult; m_pRecordBatchDef=&pResult->def(); diff --git a/contrib/native/client/src/protobuf/CMakeLists.txt b/contrib/native/client/src/protobuf/CMakeLists.txt index 1d0b3af1b09..06265aa0128 100644 --- a/contrib/native/client/src/protobuf/CMakeLists.txt +++ b/contrib/native/client/src/protobuf/CMakeLists.txt @@ -58,13 +58,13 @@ set (PROTO_CPPHDR_FILES ${PROTO_HDR_DIR}/BitData.pb.h ${PROTO_HDR_DIR}/ExecutionProtos.pb.h ${PROTO_HDR_DIR}/SchemaDef.pb.h + ${PROTO_HDR_DIR}/User.pb.h ) set (PROTO_INC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include/drill/protobuf) set (PROTO_CPPINC_FILES ${PROTO_INC_DIR}/Types.pb.h - ${PROTO_INC_DIR}/User.pb.h ) set(ProtoIncludes ${PROTO_CPPINC_FILES} ) From 799edd130f4249f22baf378b24d5543c157e1a36 Mon Sep 17 00:00:00 2001 From: Alexander Zarei Date: Wed, 27 Aug 2014 14:48:08 -0700 Subject: [PATCH 8/8] Separating DLL creation changes from additional modifications --- .../client/patches/zookeeper-3.4.6-x64.patch | 23 +- contrib/native/client/readme.win.txt | 15 +- .../client/src/clientlib/drillClient.cpp | 4 +- .../client/src/clientlib/drillClientImpl.cpp | 5 + .../client/src/clientlib/drillClientImpl.hpp | 2 - .../client/src/clientlib/recordBatch.cpp | 34 +- .../client/src/include/drill/drillClient.hpp | 20 +- .../src/include/drill/protobuf/Types.pb.h | 5 +- .../client/src/include/drill/recordBatch.hpp | 12 +- .../client/src/protobuf/BitControl.pb.cc | 305 +----------------- .../client/src/protobuf/BitControl.pb.h | 192 +---------- .../native/client/src/protobuf/Types.pb.cc | 10 +- .../client/src/protobuf/UserBitShared.pb.cc | 28 +- .../client/src/protobuf/UserBitShared.pb.h | 7 +- 14 files changed, 97 insertions(+), 565 deletions(-) diff --git a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch index e2b9fee3d60..96f2d103252 100644 --- a/contrib/native/client/patches/zookeeper-3.4.6-x64.patch +++ b/contrib/native/client/patches/zookeeper-3.4.6-x64.patch @@ -1,3 +1,13 @@ +From 64697ddd8a90f29d1693658f04e975e435e3c869 Mon Sep 17 00:00:00 2001 +From: unknown +Date: Thu, 5 Jun 2014 16:40:48 -0700 +Subject: [PATCH] Allow zookeeper to build in x64 + +--- + src/c/include/winstdint.h | 4 ++++ + src/c/src/mt_adaptor.c | 54 +++++++++++++++++++++++------------------------ + 2 files changed, 30 insertions(+), 28 deletions(-) + diff --git a/src/c/include/winstdint.h b/src/c/include/winstdint.h index d02608a..df405f7 100644 --- a/src/c/include/winstdint.h @@ -148,13 +158,6 @@ index 974063f..5ce0fd9 100644 void enter_critical(zhandle_t* zh) { struct adaptor_threads *adaptor = zh->adaptor_priv; -diff --git a/src/c/zookeeper.vcproj b/src/c/zookeeper.vcproj -index dc3ab43..ffc6f90 100644 ---- a/src/c/zookeeper.vcproj -+++ b/src/c/zookeeper.vcproj -@@ -1,4 +1,4 @@ --??? -+ - Configuration Properties->General->Platform Toolset = Windows7.1SDK - If your are running Windows 7 and having problem isntalling windows SDK follow the instructions here http://stackoverflow.com/questions/19366006/error-when-installing-windows-sdk-7-1 + If your are running Windows 7 and having problem isntalling windows SDK follow the instructions here http://stackoverflow.com/questions/19366006/error-when-installing-windows-sdk-7-1 1.3 [Optional] Windows Driver kit The Windows Driver kit is required to get the 64 bit assembler ml64. The @@ -78,8 +78,8 @@ Windows platforms should be more or less similar. 2.1 Boost (version 1.55) a) Download Boost from: i) http://www.boost.org/users/history/version_1_55_0.html - ii) open boost_1_55_0\boost/archive/iterators/transform_width.hpp and add the following to the include statements: #include - iii) Yes somehow this header was not included and has been missed! See here for more info: https://svn.boost.org/trac/boost/ticket/8757 + ii) open boost_1_55_0\boost/archive/iterators/transform_width.hpp and add the following to the include statements: #include + iii) Yes somehow this header was not included and has been missed! See here for more info: https://svn.boost.org/trac/boost/ticket/8757 b) i) Boost 32 bit build - Open a Visual Studio command prompt from the Visual Studio IDE ii) Boost 64 bit build - @@ -116,7 +116,7 @@ Windows platforms should be more or less similar. build. It is safe to ignore it as boost-context is not needed for Drill 2.2 Protobuf (2.5.0) - Get protobuf from here: https://protobuf.googlecode.com/files/protobuf-2.5.0.zip + Get protobuf from here: https://protobuf.googlecode.com/files/protobuf-2.5.0.zip a) Protobuf builds static libraries @@ -125,8 +125,7 @@ Windows platforms should be more or less similar. c) If build for 64 bit, add a 64 bit project configuration for each project. (Make sure the platform toolset is set to Windows7.1SDK) d) Build the protobuf project first (not the solution) - e) Build the solution! - f) If building the solution failed build once more and it will probabley build for the second time!! + e) Build the solution! 2.3 Zookeeper (3.4.6) a) Set the ZOOKEEPER_HOME environment variable @@ -140,9 +139,7 @@ Windows platforms should be more or less similar. ii) Change the output type for the zookeeper project to a static lib Properties->Configuration Properties->General->Configuration Type = Static Library iii) In the cli project add the preprocessor define USE_STATIC_LIB - iv) Build. Build zookeeper lib first, then build cli - v)If building the solution failed build once more and it will probabley build for the second time!! - + iv) Build. Build zookeeper lib first, then build cli 3 Building Drill Clientlib 3.1 SET the following environment variables diff --git a/contrib/native/client/src/clientlib/drillClient.cpp b/contrib/native/client/src/clientlib/drillClient.cpp index 4c8738bc7fa..db7d5a8687f 100644 --- a/contrib/native/client/src/clientlib/drillClient.cpp +++ b/contrib/native/client/src/clientlib/drillClient.cpp @@ -278,7 +278,7 @@ void DrillClient::close() { status_t DrillClient::submitQuery(Drill::QueryType t, const std::string& plan, pfnQueryResultsListener listener, void* listenerCtx, QueryHandle_t* qHandle){ - ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); + ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(castedType, plan, listener, listenerCtx); *qHandle=(QueryHandle_t)pResult; return QRY_SUCCESS; @@ -286,7 +286,7 @@ status_t DrillClient::submitQuery(Drill::QueryType t, const std::string& plan, p RecordIterator* DrillClient::submitQuery(Drill::QueryType t, const std::string& plan, DrillClientError* err){ RecordIterator* pIter=NULL; - ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); + ::exec::shared::QueryType castedType = static_cast<::exec::shared::QueryType> (t); DrillClientQueryResult* pResult=this->m_pImpl->SubmitQuery(castedType, plan, NULL, NULL); if(pResult){ pIter=new RecordIterator(pResult); diff --git a/contrib/native/client/src/clientlib/drillClientImpl.cpp b/contrib/native/client/src/clientlib/drillClientImpl.cpp index 67123e3a338..ff9e419adef 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.cpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.cpp @@ -25,6 +25,11 @@ #include #include #include +#ifdef _WIN32 +#include +#else +#include +#endif #include "drill/drillClient.hpp" #include "drill/recordBatch.hpp" diff --git a/contrib/native/client/src/clientlib/drillClientImpl.hpp b/contrib/native/client/src/clientlib/drillClientImpl.hpp index 69c5212f4ed..22574374bd5 100644 --- a/contrib/native/client/src/clientlib/drillClientImpl.hpp +++ b/contrib/native/client/src/clientlib/drillClientImpl.hpp @@ -37,9 +37,7 @@ #include #include #ifdef _WIN32 -extern "C"{ #include -} #else #include #endif diff --git a/contrib/native/client/src/clientlib/recordBatch.cpp b/contrib/native/client/src/clientlib/recordBatch.cpp index c20f3d6e34c..50afb19e36b 100644 --- a/contrib/native/client/src/clientlib/recordBatch.cpp +++ b/contrib/native/client/src/clientlib/recordBatch.cpp @@ -302,14 +302,14 @@ ret_t FieldBatch::load(){ } RecordBatch::RecordBatch(exec::shared::QueryResult* pResult, AllocatedBufferPtr r, ByteBuf_t b) - :m_fieldDefs(new(std::vector)){ - m_pQueryResult=pResult; - m_pRecordBatchDef=&pResult->def(); - m_numRecords=pResult->row_count(); - m_allocatedBuffer=r; - m_buffer=b; - m_numFields=pResult->def().field_size(); - m_bHasSchemaChanged=false; + :m_fieldDefs(new(std::vector)){ + m_pQueryResult=pResult; + m_pRecordBatchDef=&pResult->def(); + m_numRecords=pResult->row_count(); + m_allocatedBuffer=r; + m_buffer=b; + m_numFields=pResult->def().field_size(); + m_bHasSchemaChanged=false; } @@ -384,24 +384,24 @@ void RecordBatch::print(std::ostream& s, size_t num){ } size_t RecordBatch::getNumFields() { - return m_pRecordBatchDef->field_size(); + return m_pRecordBatchDef->field_size(); } bool RecordBatch::isLastChunk() { - return m_pQueryResult->is_last_chunk(); + return m_pQueryResult->is_last_chunk(); } void FieldMetadata::set(const exec::shared::SerializedField& f){ - m_name=f.name_part().name(); - m_minorType=f.major_type().minor_type(); - m_dataMode=f.major_type().mode(); - m_valueCount=f.value_count(); - m_scale=f.major_type().scale(); - m_precision=f.major_type().precision(); - m_bufferLength=f.buffer_length(); + m_name=f.name_part().name(); + m_minorType=f.major_type().minor_type(); + m_dataMode=f.major_type().mode(); + m_valueCount=f.value_count(); + m_scale=f.major_type().scale(); + m_precision=f.major_type().precision(); + m_bufferLength=f.buffer_length(); } diff --git a/contrib/native/client/src/include/drill/drillClient.hpp b/contrib/native/client/src/include/drill/drillClient.hpp index 6be44581bd3..d9c4dd6fba3 100644 --- a/contrib/native/client/src/include/drill/drillClient.hpp +++ b/contrib/native/client/src/include/drill/drillClient.hpp @@ -46,11 +46,11 @@ namespace exec { - namespace shared - { - class DrillPBError; - enum QueryType; - }; + namespace shared + { + class DrillPBError; + enum QueryType; + }; }; namespace Drill { @@ -62,11 +62,11 @@ class FieldMetadata; class RecordBatch; class SchemaDef; - enum QueryType { - SQL = 1, - LOGICAL = 2, - PHYSICAL = 3 - }; + enum QueryType { + SQL = 1, + LOGICAL = 2, + PHYSICAL = 3 + }; class DECLSPEC_DRILL_CLIENT DrillClientError{ public: diff --git a/contrib/native/client/src/include/drill/protobuf/Types.pb.h b/contrib/native/client/src/include/drill/protobuf/Types.pb.h index b26177c44ba..853415317e0 100644 --- a/contrib/native/client/src/include/drill/protobuf/Types.pb.h +++ b/contrib/native/client/src/include/drill/protobuf/Types.pb.h @@ -72,12 +72,11 @@ enum MinorType { DM_UNKNOWN = 37, INTERVALYEAR = 38, INTERVALDAY = 39, - LIST = 40, - GENERIC_OBJECT = 41 + LIST = 40 }; bool MinorType_IsValid(int value); const MinorType MinorType_MIN = LATE; -const MinorType MinorType_MAX = GENERIC_OBJECT; +const MinorType MinorType_MAX = LIST; const int MinorType_ARRAYSIZE = MinorType_MAX + 1; const ::google::protobuf::EnumDescriptor* MinorType_descriptor(); diff --git a/contrib/native/client/src/include/drill/recordBatch.hpp b/contrib/native/client/src/include/drill/recordBatch.hpp index 6b6546c8f03..5eca7cb3874 100644 --- a/contrib/native/client/src/include/drill/recordBatch.hpp +++ b/contrib/native/client/src/include/drill/recordBatch.hpp @@ -52,12 +52,12 @@ #endif namespace exec{ - namespace shared { - class SerializedField; - class RecordBatchDef; - class QueryResult; - - };}; + namespace shared { + class SerializedField; + class RecordBatchDef; + class QueryResult; + }; +}; namespace Drill { diff --git a/contrib/native/client/src/protobuf/BitControl.pb.cc b/contrib/native/client/src/protobuf/BitControl.pb.cc index a7623861e2a..4f8749cee2c 100644 --- a/contrib/native/client/src/protobuf/BitControl.pb.cc +++ b/contrib/native/client/src/protobuf/BitControl.pb.cc @@ -37,9 +37,6 @@ const ::google::protobuf::internal::GeneratedMessageReflection* const ::google::protobuf::Descriptor* WorkQueueStatus_descriptor_ = NULL; const ::google::protobuf::internal::GeneratedMessageReflection* WorkQueueStatus_reflection_ = NULL; -const ::google::protobuf::Descriptor* FinishedReceiver_descriptor_ = NULL; -const ::google::protobuf::internal::GeneratedMessageReflection* - FinishedReceiver_reflection_ = NULL; const ::google::protobuf::EnumDescriptor* RpcType_descriptor_ = NULL; } // namespace @@ -145,22 +142,6 @@ void protobuf_AssignDesc_BitControl_2eproto() { ::google::protobuf::DescriptorPool::generated_pool(), ::google::protobuf::MessageFactory::generated_factory(), sizeof(WorkQueueStatus)); - FinishedReceiver_descriptor_ = file->message_type(5); - static const int FinishedReceiver_offsets_[2] = { - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, receiver_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, sender_), - }; - FinishedReceiver_reflection_ = - new ::google::protobuf::internal::GeneratedMessageReflection( - FinishedReceiver_descriptor_, - FinishedReceiver::default_instance_, - FinishedReceiver_offsets_, - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _has_bits_[0]), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FinishedReceiver, _unknown_fields_), - -1, - ::google::protobuf::DescriptorPool::generated_pool(), - ::google::protobuf::MessageFactory::generated_factory(), - sizeof(FinishedReceiver)); RpcType_descriptor_ = file->enum_type(0); } @@ -184,8 +165,6 @@ void protobuf_RegisterTypes(const ::std::string&) { PlanFragment_descriptor_, &PlanFragment::default_instance()); ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( WorkQueueStatus_descriptor_, &WorkQueueStatus::default_instance()); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( - FinishedReceiver_descriptor_, &FinishedReceiver::default_instance()); } } // namespace @@ -201,8 +180,6 @@ void protobuf_ShutdownFile_BitControl_2eproto() { delete PlanFragment_reflection_; delete WorkQueueStatus::default_instance_; delete WorkQueueStatus_reflection_; - delete FinishedReceiver::default_instance_; - delete FinishedReceiver_reflection_; } void protobuf_AddDesc_BitControl_2eproto() { @@ -239,18 +216,15 @@ void protobuf_AddDesc_BitControl_2eproto() { "e_zone\030\020 \001(\005\022\024\n\014options_json\030\021 \001(\t\"f\n\017Wo" "rkQueueStatus\022(\n\010endpoint\030\001 \001(\0132\026.exec.D" "rillbitEndpoint\022\024\n\014queue_length\030\002 \001(\005\022\023\n" - "\013report_time\030\003 \001(\003\"h\n\020FinishedReceiver\022*" - "\n\010receiver\030\001 \001(\0132\030.exec.bit.FragmentHand" - "le\022(\n\006sender\030\002 \001(\0132\030.exec.bit.FragmentHa" - "ndle*\242\002\n\007RpcType\022\r\n\tHANDSHAKE\020\000\022\007\n\003ACK\020\001" - "\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIATILIZE_FRAGMENT" - "\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020\006\022\031\n\025REQ_RECEI" - "VER_FINISHED\020\007\022\027\n\023REQ_FRAGMENT_STATUS\020\010\022" - "\022\n\016REQ_BIT_STATUS\020\t\022\024\n\020REQ_QUERY_STATUS\020" - "\n\022\030\n\024RESP_FRAGMENT_HANDLE\020\013\022\030\n\024RESP_FRAG" - "MENT_STATUS\020\014\022\023\n\017RESP_BIT_STATUS\020\r\022\025\n\021RE" - "SP_QUERY_STATUS\020\016B+\n\033org.apache.drill.ex" - "ec.protoB\nBitControlH\001", 1422); + "\013report_time\030\003 \001(\003*\207\002\n\007RpcType\022\r\n\tHANDSH" + "AKE\020\000\022\007\n\003ACK\020\001\022\013\n\007GOODBYE\020\002\022\033\n\027REQ_INIAT" + "ILIZE_FRAGMENT\020\003\022\027\n\023REQ_CANCEL_FRAGMENT\020" + "\006\022\027\n\023REQ_FRAGMENT_STATUS\020\007\022\022\n\016REQ_BIT_ST" + "ATUS\020\010\022\024\n\020REQ_QUERY_STATUS\020\t\022\030\n\024RESP_FRA" + "GMENT_HANDLE\020\n\022\030\n\024RESP_FRAGMENT_STATUS\020\013" + "\022\023\n\017RESP_BIT_STATUS\020\014\022\025\n\021RESP_QUERY_STAT" + "US\020\rB+\n\033org.apache.drill.exec.protoB\nBit" + "ControlH\001", 1289); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "BitControl.proto", &protobuf_RegisterTypes); BitControlHandshake::default_instance_ = new BitControlHandshake(); @@ -258,13 +232,11 @@ void protobuf_AddDesc_BitControl_2eproto() { FragmentStatus::default_instance_ = new FragmentStatus(); PlanFragment::default_instance_ = new PlanFragment(); WorkQueueStatus::default_instance_ = new WorkQueueStatus(); - FinishedReceiver::default_instance_ = new FinishedReceiver(); BitControlHandshake::default_instance_->InitAsDefaultInstance(); BitStatus::default_instance_->InitAsDefaultInstance(); FragmentStatus::default_instance_->InitAsDefaultInstance(); PlanFragment::default_instance_->InitAsDefaultInstance(); WorkQueueStatus::default_instance_->InitAsDefaultInstance(); - FinishedReceiver::default_instance_->InitAsDefaultInstance(); ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_BitControl_2eproto); } @@ -292,7 +264,6 @@ bool RpcType_IsValid(int value) { case 11: case 12: case 13: - case 14: return true; default: return false; @@ -2171,264 +2142,6 @@ ::google::protobuf::Metadata WorkQueueStatus::GetMetadata() const { } -// =================================================================== - -#ifndef _MSC_VER -const int FinishedReceiver::kReceiverFieldNumber; -const int FinishedReceiver::kSenderFieldNumber; -#endif // !_MSC_VER - -FinishedReceiver::FinishedReceiver() - : ::google::protobuf::Message() { - SharedCtor(); -} - -void FinishedReceiver::InitAsDefaultInstance() { - receiver_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance()); - sender_ = const_cast< ::exec::bit::FragmentHandle*>(&::exec::bit::FragmentHandle::default_instance()); -} - -FinishedReceiver::FinishedReceiver(const FinishedReceiver& from) - : ::google::protobuf::Message() { - SharedCtor(); - MergeFrom(from); -} - -void FinishedReceiver::SharedCtor() { - _cached_size_ = 0; - receiver_ = NULL; - sender_ = NULL; - ::memset(_has_bits_, 0, sizeof(_has_bits_)); -} - -FinishedReceiver::~FinishedReceiver() { - SharedDtor(); -} - -void FinishedReceiver::SharedDtor() { - if (this != default_instance_) { - delete receiver_; - delete sender_; - } -} - -void FinishedReceiver::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* FinishedReceiver::descriptor() { - protobuf_AssignDescriptorsOnce(); - return FinishedReceiver_descriptor_; -} - -const FinishedReceiver& FinishedReceiver::default_instance() { - if (default_instance_ == NULL) protobuf_AddDesc_BitControl_2eproto(); - return *default_instance_; -} - -FinishedReceiver* FinishedReceiver::default_instance_ = NULL; - -FinishedReceiver* FinishedReceiver::New() const { - return new FinishedReceiver; -} - -void FinishedReceiver::Clear() { - if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { - if (has_receiver()) { - if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear(); - } - if (has_sender()) { - if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear(); - } - } - ::memset(_has_bits_, 0, sizeof(_has_bits_)); - mutable_unknown_fields()->Clear(); -} - -bool FinishedReceiver::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!(EXPRESSION)) return false - ::google::protobuf::uint32 tag; - while ((tag = input->ReadTag()) != 0) { - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // optional .exec.bit.FragmentHandle receiver = 1; - case 1: { - if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, mutable_receiver())); - } else { - goto handle_uninterpreted; - } - if (input->ExpectTag(18)) goto parse_sender; - break; - } - - // optional .exec.bit.FragmentHandle sender = 2; - case 2: { - if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - parse_sender: - DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, mutable_sender())); - } else { - goto handle_uninterpreted; - } - if (input->ExpectAtEnd()) return true; - break; - } - - default: { - handle_uninterpreted: - if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == - ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { - return true; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, mutable_unknown_fields())); - break; - } - } - } - return true; -#undef DO_ -} - -void FinishedReceiver::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // optional .exec.bit.FragmentHandle receiver = 1; - if (has_receiver()) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 1, this->receiver(), output); - } - - // optional .exec.bit.FragmentHandle sender = 2; - if (has_sender()) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, this->sender(), output); - } - - if (!unknown_fields().empty()) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - unknown_fields(), output); - } -} - -::google::protobuf::uint8* FinishedReceiver::SerializeWithCachedSizesToArray( - ::google::protobuf::uint8* target) const { - // optional .exec.bit.FragmentHandle receiver = 1; - if (has_receiver()) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteMessageNoVirtualToArray( - 1, this->receiver(), target); - } - - // optional .exec.bit.FragmentHandle sender = 2; - if (has_sender()) { - target = ::google::protobuf::internal::WireFormatLite:: - WriteMessageNoVirtualToArray( - 2, this->sender(), target); - } - - if (!unknown_fields().empty()) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - unknown_fields(), target); - } - return target; -} - -int FinishedReceiver::ByteSize() const { - int total_size = 0; - - if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { - // optional .exec.bit.FragmentHandle receiver = 1; - if (has_receiver()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->receiver()); - } - - // optional .exec.bit.FragmentHandle sender = 2; - if (has_sender()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->sender()); - } - - } - if (!unknown_fields().empty()) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - unknown_fields()); - } - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = total_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void FinishedReceiver::MergeFrom(const ::google::protobuf::Message& from) { - GOOGLE_CHECK_NE(&from, this); - const FinishedReceiver* source = - ::google::protobuf::internal::dynamic_cast_if_available( - &from); - if (source == NULL) { - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - MergeFrom(*source); - } -} - -void FinishedReceiver::MergeFrom(const FinishedReceiver& from) { - GOOGLE_CHECK_NE(&from, this); - if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { - if (from.has_receiver()) { - mutable_receiver()->::exec::bit::FragmentHandle::MergeFrom(from.receiver()); - } - if (from.has_sender()) { - mutable_sender()->::exec::bit::FragmentHandle::MergeFrom(from.sender()); - } - } - mutable_unknown_fields()->MergeFrom(from.unknown_fields()); -} - -void FinishedReceiver::CopyFrom(const ::google::protobuf::Message& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void FinishedReceiver::CopyFrom(const FinishedReceiver& from) { - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool FinishedReceiver::IsInitialized() const { - - return true; -} - -void FinishedReceiver::Swap(FinishedReceiver* other) { - if (other != this) { - std::swap(receiver_, other->receiver_); - std::swap(sender_, other->sender_); - std::swap(_has_bits_[0], other->_has_bits_[0]); - _unknown_fields_.Swap(&other->_unknown_fields_); - std::swap(_cached_size_, other->_cached_size_); - } -} - -::google::protobuf::Metadata FinishedReceiver::GetMetadata() const { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::Metadata metadata; - metadata.descriptor = FinishedReceiver_descriptor_; - metadata.reflection = FinishedReceiver_reflection_; - return metadata; -} - - // @@protoc_insertion_point(namespace_scope) } // namespace control diff --git a/contrib/native/client/src/protobuf/BitControl.pb.h b/contrib/native/client/src/protobuf/BitControl.pb.h index ba171a67faf..06af929d8f4 100644 --- a/contrib/native/client/src/protobuf/BitControl.pb.h +++ b/contrib/native/client/src/protobuf/BitControl.pb.h @@ -44,7 +44,6 @@ class BitStatus; class FragmentStatus; class PlanFragment; class WorkQueueStatus; -class FinishedReceiver; enum RpcType { HANDSHAKE = 0, @@ -52,14 +51,13 @@ enum RpcType { GOODBYE = 2, REQ_INIATILIZE_FRAGMENT = 3, REQ_CANCEL_FRAGMENT = 6, - REQ_RECEIVER_FINISHED = 7, - REQ_FRAGMENT_STATUS = 8, - REQ_BIT_STATUS = 9, - REQ_QUERY_STATUS = 10, - RESP_FRAGMENT_HANDLE = 11, - RESP_FRAGMENT_STATUS = 12, - RESP_BIT_STATUS = 13, - RESP_QUERY_STATUS = 14 + REQ_FRAGMENT_STATUS = 7, + REQ_BIT_STATUS = 8, + REQ_QUERY_STATUS = 9, + RESP_FRAGMENT_HANDLE = 10, + RESP_FRAGMENT_STATUS = 11, + RESP_BIT_STATUS = 12, + RESP_QUERY_STATUS = 13 }; bool RpcType_IsValid(int value); const RpcType RpcType_MIN = HANDSHAKE; @@ -705,102 +703,6 @@ class WorkQueueStatus : public ::google::protobuf::Message { void InitAsDefaultInstance(); static WorkQueueStatus* default_instance_; }; -// ------------------------------------------------------------------- - -class FinishedReceiver : public ::google::protobuf::Message { - public: - FinishedReceiver(); - virtual ~FinishedReceiver(); - - FinishedReceiver(const FinishedReceiver& from); - - inline FinishedReceiver& operator=(const FinishedReceiver& from) { - CopyFrom(from); - return *this; - } - - inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const { - return _unknown_fields_; - } - - inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() { - return &_unknown_fields_; - } - - static const ::google::protobuf::Descriptor* descriptor(); - static const FinishedReceiver& default_instance(); - - void Swap(FinishedReceiver* other); - - // implements Message ---------------------------------------------- - - FinishedReceiver* New() const; - void CopyFrom(const ::google::protobuf::Message& from); - void MergeFrom(const ::google::protobuf::Message& from); - void CopyFrom(const FinishedReceiver& from); - void MergeFrom(const FinishedReceiver& from); - void Clear(); - bool IsInitialized() const; - - int ByteSize() const; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input); - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const; - ::google::protobuf::uint8* SerializeWithCachedSizesToArray(::google::protobuf::uint8* output) const; - int GetCachedSize() const { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const; - public: - - ::google::protobuf::Metadata GetMetadata() const; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // optional .exec.bit.FragmentHandle receiver = 1; - inline bool has_receiver() const; - inline void clear_receiver(); - static const int kReceiverFieldNumber = 1; - inline const ::exec::bit::FragmentHandle& receiver() const; - inline ::exec::bit::FragmentHandle* mutable_receiver(); - inline ::exec::bit::FragmentHandle* release_receiver(); - inline void set_allocated_receiver(::exec::bit::FragmentHandle* receiver); - - // optional .exec.bit.FragmentHandle sender = 2; - inline bool has_sender() const; - inline void clear_sender(); - static const int kSenderFieldNumber = 2; - inline const ::exec::bit::FragmentHandle& sender() const; - inline ::exec::bit::FragmentHandle* mutable_sender(); - inline ::exec::bit::FragmentHandle* release_sender(); - inline void set_allocated_sender(::exec::bit::FragmentHandle* sender); - - // @@protoc_insertion_point(class_scope:exec.bit.control.FinishedReceiver) - private: - inline void set_has_receiver(); - inline void clear_has_receiver(); - inline void set_has_sender(); - inline void clear_has_sender(); - - ::google::protobuf::UnknownFieldSet _unknown_fields_; - - ::exec::bit::FragmentHandle* receiver_; - ::exec::bit::FragmentHandle* sender_; - - mutable int _cached_size_; - ::google::protobuf::uint32 _has_bits_[(2 + 31) / 32]; - - friend void protobuf_AddDesc_BitControl_2eproto(); - friend void protobuf_AssignDesc_BitControl_2eproto(); - friend void protobuf_ShutdownFile_BitControl_2eproto(); - - void InitAsDefaultInstance(); - static FinishedReceiver* default_instance_; -}; // =================================================================== @@ -1580,86 +1482,6 @@ inline void WorkQueueStatus::set_report_time(::google::protobuf::int64 value) { report_time_ = value; } -// ------------------------------------------------------------------- - -// FinishedReceiver - -// optional .exec.bit.FragmentHandle receiver = 1; -inline bool FinishedReceiver::has_receiver() const { - return (_has_bits_[0] & 0x00000001u) != 0; -} -inline void FinishedReceiver::set_has_receiver() { - _has_bits_[0] |= 0x00000001u; -} -inline void FinishedReceiver::clear_has_receiver() { - _has_bits_[0] &= ~0x00000001u; -} -inline void FinishedReceiver::clear_receiver() { - if (receiver_ != NULL) receiver_->::exec::bit::FragmentHandle::Clear(); - clear_has_receiver(); -} -inline const ::exec::bit::FragmentHandle& FinishedReceiver::receiver() const { - return receiver_ != NULL ? *receiver_ : *default_instance_->receiver_; -} -inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_receiver() { - set_has_receiver(); - if (receiver_ == NULL) receiver_ = new ::exec::bit::FragmentHandle; - return receiver_; -} -inline ::exec::bit::FragmentHandle* FinishedReceiver::release_receiver() { - clear_has_receiver(); - ::exec::bit::FragmentHandle* temp = receiver_; - receiver_ = NULL; - return temp; -} -inline void FinishedReceiver::set_allocated_receiver(::exec::bit::FragmentHandle* receiver) { - delete receiver_; - receiver_ = receiver; - if (receiver) { - set_has_receiver(); - } else { - clear_has_receiver(); - } -} - -// optional .exec.bit.FragmentHandle sender = 2; -inline bool FinishedReceiver::has_sender() const { - return (_has_bits_[0] & 0x00000002u) != 0; -} -inline void FinishedReceiver::set_has_sender() { - _has_bits_[0] |= 0x00000002u; -} -inline void FinishedReceiver::clear_has_sender() { - _has_bits_[0] &= ~0x00000002u; -} -inline void FinishedReceiver::clear_sender() { - if (sender_ != NULL) sender_->::exec::bit::FragmentHandle::Clear(); - clear_has_sender(); -} -inline const ::exec::bit::FragmentHandle& FinishedReceiver::sender() const { - return sender_ != NULL ? *sender_ : *default_instance_->sender_; -} -inline ::exec::bit::FragmentHandle* FinishedReceiver::mutable_sender() { - set_has_sender(); - if (sender_ == NULL) sender_ = new ::exec::bit::FragmentHandle; - return sender_; -} -inline ::exec::bit::FragmentHandle* FinishedReceiver::release_sender() { - clear_has_sender(); - ::exec::bit::FragmentHandle* temp = sender_; - sender_ = NULL; - return temp; -} -inline void FinishedReceiver::set_allocated_sender(::exec::bit::FragmentHandle* sender) { - delete sender_; - sender_ = sender; - if (sender) { - set_has_sender(); - } else { - clear_has_sender(); - } -} - // @@protoc_insertion_point(namespace_scope) diff --git a/contrib/native/client/src/protobuf/Types.pb.cc b/contrib/native/client/src/protobuf/Types.pb.cc index a14f40453fb..c7fa2eaaff8 100644 --- a/contrib/native/client/src/protobuf/Types.pb.cc +++ b/contrib/native/client/src/protobuf/Types.pb.cc @@ -91,7 +91,7 @@ void protobuf_AddDesc_Types_2eproto() { "inor_type\030\001 \001(\0162\021.common.MinorType\022\036\n\004mo" "de\030\002 \001(\0162\020.common.DataMode\022\r\n\005width\030\003 \001(" "\005\022\021\n\tprecision\030\004 \001(\005\022\r\n\005scale\030\005 \001(\005\022\020\n\010t" - "imeZone\030\006 \001(\005*\220\004\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" + "imeZone\030\006 \001(\005*\374\003\n\tMinorType\022\010\n\004LATE\020\000\022\007\n" "\003MAP\020\001\022\013\n\007TINYINT\020\003\022\014\n\010SMALLINT\020\004\022\007\n\003INT" "\020\005\022\n\n\006BIGINT\020\006\022\014\n\010DECIMAL9\020\007\022\r\n\tDECIMAL1" "8\020\010\022\023\n\017DECIMAL28SPARSE\020\t\022\023\n\017DECIMAL38SPA" @@ -104,10 +104,9 @@ void protobuf_AddDesc_Types_2eproto() { "\036\022\t\n\005UINT4\020\037\022\t\n\005UINT8\020 \022\022\n\016DECIMAL28DENS" "E\020!\022\022\n\016DECIMAL38DENSE\020\"\022\016\n\nDM_UNKNOWN\020%\022" "\020\n\014INTERVALYEAR\020&\022\017\n\013INTERVALDAY\020\'\022\010\n\004LI" - "ST\020(\022\022\n\016GENERIC_OBJECT\020)*=\n\010DataMode\022\017\n\013" - "DM_OPTIONAL\020\000\022\017\n\013DM_REQUIRED\020\001\022\017\n\013DM_REP" - "EATED\020\002B-\n\035org.apache.drill.common.types" - "B\nTypeProtosH\001", 814); + "ST\020(*=\n\010DataMode\022\017\n\013DM_OPTIONAL\020\000\022\017\n\013DM_" + "REQUIRED\020\001\022\017\n\013DM_REPEATED\020\002B-\n\035org.apach" + "e.drill.common.typesB\nTypeProtosH\001", 794); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "Types.proto", &protobuf_RegisterTypes); MajorType::default_instance_ = new MajorType(); @@ -163,7 +162,6 @@ bool MinorType_IsValid(int value) { case 38: case 39: case 40: - case 41: return true; default: return false; diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc index 0e7649f2717..88507e5a303 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc @@ -532,25 +532,24 @@ void protobuf_AddDesc_UserBitShared_2eproto() { "\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003*k\n\rFragmen" "tState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATI" "ON\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCE" - "LLED\020\004\022\n\n\006FAILED\020\005*\224\005\n\020CoreOperatorType\022" + "LLED\020\004\022\n\n\006FAILED\020\005*\372\004\n\020CoreOperatorType\022" "\021\n\rSINGLE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001" "\022\n\n\006FILTER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH" "_JOIN\020\004\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITIO" "N_SENDER\020\006\022\t\n\005LIMIT\020\007\022\024\n\020MERGING_RECEIVE" "R\020\010\022\034\n\030ORDERED_PARTITION_SENDER\020\t\022\013\n\007PRO" - "JECT\020\n\022\026\n\022UNORDERED_RECEIVER\020\013\022\020\n\014RANGE_" - "SENDER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR" - "_REMOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nT" - "OP_N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020" - "\022\022\t\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_RO" - "W_GROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYS" - "TEM_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016P" - "ARQUET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013" - "TEXT_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON" - "_SUB_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n" - "\017COMPLEX_TO_JSON\020\037\022\025\n\021PRODUCER_CONSUMER\020" - " B.\n\033org.apache.drill.exec.protoB\rUserBi" - "tSharedH\001", 3569); + "JECT\020\n\022\023\n\017RANDOM_RECEIVER\020\013\022\020\n\014RANGE_SEN" + "DER\020\014\022\n\n\006SCREEN\020\r\022\034\n\030SELECTION_VECTOR_RE" + "MOVER\020\016\022\027\n\023STREAMING_AGGREGATE\020\017\022\016\n\nTOP_" + "N_SORT\020\020\022\021\n\rEXTERNAL_SORT\020\021\022\t\n\005TRACE\020\022\022\t" + "\n\005UNION\020\023\022\014\n\010OLD_SORT\020\024\022\032\n\026PARQUET_ROW_G" + "ROUP_SCAN\020\025\022\021\n\rHIVE_SUB_SCAN\020\026\022\025\n\021SYSTEM" + "_TABLE_SCAN\020\027\022\021\n\rMOCK_SUB_SCAN\020\030\022\022\n\016PARQ" + "UET_WRITER\020\031\022\023\n\017DIRECT_SUB_SCAN\020\032\022\017\n\013TEX" + "T_WRITER\020\033\022\021\n\rTEXT_SUB_SCAN\020\034\022\021\n\rJSON_SU" + "B_SCAN\020\035\022\030\n\024INFO_SCHEMA_SUB_SCAN\020\036\022\023\n\017CO" + "MPLEX_TO_JSON\020\037B.\n\033org.apache.drill.exec" + ".protoB\rUserBitSharedH\001", 3543); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "UserBitShared.proto", &protobuf_RegisterTypes); UserCredentials::default_instance_ = new UserCredentials(); @@ -678,7 +677,6 @@ bool CoreOperatorType_IsValid(int value) { case 29: case 30: case 31: - case 32: return true; default: return false; diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h index e7d40b19d1f..6ef17c38f27 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.h +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h @@ -171,7 +171,7 @@ enum CoreOperatorType { MERGING_RECEIVER = 8, ORDERED_PARTITION_SENDER = 9, PROJECT = 10, - UNORDERED_RECEIVER = 11, + RANDOM_RECEIVER = 11, RANGE_SENDER = 12, SCREEN = 13, SELECTION_VECTOR_REMOVER = 14, @@ -191,12 +191,11 @@ enum CoreOperatorType { TEXT_SUB_SCAN = 28, JSON_SUB_SCAN = 29, INFO_SCHEMA_SUB_SCAN = 30, - COMPLEX_TO_JSON = 31, - PRODUCER_CONSUMER = 32 + COMPLEX_TO_JSON = 31 }; bool CoreOperatorType_IsValid(int value); const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER; -const CoreOperatorType CoreOperatorType_MAX = PRODUCER_CONSUMER; +const CoreOperatorType CoreOperatorType_MAX = COMPLEX_TO_JSON; const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1; const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor();