From d7ff35a3f03485d8652acc0406f9346bef89851f Mon Sep 17 00:00:00 2001 From: Nobuaki Sukegawa Date: Tue, 1 Mar 2016 01:41:47 +0900 Subject: [PATCH 1/4] THRIFT-3699 Fix integer limit symbol includes in Python C extension --- lib/py/src/ext/module.cpp | 28 ++++++++++------------------ lib/py/src/ext/protocol.h | 8 ++++++-- lib/py/src/ext/protocol.tcc | 16 +++++++++++----- lib/py/src/ext/types.h | 5 +++++ 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/lib/py/src/ext/module.cpp b/lib/py/src/ext/module.cpp index 5ffc155357d..34ec7f624e7 100644 --- a/lib/py/src/ext/module.cpp +++ b/lib/py/src/ext/module.cpp @@ -21,8 +21,8 @@ #include "types.h" #include "binary.h" #include "compact.h" +#include #include -#include // TODO(dreiss): defval appears to be unused. Look into removing it. // TODO(dreiss): Make parse_spec_args recursive, and cache the output @@ -87,11 +87,18 @@ static PyObject* decode_impl(PyObject* args) { } T protocol; +#ifdef _MSC_VER + // workaround strange VC++ 2015 bug where #else path does not compile + int32_t default_limit = INT32_MAX; +#else + int32_t default_limit = std::numeric_limits::max(); +#endif protocol.setStringLengthLimit( - as_long_then_delete(PyObject_GetAttr(oprot, INTERN_STRING(string_length_limit)), INT32_MAX)); + as_long_then_delete(PyObject_GetAttr(oprot, INTERN_STRING(string_length_limit)), + default_limit)); protocol.setContainerLengthLimit( as_long_then_delete(PyObject_GetAttr(oprot, INTERN_STRING(container_length_limit)), - INT32_MAX)); + default_limit)); ScopedPyObject transport(PyObject_GetAttr(oprot, INTERN_STRING(trans))); if (!transport) { return NULL; @@ -170,21 +177,6 @@ void initfastbinary() { #endif - const rlim_t kStackSize = 16 * 1024 * 1024; // min stack size = 16 MB - struct rlimit rl; - int result; - - result = getrlimit(RLIMIT_STACK, &rl); - if (result == 0) { - if (rl.rlim_cur < kStackSize) { - rl.rlim_cur = kStackSize; - result = setrlimit(RLIMIT_STACK, &rl); - if (result != 0) { - fprintf(stderr, "setrlimit returned result = %d\n", result); - } - } - } - #define INIT_INTERN_STRING(value) \ do { \ INTERN_STRING(value) = PyString_InternFromString(#value); \ diff --git a/lib/py/src/ext/protocol.h b/lib/py/src/ext/protocol.h index bf3a6b8f43d..126dbc37705 100644 --- a/lib/py/src/ext/protocol.h +++ b/lib/py/src/ext/protocol.h @@ -21,6 +21,8 @@ #define THRIFT_PY_PROTOCOL_H #include "ext/types.h" +#include +#include namespace apache { namespace thrift { @@ -30,14 +32,16 @@ template class ProtocolBase { public: - ProtocolBase() : stringLimit_(INT32_MAX), containerLimit_(INT32_MAX), output_(NULL) {} + ProtocolBase() + : stringLimit_(std::numeric_limits::max()), + containerLimit_(std::numeric_limits::max()), + output_(NULL) {} inline virtual ~ProtocolBase(); bool prepareDecodeBufferFromTransport(PyObject* trans); PyObject* readStruct(PyObject* output, PyObject* klass, PyObject* spec_seq); - bool prepareEncodeBuffer(); bool encodeValue(PyObject* value, TType type, PyObject* typeargs); diff --git a/lib/py/src/ext/protocol.tcc b/lib/py/src/ext/protocol.tcc index 9c258419793..2f0d083978a 100644 --- a/lib/py/src/ext/protocol.tcc +++ b/lib/py/src/ext/protocol.tcc @@ -20,6 +20,8 @@ #ifndef THRIFT_PY_PROTOCOL_TCC #define THRIFT_PY_PROTOCOL_TCC +#include + #define CHECK_RANGE(v, min, max) (((v) <= (max)) && ((v) >= (min))) #define INIT_OUTBUF_SIZE 128 @@ -210,7 +212,7 @@ inline bool check_ssize_t_32(Py_ssize_t len) { if (INT_CONV_ERROR_OCCURRED(len)) { return false; } - if (!CHECK_RANGE(len, 0, INT32_MAX)) { + if (!CHECK_RANGE(len, 0, std::numeric_limits::max())) { PyErr_SetString(PyExc_OverflowError, "size out of range: exceeded INT32_MAX"); return false; } @@ -358,7 +360,8 @@ bool ProtocolBase::encodeValue(PyObject* value, TType type, PyObject* type case T_I08: { int8_t val; - if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + if (!parse_pyint(value, &val, std::numeric_limits::min(), + std::numeric_limits::max())) { return false; } @@ -368,7 +371,8 @@ bool ProtocolBase::encodeValue(PyObject* value, TType type, PyObject* type case T_I16: { int16_t val; - if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + if (!parse_pyint(value, &val, std::numeric_limits::min(), + std::numeric_limits::max())) { return false; } @@ -378,7 +382,8 @@ bool ProtocolBase::encodeValue(PyObject* value, TType type, PyObject* type case T_I32: { int32_t val; - if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + if (!parse_pyint(value, &val, std::numeric_limits::min(), + std::numeric_limits::max())) { return false; } @@ -392,7 +397,8 @@ bool ProtocolBase::encodeValue(PyObject* value, TType type, PyObject* type return false; } - if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + if (!CHECK_RANGE(nval, std::numeric_limits::min(), + std::numeric_limits::max())) { PyErr_SetString(PyExc_OverflowError, "int out of range"); return false; } diff --git a/lib/py/src/ext/types.h b/lib/py/src/ext/types.h index fd0a1971d73..2fc9d9cc753 100644 --- a/lib/py/src/ext/types.h +++ b/lib/py/src/ext/types.h @@ -22,6 +22,11 @@ #include +#ifdef _MSC_VER +#define __STDC_LIMIT_MACROS +#endif +#include + #if PY_MAJOR_VERSION >= 3 #include From f0fb1d2bfa423322beb7a608834c47d065dffce2 Mon Sep 17 00:00:00 2001 From: Nobuaki Sukegawa Date: Sun, 28 Feb 2016 12:50:03 +0900 Subject: [PATCH 2/4] THRIFT-3694 [Windows] Disable tests of a few servers that are not supported --- test/py/RunClientServer.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py index 15c4d2b6d2d..c1443ec1784 100755 --- a/test/py/RunClientServer.py +++ b/test/py/RunClientServer.py @@ -21,6 +21,7 @@ from __future__ import division from __future__ import print_function +import platform import copy import os import signal @@ -56,15 +57,19 @@ 'json', ] -SERVERS = [ - "TSimpleServer", - "TThreadedServer", - "TThreadPoolServer", - "TProcessPoolServer", - "TForkingServer", - "TNonblockingServer", - "THttpServer", -] + +def default_servers(): + servers = [ + 'TSimpleServer', + 'TThreadedServer', + 'TThreadPoolServer', + 'TNonblockingServer', + 'THttpServer', + ] + if platform.system() != 'Windows': + servers.append('TProcessPoolServer') + servers.append('TForkingServer') + return servers def relfile(fname): @@ -77,7 +82,7 @@ def setup_pypath(libdir, gendir): pypath = env.get('PYTHONPATH', None) if pypath: dirs.append(pypath) - env['PYTHONPATH'] = ':'.join(dirs) + env['PYTHONPATH'] = os.pathsep.join(dirs) if gendir.endswith('gen-py-no_utf8strings'): env['THRIFT_TEST_PY_NO_UTF8STRINGS'] = '1' return env @@ -135,7 +140,6 @@ def ensureServerAlive(): # Wait for the server to start accepting connections on the given port. sleep_time = 0.1 # Seconds max_attempts = 100 - # try: attempt = 0 while True: sock4 = socket.socket() @@ -172,7 +176,8 @@ def ensureServerAlive(): 'processes to terminate via alarm' % (server_class, proto, use_zlib, use_ssl, extra_sleep)) time.sleep(extra_sleep) - os.kill(serverproc.pid, signal.SIGKILL) + sig = signal.SIGKILL if platform.system() != 'Windows' else signal.SIGABRT + os.kill(serverproc.pid, sig) serverproc.wait() @@ -274,9 +279,9 @@ def main(): generated_dirs.append('gen-py-%s' % (gp_dir)) # commandline permits a single class name to be specified to override SERVERS=[...] - servers = SERVERS + servers = default_servers() if len(args) == 1: - if args[0] in SERVERS: + if args[0] in servers: servers = args else: print('Unavailable server type "%s", please choose one of: %s' % (args[0], servers)) From 3c4fd55d3a0d2b5724f1ab940e0a3233d9e41689 Mon Sep 17 00:00:00 2001 From: Nobuaki Sukegawa Date: Sun, 28 Feb 2016 12:44:02 +0900 Subject: [PATCH 3/4] THRIFT-3693 Fix include issue in C++ TSSLSocketInterruptTest on Windows --- lib/cpp/src/thrift/transport/TSSLSocket.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cpp/src/thrift/transport/TSSLSocket.h b/lib/cpp/src/thrift/transport/TSSLSocket.h index ba8abf418b1..03157d7d174 100644 --- a/lib/cpp/src/thrift/transport/TSSLSocket.h +++ b/lib/cpp/src/thrift/transport/TSSLSocket.h @@ -20,11 +20,12 @@ #ifndef _THRIFT_TRANSPORT_TSSLSOCKET_H_ #define _THRIFT_TRANSPORT_TSSLSOCKET_H_ 1 +// Put this first to avoid WIN32 build failure +#include #include #include #include #include -#include namespace apache { namespace thrift { From cc59aa67f061c15d8504c5df97b29013b3700e14 Mon Sep 17 00:00:00 2001 From: Nobuaki Sukegawa Date: Sun, 28 Feb 2016 12:26:48 +0900 Subject: [PATCH 4/4] THRIFT-3692 (Re)enable Appveyor C++ and Python build --- appveyor.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f36c7f06e10..8afb0abfa71 100755 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,14 +27,15 @@ os: - Visual Studio 2015 environment: - BOOST_ROOT: c:\Libraries\boost - BOOST_LIBRARYDIR: c:\Libraries\boost\stage\lib + BOOST_ROOT: C:\Libraries\boost_1_59_0 + BOOST_LIBRARYDIR: C:\Libraries\boost_1_59_0\lib64-msvc-14.0 install: - '"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64' +- cd \ # Zlib - appveyor DownloadFile https://github.com/madler/zlib/archive/v1.2.8.tar.gz -- 7z x v1.2.8.tar.gz -so | 7z x -si -ttar +- 7z x v1.2.8.tar.gz -so | 7z x -si -ttar > nul - cd zlib-1.2.8 - cmake -G "Visual Studio 14 2015 Win64" . - cmake --build . --config release @@ -44,7 +45,7 @@ install: - ps: Start-Process Win64OpenSSL-1_0_2f.exe -ArgumentList "/silent /verysilent /sp- /suppressmsgboxes" -Wait # Libevent - appveyor DownloadFile https://github.com/libevent/libevent/releases/download/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz -- 7z x libevent-2.0.22-stable.tar.gz -so | 7z x -si -ttar +- 7z x libevent-2.0.22-stable.tar.gz -so | 7z x -si -ttar > nul - cd libevent-2.0.22-stable - nmake -f Makefile.nmake - mkdir lib @@ -53,10 +54,8 @@ install: - move *.h include\ - cd .. - cinst winflexbison -# - cinst ant -ignoreDependencies -- cd \ - appveyor DownloadFile http://www.us.apache.org/dist//ant/binaries/apache-ant-1.9.6-bin.zip -- 7z x apache-ant-1.9.6-bin.zip +- 7z x apache-ant-1.9.6-bin.zip > nul - cd %APPVEYOR_BUILD_FOLDER% # TODO: Enable Haskell build # - cinst HaskellPlatform -version 2014.2.0.0 @@ -68,17 +67,25 @@ build_script: - set PATH=%JAVA_HOME%\bin;%PATH% # - set PATH=%PATH%;C:\Program Files (x86)\Haskell Platform\2014.2.0.0\bin # - set PATH=%PATH%;C:\Program Files (x86)\Haskell Platform\2014.2.0.0\lib\extralibs\bin -- set PATH=C:\Python27-x64;%PATH% +- set PATH=C:\Python27-x64\scripts;C:\Python27-x64;%PATH% - mkdir cmake-build - cd cmake-build -- cmake -G "Visual Studio 14 2015 Win64" -DWITH_JAVA=ON -DWITH_SHARED_LIB=OFF -DLIBEVENT_ROOT=%APPVEYOR_BUILD_FOLDER%\libevent-2.0.22-stable -DZLIB_INCLUDE_DIR=%APPVEYOR_BUILD_FOLDER%\zlib-1.2.8 -DZLIB_LIBRARY=%APPVEYOR_BUILD_FOLDER%\zlib-1.2.8\release\zlibstatic.lib -DBOOST_ROOT="%BOOST_ROOT% -DBOOST_LIBRARYDIR="%BOOST_LIBRARYDIR% .. +- cmake -G "Visual Studio 14 2015 Win64" -DWITH_SHARED_LIB=OFF -DLIBEVENT_ROOT=C:\libevent-2.0.22-stable -DZLIB_INCLUDE_DIR=C:\zlib-1.2.8 -DZLIB_LIBRARY=C:\zlib-1.2.8\release\zlibstatic.lib -DBOOST_ROOT="%BOOST_ROOT%" -DBOOST_LIBRARYDIR="%BOOST_LIBRARYDIR%" .. - findstr /b /e BUILD_COMPILER:BOOL=ON CMakeCache.txt +- findstr /b /e BUILD_CPP:BOOL=ON CMakeCache.txt - findstr /b /e BUILD_JAVA:BOOL=ON CMakeCache.txt +- findstr /b /e BUILD_PYTHON:BOOL=ON CMakeCache.txt +# - findstr /b /e BUILD_C_GLIB:BOOL=ON CMakeCache.txt +# - findstr /b /e BUILD_HASKELL:BOOL=ON CMakeCache.txt - findstr /b /e BUILD_TESTING:BOOL=ON CMakeCache.txt # - cmake --build . - cmake --build . --config Release # TODO: Fix cpack # - cpack -- ctest -C Release -VV -E "(concurrency_test|processor_test|python_test$|^Python|^Java)" +# TODO: Run more tests +# CTest fails to invoke ant seemingly due to "ant.bat" v.s. "ant" (shell script) conflict. +# Currently, everything that involves OpenSSL seems to hang forever on our Appveyor setup. +# Also a few C++ tests hang (on Appveyor or on Windows in general). +- ctest -C Release --timeout 600 -VV -E "(concurrency_test|processor_test|TInterruptTest|StressTestNonBlocking|OpenSSLManualInitTest|SecurityTest|PythonTestSSLSocket|python_test$|^Java)" #TODO make it perfect ;-r