Skip to content

Commit 4a3a2b7

Browse files
granthenkeadembo
granthenke
authored and
adembo
committed
KUDU-2412: Fix python client compilation in el6 environments
This uses the C preprocessor to check the value of KUDU_INT128_SUPPORTED from the C++ headers and defines a Cython macro PYKUDU_INT128_SUPPORTED. PYKUDU_INT128_SUPPORTED is generated by replacing the KUDU_INT128_SUPPORTED placeholder in config.pxi.in to generate config.pxi which is then included in client.pyx. This is then used to conditionally generate the bits of Python code that depend on int128-related methods. See here for documentation: http://cython.readthedocs.io/en/latest/src/userguide/language_basics.html#conditional-statements Change-Id: Ibd93b57020b80597baae9c8d3e0434c46f7fc3d7 Reviewed-on: http://gerrit.cloudera.org:8080/10114 Reviewed-by: Adar Dembo <adar@cloudera.com> Tested-by: Grant Henke <granthenke@apache.org> (cherry picked from commit 3a801d1) Reviewed-on: http://gerrit.cloudera.org:8080/10140 Tested-by: Adar Dembo <adar@cloudera.com>
1 parent 45b3f21 commit 4a3a2b7

File tree

10 files changed

+127
-34
lines changed

10 files changed

+127
-34
lines changed

‎python/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,4 @@ var/
6969

7070
# automatically generated during local development
7171
kudu/version.py
72+
kudu/config.pxi

‎python/kudu/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
READ_AT_SNAPSHOT,
3030
READ_YOUR_WRITES,
3131
EXCLUSIVE_BOUND,
32-
INCLUSIVE_BOUND)
32+
INCLUSIVE_BOUND,
33+
CLIENT_SUPPORTS_DECIMAL)
3334

3435
from kudu.errors import (KuduException, KuduBadStatus, KuduNotFound, # noqa
3536
KuduNotSupported,

‎python/kudu/client.pyx‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
# distutils: language = c++
1919
# cython: embedsignature = True
2020

21+
include "config.pxi"
22+
2123
from libcpp.string cimport string
2224
from libcpp cimport bool as c_bool
2325
from libcpp.map cimport map
@@ -35,6 +37,15 @@ from errors import KuduException
3537

3638
import six
3739

40+
# True if this python client was compiled with a
41+
# compiler that supports __int128 which is required
42+
# for decimal type support. This is generally true
43+
# except for EL6 environments.
44+
IF PYKUDU_INT128_SUPPORTED == 1:
45+
CLIENT_SUPPORTS_DECIMAL = True
46+
ELSE:
47+
CLIENT_SUPPORTS_DECIMAL = False
48+
3849
# Replica selection enums
3950
LEADER_ONLY = ReplicaSelection_Leader
4051
CLOSEST_REPLICA = ReplicaSelection_Closest
@@ -1319,9 +1330,12 @@ cdef class Row:
13191330
return val
13201331

13211332
cdef inline __get_unscaled_decimal(self, int i):
1322-
cdef int128_t val
1323-
check_status(self.row.GetUnscaledDecimal(i, &val))
1324-
return val
1333+
IF PYKUDU_INT128_SUPPORTED == 1:
1334+
cdef int128_t val
1335+
check_status(self.row.GetUnscaledDecimal(i, &val))
1336+
return val
1337+
ELSE:
1338+
raise KuduException("The decimal type is not supported when GCC version is < 4.6.0" % self)
13251339

13261340
cdef inline get_decimal(self, int i):
13271341
scale = self.parent.batch.projection_schema().Column(i).type_attributes().scale()
@@ -2454,7 +2468,6 @@ cdef class PartialRow:
24542468
elif t == KUDU_STRING:
24552469
if isinstance(value, unicode):
24562470
value = value.encode('utf8')
2457-
24582471
slc = Slice(<char*> value, len(value))
24592472
check_status(self.row.SetStringCopy(i, slc))
24602473
elif t == KUDU_BINARY:
@@ -2468,8 +2481,10 @@ cdef class PartialRow:
24682481
check_status(self.row.SetUnixTimeMicros(i, <int64_t>
24692482
to_unixtime_micros(value)))
24702483
elif t == KUDU_DECIMAL:
2471-
check_status(self.row.SetUnscaledDecimal(i, <int128_t>
2472-
to_unscaled_decimal(value)))
2484+
IF PYKUDU_INT128_SUPPORTED == 1:
2485+
check_status(self.row.SetUnscaledDecimal(i, <int128_t>to_unscaled_decimal(value)))
2486+
ELSE:
2487+
raise KuduException("The decimal type is not supported when GCC version is < 4.6.0" % self)
24732488
else:
24742489
raise TypeError("Cannot set kudu type <{0}>.".format(_type_names[t]))
24752490

‎python/kudu/config.pxi.in‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// This file is processed by setup.py
19+
// See generate_config_pxi.
20+
21+
// KUDU_INT128_SUPPORTED is substituted from the C++ macro.
22+
// PYKUDU_INT128_SUPPORTED becomes a Cython macro.
23+
DEF PYKUDU_INT128_SUPPORTED = KUDU_INT128_SUPPORTED

‎python/kudu/libkudu_client.pxd‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ cdef extern from "kudu/util/status.h" namespace "kudu" nogil:
6363
c_bool IsAborted()
6464

6565
cdef extern from "kudu/util/int128.h" namespace "kudu":
66+
# see https://gist.github.com/ricrogz/7f9c405450689866139c49476b959044
67+
# This can be defined on the Python side even if it's not defined on the
68+
# C++ side.
6669
ctypedef int int128_t
6770

6871
cdef extern from "kudu/util/monotime.h" namespace "kudu" nogil:

‎python/kudu/schema.pyx‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@
1818
# distutils: language = c++
1919
# cython: embedsignature = True
2020

21+
include "config.pxi"
22+
2123
from cython.operator cimport dereference as deref
2224

2325
from kudu.compat import tobytes, frombytes
2426
from kudu.schema cimport *
2527
from kudu.errors cimport check_status
2628
from kudu.client cimport PartialRow
2729
from kudu.util import get_decimal_scale, to_unixtime_micros, to_unscaled_decimal
30+
from errors import KuduException
2831

2932
import six
3033

@@ -762,9 +765,12 @@ cdef class KuduValue:
762765
value = to_unixtime_micros(value)
763766
self._value = C_KuduValue.FromInt(value)
764767
elif (type_.name == 'decimal'):
765-
scale = get_decimal_scale(value)
766-
value = to_unscaled_decimal(value)
767-
self._value = C_KuduValue.FromDecimal(value, scale)
768+
IF PYKUDU_INT128_SUPPORTED == 1:
769+
scale = get_decimal_scale(value)
770+
value = to_unscaled_decimal(value)
771+
self._value = C_KuduValue.FromDecimal(value, scale)
772+
ELSE:
773+
raise KuduException("The decimal type is not supported when GCC version is < 4.6.0" % self)
768774
else:
769775
raise TypeError("Cannot initialize KuduValue for kudu type <{0}>"
770776
.format(type_.name))

‎python/kudu/tests/test_scanner.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,9 @@ def test_float_pred(self):
267267
self._test_float_pred()
268268

269269
def test_decimal_pred(self):
270-
# Test a decimal predicate
271-
self._test_decimal_pred()
270+
if kudu.CLIENT_SUPPORTS_DECIMAL:
271+
# Test a decimal predicate
272+
self._test_decimal_pred()
272273

273274
def test_binary_pred(self):
274275
# Test a binary predicate

‎python/kudu/tests/test_scantoken.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,9 @@ def test_unixtime_micros_pred(self):
254254
self._test_unixtime_micros_pred()
255255

256256
def test_decimal_pred(self):
257-
# Test decimal value predicate
258-
self._test_decimal_pred()
257+
if kudu.CLIENT_SUPPORTS_DECIMAL:
258+
# Test decimal value predicate
259+
self._test_decimal_pred()
259260

260261
def test_bool_pred(self):
261262
# Test a boolean value predicate

‎python/kudu/tests/util.py‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import datetime
2626
import pytz
2727

28-
2928
class TestScanBase(KuduTestBase, unittest.TestCase):
3029

3130
@classmethod
@@ -72,7 +71,8 @@ def setUpClass(self):
7271
builder = kudu.schema_builder()
7372
builder.add_column('key').type(kudu.int64).nullable(False)
7473
builder.add_column('unixtime_micros_val', type_=kudu.unixtime_micros, nullable=False)
75-
builder.add_column('decimal_val', type_=kudu.decimal, precision=5, scale=2)
74+
if kudu.CLIENT_SUPPORTS_DECIMAL:
75+
builder.add_column('decimal_val', type_=kudu.decimal, precision=5, scale=2)
7676
builder.add_column('string_val', type_=kudu.string, compression=kudu.COMPRESSION_LZ4, encoding='prefix')
7777
builder.add_column('bool_val', type_=kudu.bool)
7878
builder.add_column('double_val', type_=kudu.double)
@@ -104,16 +104,28 @@ def setUpClass(self):
104104
self.type_table = self.client.table(table_name)
105105

106106
# Insert new rows
107-
self.type_test_rows = [
108-
(1, datetime.datetime(2016, 1, 1).replace(tzinfo=pytz.utc), Decimal('111.11'),
109-
"Test One", True, 1.7976931348623157 * (10^308), 127,
110-
b'\xce\x99\xce\xbf\xcf\x81\xce\xb4\xce\xb1\xce\xbd\xce\xaf\xce\xb1',
111-
3.402823 * (10^38)),
112-
(2, datetime.datetime.utcnow().replace(tzinfo=pytz.utc), Decimal('0.99'),
113-
"测试二", False, 200.1, -1,
114-
b'\xd0\x98\xd0\xbe\xd1\x80\xd0\xb4\xd0\xb0\xd0\xbd\xd0\xb8\xd1\x8f',
115-
-150.2)
116-
]
107+
if kudu.CLIENT_SUPPORTS_DECIMAL:
108+
self.type_test_rows = [
109+
(1, datetime.datetime(2016, 1, 1).replace(tzinfo=pytz.utc), Decimal('111.11'),
110+
"Test One", True, 1.7976931348623157 * (10^308), 127,
111+
b'\xce\x99\xce\xbf\xcf\x81\xce\xb4\xce\xb1\xce\xbd\xce\xaf\xce\xb1',
112+
3.402823 * (10^38)),
113+
(2, datetime.datetime.utcnow().replace(tzinfo=pytz.utc), Decimal('0.99'),
114+
"测试二", False, 200.1, -1,
115+
b'\xd0\x98\xd0\xbe\xd1\x80\xd0\xb4\xd0\xb0\xd0\xbd\xd0\xb8\xd1\x8f',
116+
-150.2)
117+
]
118+
else:
119+
self.type_test_rows = [
120+
(1, datetime.datetime(2016, 1, 1).replace(tzinfo=pytz.utc),
121+
"Test One", True, 1.7976931348623157 * (10 ^ 308), 127,
122+
b'\xce\x99\xce\xbf\xcf\x81\xce\xb4\xce\xb1\xce\xbd\xce\xaf\xce\xb1',
123+
3.402823 * (10 ^ 38)),
124+
(2, datetime.datetime.utcnow().replace(tzinfo=pytz.utc),
125+
"测试二", False, 200.1, -1,
126+
b'\xd0\x98\xd0\xbe\xd1\x80\xd0\xb4\xd0\xb0\xd0\xbd\xd0\xb8\xd1\x8f',
127+
-150.2)
128+
]
117129
session = self.client.new_session()
118130
for row in self.type_test_rows:
119131
op = self.type_table.new_insert(row)
@@ -234,12 +246,13 @@ def _test_float_pred(self):
234246
)
235247

236248
def _test_decimal_pred(self):
237-
self.verify_pred_type_scans(
238-
preds=[
239-
self.type_table['decimal_val'] == Decimal('111.11')
240-
],
241-
row_indexes=slice(0, 1),
242-
)
249+
if kudu.CLIENT_SUPPORTS_DECIMAL:
250+
self.verify_pred_type_scans(
251+
preds=[
252+
self.type_table['decimal_val'] == Decimal('111.11')
253+
],
254+
row_indexes=slice(0, 1),
255+
)
243256

244257
def _test_binary_pred(self):
245258
self.verify_pred_type_scans(

‎python/setup.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
from distutils.command.clean import clean as _clean
2828
from distutils.extension import Extension
2929
import os
30+
import subprocess
3031

3132
# Workaround a Python bug in which multiprocessing's atexit handler doesn't
3233
# play well with pytest. See http://bugs.python.org/issue15881 for details
@@ -64,13 +65,39 @@ def write_version_py(filename=os.path.join(setup_dir, 'kudu/version.py')):
6465
class clean(_clean):
6566
def run(self):
6667
_clean.run(self)
67-
for x in ['kudu/client.cpp', 'kudu/schema.cpp',
68-
'kudu/errors.cpp']:
68+
for x in ['kudu/client.cpp',
69+
'kudu/config.pxi',
70+
'kudu/errors.cpp',
71+
'kudu/schema.cpp']:
6972
try:
7073
os.remove(x)
7174
except OSError:
7275
pass
7376

77+
# Identify the cc version used and check for __int128 support
78+
def generate_config_pxi(include_dirs):
79+
""" Generate config.pxi from config.pxi.in """
80+
int128_h = None
81+
for d in include_dirs:
82+
candidate = os.path.join(d, 'kudu/util/int128.h')
83+
if os.path.exists(candidate):
84+
int128_h = candidate
85+
break
86+
if int128_h is None:
87+
raise Exception("could not find int128.h in Kudu include dirs")
88+
src = os.path.join(setup_dir, 'kudu/config.pxi.in')
89+
dst = os.path.join(setup_dir, 'kudu/config.pxi')
90+
dst_tmp = dst + '.tmp'
91+
cc = os.getenv("CC","cc")
92+
subprocess.check_call([cc, "-x", "c++", "-o", dst_tmp,
93+
"-E", '-imacros', int128_h, src])
94+
# If our generated file is the same as the prior version,
95+
# don't replace it. This avoids rebuilding everything on every
96+
# run of setup.py.
97+
if os.path.exists(dst) and open(dst).read() == open(dst_tmp).read():
98+
os.unlink(dst_tmp)
99+
else:
100+
os.rename(dst_tmp, dst)
74101

75102
# If we're in the context of the Kudu git repository, build against the
76103
# latest in-tree build artifacts
@@ -111,6 +138,8 @@ def run(self):
111138
LIBRARY_DIRS = [kudu_lib_dir]
112139
RT_LIBRARY_DIRS = LIBRARY_DIRS
113140

141+
generate_config_pxi(INCLUDE_PATHS)
142+
114143
ext_submodules = ['client', 'errors', 'schema']
115144

116145
extensions = []

0 commit comments

Comments
 (0)