304 changes: 225 additions & 79 deletions ibis/sql/tests/test_compiler.py

Large diffs are not rendered by default.

16 changes: 10 additions & 6 deletions ibis/sql/tests/test_sqlalchemy.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def test_joins(self):
rt.outerjoin(nt, spred)),
]
for ibis_joined, joined_sqla in fully_mat_joins:
expected = sa.select(['*']).select_from(joined_sqla)
expected = sa.select([joined_sqla])
self._compare_sqla(ibis_joined, expected)

subselect_joins = [
Expand Down Expand Up @@ -260,7 +260,7 @@ def test_join_just_materialized(self):
sqla_joined = (nt.join(rt, nt.c.n_regionkey == rt.c.r_regionkey)
.join(ct, nt.c.n_nationkey == ct.c.c_nationkey))

expected = sa.select(['*']).select_from(sqla_joined)
expected = sa.select([sqla_joined])

self._compare_sqla(joined, expected)

Expand Down Expand Up @@ -559,15 +559,19 @@ def test_sort_aggregation_translation_failure(self):
# after Aggregate to produce a single select statement rather than an
# inline view.
t = self.alltypes
sat = self.sa_alltypes.alias('t0')

agg = (t.group_by('string_col')
.aggregate(t.double_col.max().name('foo')))
expr = agg.sort_by(ibis.desc('foo'))

ex = (sa.select([sat.c.string_col,
F.max(sat.c.double_col).label('foo')])
.group_by(sat.c.string_col)
sat = self.sa_alltypes.alias('t1')
base = (sa.select([sat.c.string_col,
F.max(sat.c.double_col).label('foo')])
.group_by(sat.c.string_col)).alias('t0')

ex = (sa.select([base.c.string_col,
base.c.foo])
.select_from(base)
.order_by(sa.desc('foo')))

self._compare_sqla(expr, ex)
Expand Down
5 changes: 3 additions & 2 deletions ibis/sql/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


import ibis.expr.analysis as L

import ibis.expr.operations as ops
import ibis.expr.types as ir
import ibis.util as util
Expand Down Expand Up @@ -107,7 +108,7 @@ def _visit_table(self, expr):
# Foreign ref
self.foreign_table = expr
else:
if not isinstance(node, ir.BlockingTableNode):
if not node.blocks():
for arg in node.flat_args():
if isinstance(arg, ir.Expr):
self._visit(arg)
Expand All @@ -121,7 +122,7 @@ def _is_root(self, what):
def _find_blocking_table(expr):
node = expr.op()

if isinstance(node, ir.BlockingTableNode):
if node.blocks():
return expr

for arg in node.flat_args():
Expand Down
2 changes: 1 addition & 1 deletion ibis/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import ibis

groups = ['hdfs', 'impala', 'madlib', 'sqlite']
groups = ['hdfs', 'impala', 'madlib', 'sqlite', 'kudu']


def pytest_configure(config):
Expand Down
11 changes: 11 additions & 0 deletions ibis/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,3 +219,14 @@ def __set__(self, obj, value):
return

cache[self.name] = value


def approx_equal(a, b, eps):
assert abs(a - b) < eps


def implements(f):
def decorator(g):
g.__doc__ = f.__doc__
return g
return decorator
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
pytest
numpy>=1.7.0
pandas>=0.12.0
impyla>=0.12.0
impyla>=0.13.2
hdfs>=2.0.0
sqlalchemy>=1.0.0
six
2 changes: 1 addition & 1 deletion scripts/test_data_admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def build_udfs():
print('Building UDFs')
ibis_home_dir = osp.dirname(osp.dirname(osp.abspath(__file__)))
udf_dir = pjoin(ibis_home_dir, 'testing', 'udf')
check_call('cmake . && make', shell=True, cwd=udf_dir)
check_call('cmake . && make VERBOSE=1', shell=True, cwd=udf_dir)


def upload_udfs(con):
Expand Down
70 changes: 68 additions & 2 deletions testing/udf/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,83 @@ find_program(CLANG_EXECUTABLE clang++)

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb")

# find boost headers and libs
if ($ENV{ENABLE_NATIVE_TOOLCHAIN})
message(STATUS "Found Impala Toolchain, enabling toolchain boost")
set(ENV{BOOST_ROOT} "$ENV{IMPALA_TOOLCHAIN}/boost-$ENV{BOOST_VERSION}")
set(Boost_USE_STATIC_LIBS NOT ${BUILD_SHARED_LIBS})
set(Boost_USE_STATIC_RUNTIME ON)
else()
set(Boost_DEBUG TRUE)
set(Boost_USE_MULTITHREADED ON)
endif()

find_package(Boost REQUIRED)
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
set(LIBS ${LIBS} ${Boost_LIBRARIES})
message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
message(STATUS "Boost libraries: " ${Boost_LIBRARIES})

set(CLANG_INCLUDE_FLAGS)

if ($ENV{ENABLE_NATIVE_TOOLCHAIN})
set(IMPALA_TOOLCHAIN ON)

# Set the root directory for the toolchain
set(TOOLCHAIN_ROOT $ENV{IMPALA_TOOLCHAIN})

if($ENV{USE_SYSTEM_GCC} EQUAL "0")
# If Impala is built with the GCC from the toolchain, change compiler and link paths
set(GCC_ROOT $ENV{IMPALA_TOOLCHAIN}/gcc-$ENV{IMPALA_GCC_VERSION})
set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc)
set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)

# The rpath is needed to be able to run the binaries produced by the toolchain without
# specifying an LD_LIBRARY_PATH
set(TOOLCHAIN_LINK_FLAGS "-Wl,-rpath,${GCC_ROOT}/lib64")
set(TOOLCHAIN_LINK_FLAGS "${TOOLCHAIN_LINK_FLAGS} -L${GCC_ROOT}/lib64")
endif()

if(NOT $ENV{IMPALA_CXX_COMPILER} STREQUAL "default")
set(CMAKE_CXX_COMPILER $ENV{IMPALA_CXX_COMPILER})
endif()
endif()

message(STATUS "Setup toolchain link flags ${TOOLCHAIN_LINK_FLAGS}")

if (IMPALA_TOOLCHAIN)
# When the Toolchain is used we use LLVM 3.3 that was built in a different path that it
# is invoked from, and a GCC that resides in a different location. LVVM 3.3 relies on
# hard-coded path information about where to find the system headers and does not support
# specifying the -gcc-toolchain flag to dynamically provide this information. Because of
# these reasons we need to manually add the system c++ headers to the path when we
# compile the IR code with clang.
set(CLANG_BASE_FLAGS
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}"
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}/backward"
"-I${GCC_ROOT}/include/c++/$ENV{IMPALA_GCC_VERSION}/x86_64-unknown-linux-gnu")
set(CLANG_INCLUDE_FLAGS ${CLANG_BASE_FLAGS})
endif()

set(CLANG_INCLUDE_FLAGS
${CLANG_INCLUDE_FLAGS}
"-I${Boost_INCLUDE_DIRS}"
)

set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-DIR_COMPILE" "-DNDEBUG"
"-DHAVE_INTTYPES_H" "-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG"
"-Wno-deprecated" "-Wno-c++11-extensions" "-O1")

# Function to generate rule to cross compile a source file to an IR module.
# This should be called with the .cc src file and it will generate a
# src-file-ir target that can be built.
# e.g. COMPILE_TO_IR(test.cc) generates the "test-ir" make target.
set(IR_COMPILE_FLAGS "-emit-llvm" "-O3" "-c")
function(COMPILE_TO_IR SRC_FILE)
get_filename_component(BASE_NAME ${SRC_FILE} NAME_WE)
set(OUTPUT_FILE "build/${BASE_NAME}.ll")
add_custom_command(
OUTPUT ${OUTPUT_FILE}
COMMAND ${CLANG_EXECUTABLE} ${IR_COMPILE_FLAGS} ${SRC_FILE} -o ${OUTPUT_FILE}
COMMAND ${CLANG_EXECUTABLE} ${CLANG_INCLUDE_FLAGS} ${CLANG_IR_CXX_FLAGS} ${SRC_FILE} -o ${OUTPUT_FILE}
DEPENDS ${SRC_FILE})
add_custom_target(${BASE_NAME}-ir ALL DEPENDS ${OUTPUT_FILE})
endfunction(COMPILE_TO_IR)
Expand Down
1 change: 0 additions & 1 deletion testing/udf/lib/udf-debug.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,3 @@ inline std::string DebugString(const StringVal& val) {
}

#endif

469 changes: 253 additions & 216 deletions testing/udf/lib/udf.h

Large diffs are not rendered by default.