Merge 6760214 into 310ef06

.travis.yml

@@ -14,8 +14,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
 sudo: required
 dist: trusty
+
+language: cpp
+
+cache:
+  ccache: true
+
 addons:
   apt:
     sources:
@@ -35,6 +42,7 @@ addons:
     - bison
     - flex
     - pkg-config
+
 matrix:
   fast_finish: true
   include:
@@ -76,8 +84,6 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh
 
-language: cpp
-
 # PARQUET-626: revisit llvm toolchain when/if llvm.org apt repo resurfaces
 
 # before_install:

ci/travis_script_cpp.sh

@@ -33,18 +33,18 @@ make lint
 # fi
 
 if [ $TRAVIS_OS_NAME == "linux" ]; then
-  make -j4 || exit 1
-  ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+  make -j4
+  ctest -j2 --output-on-failure -L unittest
 # Current cpp-coveralls version 0.4 throws an error (PARQUET-1075) on Travis CI. Pin to last working version
   sudo pip install cpp_coveralls==0.3.12
   export PARQUET_ROOT=$TRAVIS_BUILD_DIR
   $TRAVIS_BUILD_DIR/ci/upload_coverage.sh
 else
-  make -j4 || exit 1
+  make -j4
   BUILD_TYPE=debug
   EXECUTABLE_DIR=$CPP_BUILD_DIR/$BUILD_TYPE
   export LD_LIBRARY_PATH=$EXECUTABLE_DIR:$LD_LIBRARY_PATH
-  ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+  ctest -j2 --output-on-failure -L unittest
 fi
 
 popd

ci/travis_script_static.sh

@@ -66,6 +66,7 @@ export LZ4_STATIC_LIB=$ARROW_EP/lz4_ep-prefix/src/lz4_ep/lib/liblz4.a
 export ZSTD_STATIC_LIB=$ARROW_EP/zstd_ep-prefix/src/zstd_ep/lib/libzstd.a
 
 cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
+      -DPARQUET_BUILD_WARNING_LEVEL=CHECKIN \
       -DPARQUET_TEST_MEMCHECK=ON \
       -DPARQUET_ARROW_LINKAGE="static" \
       -DPARQUET_BUILD_SHARED=OFF \
@@ -78,7 +79,7 @@ cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
 
 pushd $CPP_BUILD_DIR
 
-make -j4 VERBOSE=1 || exit 1
-ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+make -j4 VERBOSE=1
+ctest -j2 --output-on-failure -L unittest
 
 popd

ci/travis_script_toolchain.sh

@@ -52,16 +52,17 @@ export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
 export BOOST_ROOT=$CPP_TOOLCHAIN
 
 cmake -DPARQUET_CXXFLAGS=-Werror \
+      -DPARQUET_BUILD_WARNING_LEVEL=CHECKIN \
       -DPARQUET_TEST_MEMCHECK=ON \
       -DPARQUET_GENERATE_COVERAGE=1 \
       -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
       $TRAVIS_BUILD_DIR
 
 pushd $CPP_BUILD_DIR
 
-make -j4 || exit 1
-make install || exit 1
-ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+make -j4
+make install
+ctest -j2 --output-on-failure -L unittest
 
 popd
 

cmake_modules/ArrowExternalProject.cmake

@@ -46,7 +46,8 @@ if (MSVC AND PARQUET_USE_STATIC_CRT)
 endif()
 
 if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
-  set(ARROW_VERSION "501d60e918bd4d10c429ab34e0b8e8a87dffb732")
+  # This can be a tag or changeset
+  set(ARROW_VERSION "apache-arrow-0.10.0")
 else()
   set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
 endif()

cmake_modules/SetupCxxFlags.cmake

@@ -79,8 +79,8 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
   elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
--Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
--Wno-comma -Wno-unused-parameter -Wno-undef \
+-Wno-c++98-compat-pedantic -Wno-deprecated -Wno-deprecated-declarations \
+-Wno-weak-vtables -Wno-padded -Wno-comma -Wno-unused-parameter -Wno-undef \
 -Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
 -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
 -Wno-implicit-fallthrough -Wno-unreachable-code-return \
@@ -117,7 +117,8 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
   elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wconversion -Wno-sign-conversion")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall \
+-Wno-deprecated-declarations -Wconversion -Wno-sign-conversion")
     # Treat all compiler warnings as errors
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
   else()

src/parquet/arrow/arrow-reader-writer-test.cc

@@ -64,8 +64,8 @@ using arrow::compute::DictionaryEncode;
 using arrow::compute::FunctionContext;
 using arrow::io::BufferReader;
 
-using arrow::test::randint;
-using arrow::test::random_is_valid;
+using arrow::randint;
+using arrow::random_is_valid;
 
 using ArrowId = ::arrow::Type;
 using ParquetType = parquet::Type;
@@ -376,29 +376,6 @@ void PrintColumn(const Column& col, std::stringstream* ss) {
   }
 }
 
-void AssertTablesEqual(const Table& expected, const Table& actual,
-                       bool same_chunk_layout = true) {
-  ASSERT_EQ(expected.num_columns(), actual.num_columns());
-
-  if (same_chunk_layout) {
-    for (int i = 0; i < actual.num_columns(); ++i) {
-      AssertChunkedEqual(*expected.column(i)->data(), *actual.column(i)->data());
-    }
-  } else {
-    std::stringstream ss;
-    if (!actual.Equals(expected)) {
-      for (int i = 0; i < expected.num_columns(); ++i) {
-        ss << "Actual column " << i << std::endl;
-        PrintColumn(*actual.column(i), &ss);
-
-        ss << "Expected column " << i << std::endl;
-        PrintColumn(*expected.column(i), &ss);
-      }
-      FAIL() << ss.str();
-    }
-  }
-}
-
 void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
                        int64_t row_group_size, const std::vector<int>& column_subset,
                        std::shared_ptr<Table>* out,
@@ -428,7 +405,7 @@ void CheckSimpleRoundtrip(const std::shared_ptr<Table>& table, int64_t row_group
                               default_arrow_writer_properties()) {
   std::shared_ptr<Table> result;
   DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties);
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result, false));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result, false));
 }
 
 static std::shared_ptr<GroupNode> MakeSimpleSchema(const ::DataType& type,
@@ -1273,14 +1250,14 @@ TEST(TestArrowReadWrite, DateTimeTypes) {
       table, 1, table->num_rows(), {}, &result,
       ArrowWriterProperties::Builder().enable_deprecated_int96_timestamps()->build()));
 
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
 
   // Cast nanaoseconds to microseconds and use INT64 physical type
   ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result));
   std::shared_ptr<Table> expected;
   MakeDateTimeTypesTable(&table, true);
 
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
 }
 
 TEST(TestArrowReadWrite, CoerceTimestamps) {
@@ -1342,13 +1319,13 @@ TEST(TestArrowReadWrite, CoerceTimestamps) {
       input, 1, input->num_rows(), {}, &milli_result,
       ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MILLI)->build()));
 
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_milli_result, *milli_result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_milli_result, *milli_result));
 
   std::shared_ptr<Table> micro_result;
   ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(
       input, 1, input->num_rows(), {}, &micro_result,
       ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MICRO)->build()));
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_micro_result, *micro_result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_micro_result, *micro_result));
 }
 
 TEST(TestArrowReadWrite, CoerceTimestampsLosePrecision) {
@@ -1474,7 +1451,7 @@ TEST(TestArrowReadWrite, ConvertedDateTimeTypes) {
   std::shared_ptr<Table> result;
   ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result));
 
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_table, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_table, *result));
 }
 
 // Regression for ARROW-2802
@@ -1608,7 +1585,7 @@ TEST(TestArrowReadWrite, MultithreadedRead) {
   ASSERT_NO_FATAL_FAILURE(
       DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result));
 
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
 }
 
 TEST(TestArrowReadWrite, ReadSingleRowGroup) {
@@ -1719,7 +1696,7 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 
   auto ex_schema = ::arrow::schema(ex_fields);
   auto expected = Table::Make(ex_schema, ex_columns);
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*expected, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*expected, *result));
 }
 
 TEST(TestArrowReadWrite, ListLargeRecords) {
@@ -1745,7 +1722,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {
   // Read everything
   std::shared_ptr<Table> result;
   ASSERT_OK_NO_THROW(reader->ReadTable(&result));
-  ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+  ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
 
   ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
                               ::arrow::default_memory_pool(),
@@ -1922,7 +1899,7 @@ TEST(TestArrowReadWrite, DictionaryColumnChunkedWrite) {
 
   auto expected_table = Table::Make(schema, columns);
 
-  AssertTablesEqual(*expected_table, *result, false);
+  ::arrow::AssertTablesEqual(*expected_table, *result, false);
 }
 
 TEST(TestArrowWrite, CheckChunkSize) {

src/parquet/arrow/test-util.h

@@ -24,6 +24,14 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/decimal.h"
 
+namespace arrow {
+// PARQUET-1382: backwards-compatible shim for arrow::test namespace
+namespace test {}
+
+using namespace ::arrow::test;  // NOLINT
+
+}  // namespace arrow
+
 namespace parquet {
 namespace arrow {
 
@@ -66,7 +74,7 @@ typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type NonNullA
     size_t size, std::shared_ptr<Array>* out) {
   using c_type = typename ArrowType::c_type;
   std::vector<c_type> values;
-  ::arrow::test::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
+  ::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
                              &values);
   ::arrow::NumericBuilder<ArrowType> builder;
   RETURN_NOT_OK(builder.Append(values.data(), values.size()));
@@ -78,7 +86,7 @@ typename std::enable_if<
     is_arrow_int<ArrowType>::value && !is_arrow_date<ArrowType>::value, Status>::type
 NonNullArray(size_t size, std::shared_ptr<Array>* out) {
   std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::randint(size, 0, 64, &values);
+  ::arrow::randint(size, 0, 64, &values);
 
   // Passing data type so this will work with TimestampType too
   ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
@@ -91,7 +99,7 @@ template <class ArrowType>
 typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NonNullArray(
     size_t size, std::shared_ptr<Array>* out) {
   std::vector<typename ArrowType::c_type> values;
-  ::arrow::test::randint(size, 0, 64, &values);
+  ::arrow::randint(size, 0, 64, &values);
   for (size_t i = 0; i < size; i++) {
     values[i] *= 86400000;
   }
@@ -175,7 +183,7 @@ template <class ArrowType>
 typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NonNullArray(
     size_t size, std::shared_ptr<Array>* out) {
   std::vector<uint8_t> values;
-  ::arrow::test::randint(size, 0, 1, &values);
+  ::arrow::randint(size, 0, 1, &values);
   ::arrow::BooleanBuilder builder;
   RETURN_NOT_OK(builder.Append(values.data(), values.size()));
   return builder.Finish(out);
@@ -187,7 +195,7 @@ typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type Nullable
     size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
   using c_type = typename ArrowType::c_type;
   std::vector<c_type> values;
-  ::arrow::test::random_real(size, seed, static_cast<c_type>(-1e10),
+  ::arrow::random_real(size, seed, static_cast<c_type>(-1e10),
                              static_cast<c_type>(1e10), &values);
   std::vector<uint8_t> valid_bytes(size, 1);
 
@@ -209,7 +217,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Arra
 
   // Seed is random in Arrow right now
   (void)seed;
-  ::arrow::test::randint(size, 0, 64, &values);
+  ::arrow::randint(size, 0, 64, &values);
   std::vector<uint8_t> valid_bytes(size, 1);
 
   for (size_t i = 0; i < num_nulls; i++) {
@@ -230,7 +238,7 @@ typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NullableA
 
   // Seed is random in Arrow right now
   (void)seed;
-  ::arrow::test::randint(size, 0, 64, &values);
+  ::arrow::randint(size, 0, 64, &values);
   for (size_t i = 0; i < size; i++) {
     values[i] *= 86400000;
   }
@@ -268,7 +276,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed,
     if (!valid_bytes[i]) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
-      ::arrow::test::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
+      ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
       RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
     }
   }
@@ -297,7 +305,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed,
     if (!valid_bytes[i]) {
       RETURN_NOT_OK(builder.AppendNull());
     } else {
-      ::arrow::test::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
+      ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
       RETURN_NOT_OK(builder.Append(buffer));
     }
   }
@@ -341,7 +349,7 @@ typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NullableA
   // Seed is random in Arrow right now
   (void)seed;
 
-  ::arrow::test::randint(size, 0, 1, &values);
+  ::arrow::randint(size, 0, 1, &values);
   std::vector<uint8_t> valid_bytes(size, 1);
 
   for (size_t i = 0; i < num_nulls; i++) {