fix under miri

constellation-rs · Aug 17, 2020 · 4a4c9f3 · 4a4c9f3
1 parent 0a06de6
commit 4a4c9f3
Show file tree

Hide file tree

Showing 18 changed files with 85 additions and 44 deletions.
diff --git a/amadeus-parquet/Cargo.toml b/amadeus-parquet/Cargo.toml
@@ -29,7 +29,7 @@ educe = "0.4"
 flate2 = { version = "1.0.2", features = ["rust_backend"], default-features = false }
 futures = "0.3"
 fxhash = "0.2"
-linked-hash-map = "0.5"
+hashlink = "0.5"
 lz-fear = "0.1"
 num-bigint = "0.3"
 quick-error = "1.2.2"

diff --git a/amadeus-parquet/src/internal/compression.rs b/amadeus-parquet/src/internal/compression.rs
@@ -290,26 +290,31 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_codec_snappy() {
 		test_codec(CodecType::Snappy);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_codec_gzip() {
 		test_codec(CodecType::Gzip);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_codec_brotli() {
 		test_codec(CodecType::Brotli);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_codec_lz4() {
 		test_codec(CodecType::Lz4);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_codec_zstd() {
 		test_codec(CodecType::Zstd);
 	}
@@ -350,6 +355,7 @@ mod tests {
 	macro_rules! compress {
 		($fname:ident, $codec:expr, $col_idx:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mut codec = create_codec($codec).unwrap().unwrap();
 				let data = get_pages_bytes($col_idx);
@@ -366,6 +372,7 @@ mod tests {
 	macro_rules! decompress {
 		($fname:ident, $codec:expr, $col_idx:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let compressed_pages = {
 					let mut codec = create_codec($codec).unwrap().unwrap();

diff --git a/amadeus-parquet/src/internal/encodings/decoding.rs b/amadeus-parquet/src/internal/encodings/decoding.rs
@@ -1489,6 +1489,7 @@ mod tests {
 	macro_rules! plain {
 		($fname:ident, $num_values:expr, $batch_size:expr, $ty:ident, $pty:expr, $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mem_tracker = Rc::new(MemTracker::new());
 				let mut encoder =
@@ -1508,6 +1509,7 @@ mod tests {
 		($fname:ident, $num_values:expr, $batch_size:expr, $ty:ident, $pty:expr,
 	   $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mem_tracker = Rc::new(MemTracker::new());
 				let mut encoder = DictEncoder::<$ty>::new(Rc::new(col_desc(0, $pty)), mem_tracker);
@@ -1536,6 +1538,7 @@ mod tests {
 	macro_rules! delta_bit_pack {
 		($fname:ident, $num_values:expr, $batch_size:expr, $ty:ident, $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mut encoder = DeltaBitPackEncoder::<$ty>::new();
 

diff --git a/amadeus-parquet/src/internal/encodings/encoding.rs b/amadeus-parquet/src/internal/encodings/encoding.rs
@@ -1008,6 +1008,7 @@ mod tests {
 	const TEST_SET_SIZE: usize = 1024;
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_get_encoders() {
 		// supported encodings
 		create_and_check_encoder::<Int32Type>(Encoding::Plain, None);
@@ -1038,45 +1039,52 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_bool() {
 		BoolType::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		BoolType::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 		BoolType::test(Encoding::Rle, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_i32() {
 		Int32Type::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		Int32Type::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 		Int32Type::test(Encoding::DeltaBinaryPacked, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_i64() {
 		Int64Type::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		Int64Type::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 		Int64Type::test(Encoding::DeltaBinaryPacked, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_i96() {
 		Int96Type::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		Int96Type::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_float() {
 		FloatType::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		FloatType::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_double() {
 		DoubleType::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		DoubleType::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_byte_array() {
 		ByteArrayType::test(Encoding::Plain, TEST_SET_SIZE, -1);
 		ByteArrayType::test(Encoding::PlainDictionary, TEST_SET_SIZE, -1);
@@ -1085,13 +1093,15 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_fixed_lenbyte_array() {
 		FixedLenByteArrayType::test(Encoding::Plain, TEST_SET_SIZE, 100);
 		FixedLenByteArrayType::test(Encoding::PlainDictionary, TEST_SET_SIZE, 100);
 		FixedLenByteArrayType::test(Encoding::DeltaByteArray, TEST_SET_SIZE, 100);
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_dict_encoded_size() {
 		fn run_test<T: DataType>(type_length: i32, values: &[T::Type], expected_size: usize) {
 			let mut encoder = create_test_dict_encoder::<T>(type_length);
@@ -1116,6 +1126,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_estimated_data_encoded_size() {
 		fn run_test<T: DataType>(
 			encoding: Encoding, type_length: i32, values: &[T::Type], initial_size: usize,
@@ -1176,6 +1187,7 @@ mod tests {
 
 	// See: https://github.com/sunchao/parquet-rs/issues/47
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_issue_47() {
 		let mut encoder = create_test_encoder::<ByteArrayType>(0, Encoding::DeltaByteArray);
 		let mut decoder = create_test_decoder::<ByteArrayType>(0, Encoding::DeltaByteArray);
@@ -1378,6 +1390,7 @@ mod tests {
 	macro_rules! plain {
 		($fname:ident, $batch_size:expr, $ty:ident, $pty:expr, $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mem_tracker = Rc::new(MemTracker::new());
 				let encoder =
@@ -1391,6 +1404,7 @@ mod tests {
 	macro_rules! dict {
 		($fname:ident, $batch_size:expr, $ty:ident, $pty:expr, $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let mem_tracker = Rc::new(MemTracker::new());
 				let encoder = DictEncoder::<$ty>::new(Rc::new(col_desc(0, $pty)), mem_tracker);
@@ -1403,6 +1417,7 @@ mod tests {
 	macro_rules! delta_bit_pack {
 		($fname:ident, $batch_size:expr, $ty:ident, $gen_data_fn:expr) => {
 			#[bench]
+			#[cfg_attr(miri, ignore)]
 			fn $fname(bench: &mut Bencher) {
 				let encoder = DeltaBitPackEncoder::<$ty>::new();
 				let (bytes, values) = $gen_data_fn($batch_size);

diff --git a/amadeus-parquet/src/internal/encodings/rle.rs b/amadeus-parquet/src/internal/encodings/rle.rs
@@ -737,6 +737,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_values() {
 		for width in 1..MAX_WIDTH + 1 {
 			test_rle_values(width, 1, -1);
@@ -801,8 +802,8 @@ mod tests {
 	#[test]
 	fn test_random() {
 		let seed_len = 32;
-		let niters = 50;
-		let ngroups = 1000;
+		let niters = if !cfg!(miri) { 50 } else { 1 };
+		let ngroups = if !cfg!(miri) { 1000 } else { 200 };
 		let max_group_size = 15;
 		let mut values = vec![];
 

diff --git a/amadeus-parquet/src/internal/file/reader.rs b/amadeus-parquet/src/internal/file/reader.rs
@@ -788,6 +788,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)] // as strerror_r isn't implemented in miri
 	fn test_file_reader_try_from() {
 		// Valid file path
 		let test_file = get_test_file("alltypes_plain.parquet");
@@ -1098,6 +1099,7 @@ mod tests {
 	// Benches
 
 	#[bench]
+	#[cfg_attr(miri, ignore)]
 	fn record_reader_10k(bench: &mut Bencher) {
 		let file = get_test_file("10k-v2.parquet");
 		let len = file.metadata().unwrap().len();
@@ -1113,6 +1115,7 @@ mod tests {
 	}
 
 	#[bench]
+	#[cfg_attr(miri, ignore)]
 	fn record_reader_10k_typed(bench: &mut Bencher) {
 		let file = get_test_file("10k-v2.parquet");
 		let len = file.metadata().unwrap().len();
@@ -1129,6 +1132,7 @@ mod tests {
 	}
 
 	#[bench]
+	#[cfg_attr(miri, ignore)]
 	fn record_reader_stock_simulated(bench: &mut Bencher) {
 		let file = get_test_file("stock_simulated.parquet");
 		let len = file.metadata().unwrap().len();
@@ -1184,6 +1188,7 @@ mod tests {
 	}
 
 	#[bench]
+	#[cfg_attr(miri, ignore)]
 	fn record_reader_stock_simulated_column(bench: &mut Bencher) {
 		// WARNING THIS BENCH IS INTENDED FOR THIS DATA FILE ONLY
 		// COPY OR CHANGE THE DATA FILE MAY NOT WORK AS YOU WISH

diff --git a/amadeus-parquet/src/internal/record/impls.rs b/amadeus-parquet/src/internal/record/impls.rs
@@ -1,4 +1,4 @@
-use linked_hash_map::LinkedHashMap;
+use hashlink::LinkedHashMap;
 use std::{
 	any::type_name, collections::HashMap, convert::{TryFrom, TryInto}, fmt, hash::{BuildHasher, Hash}, marker::PhantomData, string::FromUtf8Error, sync::Arc
 };
@@ -8,8 +8,6 @@ use amadeus_types::{
 	Bson, Data, Date, DateTime, DateTimeWithoutTimezone, DateWithoutTimezone, Decimal, Enum, Group, IpAddr, Json, List, Time, TimeWithoutTimezone, Timezone, Url, Value, Webpage
 };
 
-#[cfg(debug_assertions)]
-use crate::internal::schema::parser::parse_message_type;
 use crate::internal::{
 	basic::{LogicalType, Repetition, Type as PhysicalType}, column::reader::ColumnReader, data_type::{
 		BoolType, ByteArrayType, DoubleType, FixedLenByteArrayType, FloatType, Int32Type, Int64Type, Int96, Int96Type
@@ -1112,6 +1110,8 @@ where
 				.map(|(name, schema_)| {
 					#[cfg(debug_assertions)]
 					{
+						use crate::internal::schema::parser::parse_message_type;
+
 						// Check parsing and printing by round-tripping both typed and untyped and checking correctness.
 						// TODO: do with predicates also
 

diff --git a/amadeus-parquet/src/internal/record/predicates.rs b/amadeus-parquet/src/internal/record/predicates.rs
@@ -1,5 +1,5 @@
 use fxhash::FxBuildHasher;
-use linked_hash_map::LinkedHashMap;
+use hashlink::LinkedHashMap;
 use std::collections::HashMap;
 
 use amadeus_types::{Bson, Date, DateTime, Decimal, Enum, Group, Json, List, Time, Value};

diff --git a/amadeus-parquet/src/internal/record/reader.rs b/amadeus-parquet/src/internal/record/reader.rs
@@ -24,7 +24,7 @@
 //! that are optional or repeated.
 
 use fxhash::FxBuildHasher;
-use linked_hash_map::LinkedHashMap;
+use hashlink::LinkedHashMap;
 use std::{
 	collections::HashMap, convert::TryInto, error::Error, marker::PhantomData, mem, sync::Arc
 };
@@ -948,7 +948,7 @@ where
 mod tests {
 	use super::*;
 
-	use linked_hash_map::LinkedHashMap;
+	use hashlink::LinkedHashMap;
 	use std::{collections::HashMap, sync::Arc};
 
 	use crate::internal::{
@@ -1034,6 +1034,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nulls() {
 		let rows = test_file_reader_rows::<Group>("nulls.snappy.parquet", None).unwrap();
 
@@ -1076,6 +1077,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nulls_typed() {
 		type RowTyped = (Option<(Option<i32>,)>,);
 
@@ -1096,6 +1098,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nonnullable() {
 		let rows = test_file_reader_rows::<Group>("nonnullable.impala.parquet", None).unwrap();
 
@@ -1143,6 +1146,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nonnullable_typed() {
 		type RowTyped = (
 			i64,
@@ -1178,6 +1182,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nullable() {
 		let rows = test_file_reader_rows::<Group>("nullable.impala.parquet", None).unwrap();
 
@@ -1532,6 +1537,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_nullable_typed() {
 		type RowTyped = (
 			Option<i64>,
@@ -1672,6 +1678,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_projection() {
 		let _schema = "
 		  message spark_schema {
@@ -1717,6 +1724,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_projection_map() {
 		let _schema = "
 		  message spark_schema {
@@ -1798,6 +1806,7 @@ mod tests {
 	}
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_file_reader_rows_projection_list() {
 		let _schema = "
 		  message spark_schema {

diff --git a/amadeus-parquet/src/internal/record/schemas.rs b/amadeus-parquet/src/internal/record/schemas.rs
@@ -31,7 +31,7 @@
 //! ```
 
 use fxhash::FxBuildHasher;
-use linked_hash_map::LinkedHashMap;
+use hashlink::LinkedHashMap;
 use std::{
 	fmt::{self, Debug, Display}, marker::PhantomData, mem, str::FromStr
 };
@@ -1782,6 +1782,7 @@ mod tests {
 	use amadeus_types::Value;
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn schema_printing() {
 		let _schema: RootSchema<Value> = "message org.apache.impala.ComplexTypesTbl {
             REQUIRED int64 ID (INT_64);