From b6f395b2b661eaca6fe406d56a5fee43d7afa415 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Tue, 12 Jul 2016 21:41:22 +0200 Subject: [PATCH 1/2] Fix for Parquet filter pushdown Use of previous deprecated constructor defaults to null metadata, which prevents pushdown from reaching the Parquet level. --- .../datasources/parquet/SpecificParquetRecordReaderBase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java index dfe696764796f..95f083a407c87 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java @@ -205,8 +205,7 @@ protected void initialize(String path, List columns) throws IOException } } this.sparkSchema = new ParquetSchemaConverter(config).convert(requestedSchema); - this.reader = new ParquetFileReader( - config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); + this.reader = new ParquetFileReader(config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) { this.totalRowCount += block.getRowCount(); } From e64251ad4bb3e38db0dc349084059c66db5c10e7 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Wed, 10 Aug 2016 12:38:49 +0100 Subject: [PATCH 2/2] fix style --- .../datasources/parquet/SpecificParquetRecordReaderBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java index 95f083a407c87..dfe696764796f 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java @@ -205,7 +205,8 @@ protected void initialize(String path, List columns) throws IOException } } this.sparkSchema = new ParquetSchemaConverter(config).convert(requestedSchema); - this.reader = new ParquetFileReader(config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); + this.reader = new ParquetFileReader( + config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) { this.totalRowCount += block.getRowCount(); }