From 25e84080d60d768b75acd7b12459ba2f3a678e40 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 23 Jun 2017 15:19:37 +0100 Subject: [PATCH] just fix parquetfileformat --- .../sql/execution/datasources/parquet/ParquetFileFormat.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala index 87fbf8b1bc9c4..dba1148054da6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala @@ -481,8 +481,10 @@ object ParquetFileFormat extends Logging { val parFiles = partFiles.par val pool = ThreadUtils.newForkJoinPool("readingParquetFooters", 8) parFiles.tasksupport = new ForkJoinTaskSupport(pool) + val tc = TaskContext.get() try { parFiles.flatMap { currentFile => + TaskContext.setTaskContext(tc) try { // Skips row group information since we only need the schema. // ParquetFileReader.readFooter throws RuntimeException, instead of IOException,