From 2f2deab2b17d52202427b9c1b606d313b0fb27f6 Mon Sep 17 00:00:00 2001 From: Matthew Burgess Date: Wed, 7 Mar 2018 15:19:39 -0500 Subject: [PATCH 1/2] NIFI-4944: Guard against race condition in Snappy for PutHiveStreaming --- .../apache/nifi/processors/hive/PutHiveStreaming.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java index f5f8dc6d3663..ca5a01b5d5c2 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java @@ -63,6 +63,7 @@ import org.apache.nifi.util.hive.HiveOptions; import org.apache.nifi.util.hive.HiveUtils; import org.apache.nifi.util.hive.HiveWriter; +import org.xerial.snappy.Snappy; import java.io.ByteArrayOutputStream; import java.io.File; @@ -137,6 +138,16 @@ public class PutHiveStreaming extends AbstractSessionFactoryProcessor { private static final Set RESERVED_METADATA; static { + synchronized (Snappy.class) { + // This is used to prevent a race condition in Snappy 1.0.5 where two classloaders could + // try to define the native loader class at the same time, causing an error. Make a no-op + // call here to ensure Snappy's static initializers are called + try { + Snappy.compress(""); + } catch (IOException ioe) { + // Do nothing here, should never happen as it is intended to be a no-op + } + } Set reservedMetadata = new HashSet<>(); reservedMetadata.add("avro.schema"); reservedMetadata.add("avro.codec"); From 6bf635b1cf8f565f1a1c507ca07b1b1e29dec6f1 Mon Sep 17 00:00:00 2001 From: Matthew Burgess Date: Mon, 12 Mar 2018 10:35:06 -0400 Subject: [PATCH 2/2] NIFI-4944: Removed unnecessary synchronized block, added more comments --- .../processors/hive/PutHiveStreaming.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java index ca5a01b5d5c2..35af734349ac 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java @@ -138,16 +138,17 @@ public class PutHiveStreaming extends AbstractSessionFactoryProcessor { private static final Set RESERVED_METADATA; static { - synchronized (Snappy.class) { - // This is used to prevent a race condition in Snappy 1.0.5 where two classloaders could - // try to define the native loader class at the same time, causing an error. Make a no-op - // call here to ensure Snappy's static initializers are called - try { - Snappy.compress(""); - } catch (IOException ioe) { - // Do nothing here, should never happen as it is intended to be a no-op - } + // This is used to prevent a race condition in Snappy 1.0.5 where two classloaders could + // try to define the native loader class at the same time, causing an error. Make a no-op + // call here to ensure Snappy's static initializers are called. Note that this block is + // called once by the extensions loader before any actual processor instances are created, + // so the race condition will not occur, and for each other instance, this is a no-op + try { + Snappy.compress(""); + } catch (IOException ioe) { + // Do nothing here, should never happen as it is intended to be a no-op } + Set reservedMetadata = new HashSet<>(); reservedMetadata.add("avro.schema"); reservedMetadata.add("avro.codec");