From 4b3eb7d8cae26d5ea08fd57fc1199102086da0bf Mon Sep 17 00:00:00 2001 From: kautshukla Date: Tue, 20 Nov 2018 21:07:33 +0530 Subject: [PATCH 1/3] {SequenceFileFormat} Instance check to handle cast exception from bytewritable to text --- .../org/apache/hadoop/hive/serde2/JsonSerDe.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java index f1c8477ffc44..0faf592e01cf 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hive.common.util.HiveStringUtils; @@ -127,7 +128,18 @@ public void initialize(Configuration conf, Properties tbl) public Object deserialize(Writable blob) throws SerDeException { Object row; - Text t = (Text) blob; + Text t; + + /*To check writable instance and then create Text object + * In case of sequence file format which writes in key: org.apache.hadoop.io.LongWritable & value: org.apache.hadoop.io.BytesWritable + * format. + */ + if (blob instanceof BytesWritable) { + t = new Text(((BytesWritable) blob).getBytes()); + } else { + t = (Text) blob; + } + try { row = structReader.parseStruct(new ByteArrayInputStream((t.getBytes()), 0, t.getLength())); return row; From 634a9db4b3a5180ea9f25196bcad09998e3bfd95 Mon Sep 17 00:00:00 2001 From: kautshukla Date: Tue, 20 Nov 2018 23:26:21 +0530 Subject: [PATCH 2/3] wrapping the original byteWritable --- .../apache/hadoop/hive/serde2/JsonSerDe.java | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java index 0faf592e01cf..b9926715c733 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java @@ -20,11 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.type.Date; @@ -127,24 +123,19 @@ public void initialize(Configuration conf, Properties tbl) @Override public Object deserialize(Writable blob) throws SerDeException { - Object row; - Text t; - - /*To check writable instance and then create Text object - * In case of sequence file format which writes in key: org.apache.hadoop.io.LongWritable & value: org.apache.hadoop.io.BytesWritable - * format. - */ - if (blob instanceof BytesWritable) { - t = new Text(((BytesWritable) blob).getBytes()); - } else { - t = (Text) blob; - } + String st; try { - row = structReader.parseStruct(new ByteArrayInputStream((t.getBytes()), 0, t.getLength())); - return row; + if (blob instanceof BytesWritable) { + byte[] rowByteArr = ((BytesWritable) blob).getBytes(); + st = Text.decode(rowByteArr, 0, rowByteArr.length).trim(); + } else { + st = blob.toString().trim(); + } + + return structReader.parseStruct(new ByteArrayInputStream((st.getBytes()), 0, st.getBytes().length)); } catch (Exception e) { - LOG.warn("Error [{}] parsing json text [{}].", e, t); + LOG.warn("Error [{}] parsing json text [{}].", e); throw new SerDeException(e); } } From 93b3489e9b4a774ccac2659a49ddea8ee09c6043 Mon Sep 17 00:00:00 2001 From: kautshukla Date: Tue, 20 Nov 2018 23:43:00 +0530 Subject: [PATCH 3/3] Added msg in exception log --- serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java index b9926715c733..b78e68d3b7c8 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java @@ -123,7 +123,7 @@ public void initialize(Configuration conf, Properties tbl) @Override public Object deserialize(Writable blob) throws SerDeException { - String st; + String st = null; try { if (blob instanceof BytesWritable) { @@ -135,7 +135,7 @@ public Object deserialize(Writable blob) throws SerDeException { return structReader.parseStruct(new ByteArrayInputStream((st.getBytes()), 0, st.getBytes().length)); } catch (Exception e) { - LOG.warn("Error [{}] parsing json text [{}].", e); + LOG.warn("Error [{}] parsing json text [{}].", e, st); throw new SerDeException(e); } }