From 83723c780bb0b3a60469b58c88a92c0e85cf82ba Mon Sep 17 00:00:00 2001 From: guophilipse Date: Wed, 22 Feb 2023 10:08:25 +0800 Subject: [PATCH 1/3] support different hive version --- linkis-engineconn-plugins/flink/pom.xml | 1 - linkis-engineconn-plugins/hive/pom.xml | 4 -- .../serde/CustomerDelimitedJSONSerDe.java | 57 ++++++++++++++----- .../service/hive/pom.xml | 1 - pom.xml | 1 + 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index a6c0894a5c..7fe87fb2d2 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -27,7 +27,6 @@ linkis-engineconn-plugin-flink 1.12.2 - 2.3.3 1.3.1 diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index 8fe446167c..28b60fff0a 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -26,10 +26,6 @@ linkis-engineplugin-hive - - 2.3.3 - - diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 671b0c1d19..0acde3f107 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -33,10 +33,12 @@ import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.linkis.common.utils.ClassUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -309,18 +311,6 @@ private static void writePrimitiveUTF8( binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); break; } - case INTERVAL_YEAR_MONTH: - { - wc = ((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveWritableObject(o); - binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); - break; - } - case INTERVAL_DAY_TIME: - { - wc = ((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveWritableObject(o); - binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); - break; - } case DECIMAL: { HiveDecimalObjectInspector decimalOI = (HiveDecimalObjectInspector) oi; @@ -329,7 +319,48 @@ private static void writePrimitiveUTF8( } default: { - throw new RuntimeException("Unknown primitive type: " + category); + boolean containsIntervalYearMonth = false; + boolean containsIntervalDayTime = false; + for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory : + PrimitiveObjectInspector.PrimitiveCategory.values()) { + containsIntervalYearMonth = "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()); + containsIntervalDayTime = "INTERVAL_DAY_TIME".equals(primitiveCategory.name()); + try { + if (containsIntervalYearMonth) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + if (containsIntervalDayTime) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } + } + if (containsIntervalYearMonth || containsIntervalDayTime) { + break; + } else { + throw new RuntimeException("Unknown primitive type: " + category); + } } } if (binaryData == null) { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml index 1279139446..b0c9cdef36 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml @@ -26,7 +26,6 @@ linkis-metadata-query-service-hive - 2.3.3 2.7.2 4.2.4 diff --git a/pom.xml b/pom.xml index 6e42ea39da..030b00f870 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ 1.3.2-SNAPSHOT 2.9.2 2.4.3 + 2.3.3 2.7.2 hadoop-hdfs 2.7.2 From 61b00c51cb7af540f9fb523177ee64c1ddcadcab Mon Sep 17 00:00:00 2001 From: guophilipse Date: Wed, 22 Feb 2023 11:10:29 +0800 Subject: [PATCH 2/3] fix code style --- .../serde/CustomerDelimitedJSONSerDe.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 0acde3f107..41720b4655 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -17,6 +17,8 @@ package org.apache.linkis.engineplugin.hive.serde; +import org.apache.linkis.common.utils.ClassUtils; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; @@ -38,7 +40,6 @@ import java.util.List; import java.util.Map; -import org.apache.linkis.common.utils.ClassUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -322,29 +323,29 @@ private static void writePrimitiveUTF8( boolean containsIntervalYearMonth = false; boolean containsIntervalDayTime = false; for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory : - PrimitiveObjectInspector.PrimitiveCategory.values()) { + PrimitiveObjectInspector.PrimitiveCategory.values()) { containsIntervalYearMonth = "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()); containsIntervalDayTime = "INTERVAL_DAY_TIME".equals(primitiveCategory.name()); try { if (containsIntervalYearMonth) { wc = - (WritableComparable) - ClassUtils.getClassInstance( - "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector") - .getClass() - .getMethod("getPrimitiveWritableObject", Object.class) - .invoke(oi, o); + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); break; } if (containsIntervalDayTime) { wc = - (WritableComparable) - ClassUtils.getClassInstance( - "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector") - .getClass() - .getMethod("getPrimitiveWritableObject", Object.class) - .invoke(oi, o); + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); break; } From a7520a9fc5ec0ec508d5678453665294acf2292f Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 23 Feb 2023 00:15:03 +0800 Subject: [PATCH 3/3] log exception --- .../hive/serde/CustomerDelimitedJSONSerDe.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 41720b4655..9c425ff057 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -349,12 +349,8 @@ private static void writePrimitiveUTF8( binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); break; } - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (InvocationTargetException e) { - e.printStackTrace(); - } catch (NoSuchMethodException e) { - e.printStackTrace(); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + LOG.error("Fail to invoke method:[getPrimitiveWritableObject]!", e); } } if (containsIntervalYearMonth || containsIntervalDayTime) {