From 1fd17ae488e29b35375ed02ef5b3ab99ad4661fc Mon Sep 17 00:00:00 2001 From: "Richard T. Guy" Date: Sat, 30 Mar 2013 11:16:19 -0400 Subject: [PATCH 01/11] Added basic timestamp tracking. --- .../org/openx/data/jsonserde/JsonSerDe.java | 21 +++++--- .../openx/data/jsonserde/JsonSerDeTest.java | 4 +- .../jsonserde/JsonSerDeTimeStampTest.java | 50 +++++++++++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java diff --git a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java index c76d8b7a..098c92b0 100644 --- a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java +++ b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java @@ -13,6 +13,7 @@ package org.openx.data.jsonserde; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -51,6 +52,8 @@ import org.openx.data.jsonserde.json.JSONObject; import org.openx.data.jsonserde.objectinspector.JsonObjectInspectorFactory; +import javax.print.attribute.standard.DateTimeAtCompleted; + /** * Properties: * ignore.malformed.json = true/false : malformed json will be ignored @@ -75,7 +78,7 @@ public class JsonSerDe implements SerDe { // if set, will ignore malformed JSON in deserialization boolean ignoreMalformedJson = false; - public static final String PROP_IGNORE_MALFORMED_JSON = "ignore.malformed.json"; + public static final String PROP_IGNORE_MALFORMED_JSON = "ignore.malformed.json"; /** @@ -111,8 +114,8 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException } assert (columnNames.size() == columnTypes.size()); - stats = new SerDeStats(); - + stats = new SerDeStats(); + // Create row related objects rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); rowObjectInspector = (StructObjectInspector) JsonObjectInspectorFactory.getJsonObjectInspectorFromTypeInfo(rowTypeInfo); @@ -159,6 +162,15 @@ public Object deserialize(Writable w) throws SerDeException { @Override public JSONObject put(String key, Object value) throws JSONException { + + try { + if (rowTypeInfo.getStructFieldTypeInfo(key).getTypeName().equalsIgnoreCase("timestamp")) { + value = Timestamp.valueOf((String)value); + } + } catch (IllegalArgumentException e) { + throw new JSONException("Timestamp " + value + "improperly formatted."); + } + return super.put(key.toLowerCase(), value); } }; @@ -224,9 +236,6 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe * Serializing means getting every field, and setting the appropriate * JSONObject field. Actual serialization is done at the end when * the whole JSON object is built - * @param serializer - * @param obj - * @param structObjectInspector */ private JSONObject serializeStruct( Object obj, StructObjectInspector soi, List columnNames) { diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java index e7d80166..3055d57e 100644 --- a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java @@ -82,11 +82,11 @@ static public void initialize() throws Exception { public void testDeserialize() throws Exception { System.out.println("deserialize"); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); - Object expResult = null; + JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"),"poop"); - assertTrue( result.get("three") instanceof JSONArray); + assertTrue(result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals( ((JSONArray)result.get("three")).get(0),"red"); diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java new file mode 100644 index 00000000..43a3f4c0 --- /dev/null +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java @@ -0,0 +1,50 @@ +package org.openx.data.jsonserde; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.junit.Before; +import org.junit.Test; +import org.openx.data.jsonserde.json.JSONArray; +import org.openx.data.jsonserde.json.JSONObject; + +import java.sql.Timestamp; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * User: guyrt + */ +public class JsonSerDeTimeStampTest { + + static JsonSerDe instance; + + @Before + public void setUp() throws Exception { + initialize(); + } + + static public void initialize() throws Exception { + instance = new JsonSerDe(); + Configuration conf = null; + Properties tbl = new Properties(); + tbl.setProperty(Constants.LIST_COLUMNS, "one,two,three,four,five"); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,float,array,string,timestamp"); + + instance.initialize(conf, tbl); + } + + @Test + public void testTimestampDeSerialize() throws Exception { + // Test that timestamp object can be deserialized + Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40\"}"); + + JSONObject result = (JSONObject) instance.deserialize(w); + assertEquals(result.get("five"), Timestamp.valueOf("2013-03-27 23:18:40.0")); + } + + +} From e2e8a8dbeb8d1d23109cf80ad2c3f7303ef33899 Mon Sep 17 00:00:00 2001 From: "Richard T. Guy" Date: Sat, 30 Mar 2013 12:25:12 -0400 Subject: [PATCH 02/11] Added check for missing fields. --- .../org/openx/data/jsonserde/JsonSerDe.java | 18 +++++++++--------- .../openx/data/jsonserde/JsonSerDeTest.java | 13 +++++++++++++ .../data/jsonserde/JsonSerDeTimeStampTest.java | 9 +++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java index 098c92b0..b11920b9 100644 --- a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java +++ b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java @@ -148,6 +148,7 @@ public Object deserialize(Writable w) throws SerDeException { // Try parsing row into JSON object JSONObject jObj = null; + try { jObj = new JSONObject(rowText.toString()) { @@ -160,15 +161,14 @@ public Object deserialize(Writable w) throws SerDeException { * java.lang.Object) */ @Override - public JSONObject put(String key, Object value) - throws JSONException { + public JSONObject put(String key, Object value) throws JSONException { try { - if (rowTypeInfo.getStructFieldTypeInfo(key).getTypeName().equalsIgnoreCase("timestamp")) { - value = Timestamp.valueOf((String)value); - } + if (columnNames.contains(key) && rowTypeInfo.getStructFieldTypeInfo(key).getTypeName().equalsIgnoreCase("timestamp")) { + value = Timestamp.valueOf((String)value); + } } catch (IllegalArgumentException e) { - throw new JSONException("Timestamp " + value + "improperly formatted."); + throw new JSONException("Timestamp " + value + "improperly formatted."); } return super.put(key.toLowerCase(), value); @@ -195,7 +195,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { /** * We serialize to Text - * @return + * @return * * @see org.apache.hadoop.io.Text */ @@ -228,7 +228,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe Text t = new Text(serializer.toString()); - serializedDataSize = t.getBytes().length; + serializedDataSize = t.getBytes().length; return t; } @@ -394,7 +394,7 @@ public void onMalformedJson(String msg) throws SerDeException { @Override public SerDeStats getSerDeStats() { - if(lastOperationSerialize) { + if(lastOperationSerialize) { stats.setRawDataSize(serializedDataSize); } else { stats.setRawDataSize(deserializedDataSize); diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java index 3055d57e..a415a9f3 100644 --- a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java @@ -107,6 +107,19 @@ public void testDeserialize2() throws Exception { assertEquals( ((JSONArray)result.get("three")).get(0),"red"); } + @Test + public void testDeserializePartialFieldSet() throws Exception { + Writable w = new Text("{\"missing\":\"whocares\",\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); + Object expResult = null; + JSONObject result = (JSONObject) instance.deserialize(w); + assertEquals(result.get("four"),"poop"); + + assertTrue( result.get("three") instanceof JSONArray); + + assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); + assertEquals( ((JSONArray)result.get("three")).get(0),"red"); + } + /** * Test of getSerializedClass method, of class JsonSerDe. */ diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java index 43a3f4c0..62d15a95 100644 --- a/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java @@ -46,5 +46,14 @@ public void testTimestampDeSerialize() throws Exception { assertEquals(result.get("five"), Timestamp.valueOf("2013-03-27 23:18:40.0")); } + @Test + public void testTimestampDeSerializeWithNanoseconds() throws Exception { + // Test that timestamp object can be deserialized + Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40.123456\"}"); + + JSONObject result = (JSONObject) instance.deserialize(w); + assertEquals(result.get("five"), Timestamp.valueOf("2013-03-27 23:18:40.123456")); + } + } From 339469ad5b1a69629dca68c021ea0f27fba3291e Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Sun, 5 May 2013 18:32:45 -0700 Subject: [PATCH 03/11] Merged timestamp support, plus added support for numeric timestamps --- .../org/openx/data/jsonserde/JsonSerDe.java | 26 ++++++++++++++++--- .../openx/data/jsonserde/JsonSerDeTest.java | 8 +----- .../jsonserde/JsonSerDeTimeStampTest.java | 18 +++++++++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java index 204dfcbb..9e0c21ad 100644 --- a/src/main/java/org/openx/data/jsonserde/JsonSerDe.java +++ b/src/main/java/org/openx/data/jsonserde/JsonSerDe.java @@ -20,8 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -55,6 +53,9 @@ import org.openx.data.jsonserde.objectinspector.JsonStructOIOptions; import javax.print.attribute.standard.DateTimeAtCompleted; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; /** * Properties: @@ -173,8 +174,23 @@ public Object deserialize(Writable w) throws SerDeException { public JSONObject put(String key, Object value) throws JSONException { try { - if (columnNames.contains(key) && rowTypeInfo.getStructFieldTypeInfo(key).getTypeName().equalsIgnoreCase("timestamp")) { - value = Timestamp.valueOf((String)value); + if (columnNames.contains(key) && + rowTypeInfo.getStructFieldTypeInfo(key).getCategory().equals(PrimitiveObjectInspector.Category.PRIMITIVE) && + ((PrimitiveTypeInfo) rowTypeInfo.getStructFieldTypeInfo(key)) + .getPrimitiveCategory().equals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) ) { + if(value instanceof String) { + value = Timestamp.valueOf((String)value); + } else if (value instanceof Float ) { + value = new Timestamp( (long) (((Float)value).floatValue() * 1000)); + } else if ( value instanceof Integer) { + value = new Timestamp( ((Integer)value).longValue() * 1000); + } else if ( value instanceof Long) { + value = new Timestamp( ((Long)value).longValue() * 1000); + } else if ( value instanceof Double) { + value = new Timestamp( ((Double)value).longValue() * 1000); + } else { + throw new JSONException("I don't know how to conver to timestamp a field of type " + value.getClass()) ; + } } } catch (IllegalArgumentException e) { throw new JSONException("Timestamp " + value + "improperly formatted."); @@ -444,5 +460,7 @@ private Map getMappings(Properties tbl) { } return mps; } + + } diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java index 08f8b228..1eb09be3 100644 --- a/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java @@ -90,18 +90,12 @@ public void testDeserialize() throws Exception { JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"),"poop"); - assertTrue(result.get("three") instanceof JSONArray); + assertTrue( result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals( ((JSONArray)result.get("three")).get(0),"red"); } - assertTrue(result.get("three") instanceof JSONArray); - - assertTrue(((JSONArray) result.get("three")).get(0) instanceof String); - assertEquals(((JSONArray) result.get("three")).get(0), "red"); - } - // {"one":true,"three":["red","yellow",["blue","azure","cobalt","teal"],"orange"],"two":19.5,"four":"poop"} @Test public void testDeserialize2() throws Exception { diff --git a/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java index 62d15a95..30134638 100644 --- a/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java +++ b/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java @@ -54,6 +54,24 @@ public void testTimestampDeSerializeWithNanoseconds() throws Exception { JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("five"), Timestamp.valueOf("2013-03-27 23:18:40.123456")); } + + @Test + public void testTimestampDeSerializeNumericTimestamp() throws Exception { + // Test that timestamp object can be deserialized + Writable w = new Text("{\"one\":true,\"five\":1367801925}"); + + JSONObject result = (JSONObject) instance.deserialize(w); + assertEquals(result.get("five"), Timestamp.valueOf("2013-05-05 17:58:45.0") ); + } + + @Test + public void testTimestampDeSerializeNumericTimestampWithNanoseconds() throws Exception { + // Test that timestamp object can be deserialized + Writable w = new Text("{\"one\":true,\"five\":1367801925.123}"); +// + JSONObject result = (JSONObject) instance.deserialize(w); + assertEquals(result.get("five"), Timestamp.valueOf("2013-05-05 17:58:45.0")); + } } From d58e72ded4d9eb11a4dbfccef858e3c9dc46da38 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 10 Jul 2013 09:10:59 -0700 Subject: [PATCH 04/11] gitflow release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a81474d9..2f74057c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.openx.data json-serde - 1.1.6-SNAPSHOT + 1.1.6 jar openx-json-serde From 648dd297d9e68813b1ab4fe98f5bbd8ecd667e19 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 10 Jul 2013 22:18:59 -0700 Subject: [PATCH 05/11] merged issue #28 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f74057c..c02b1c8e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.openx.data json-serde - 1.1.6 + 1.1.7-SNAPSHOT jar openx-json-serde From 53dffb5de8bc5b44e48a373b9b531fcf8cec0b2b Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 10 Jul 2013 22:34:00 -0700 Subject: [PATCH 06/11] updated readme --- README.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/README.txt b/README.txt index 886d8919..50cc694f 100644 --- a/README.txt +++ b/README.txt @@ -134,6 +134,7 @@ Versions: 1.1.2 (2012/07/26): Fixed issue with columns that are not mapped into JSON, reported by Michael Phung 1.1.4 (2012/10/04): Fixed issue #13, problem with floats, Reported by Chuck Connell 1.1.6 (2013/07/10): Fixed issue #28, error after 'alter table add columns' +1.1.7 (TBD) : Fixed issue #25, timestamp support From cfeaf208c977cba8780032cb94fdba734d39bbf1 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 7 Aug 2013 07:18:54 -0700 Subject: [PATCH 07/11] removed static list used to deserialize json struct --- .../objectinspector/JsonStructObjectInspector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/openx/data/jsonserde/objectinspector/JsonStructObjectInspector.java b/src/main/java/org/openx/data/jsonserde/objectinspector/JsonStructObjectInspector.java index efa1fc64..5db3a364 100644 --- a/src/main/java/org/openx/data/jsonserde/objectinspector/JsonStructObjectInspector.java +++ b/src/main/java/org/openx/data/jsonserde/objectinspector/JsonStructObjectInspector.java @@ -15,7 +15,6 @@ import java.util.List; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.LOG; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.openx.data.jsonserde.json.JSONException; import org.openx.data.jsonserde.json.JSONObject; @@ -100,7 +99,7 @@ public Object getStructFieldDataFromJsonObject(JSONObject data, StructField fiel } - static List values = new ArrayList(); + /** * called to map from hive to json @@ -114,7 +113,8 @@ protected String getJsonField(StructField fr) { return fr.getFieldName(); } } - + + List values = new ArrayList(); @Override public List getStructFieldsDataAsList(Object o) { JSONObject jObj = (JSONObject) o; From 8ad055d979d8e1d901fad206054dc51d1b205db1 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 7 Aug 2013 07:36:19 -0700 Subject: [PATCH 08/11] updated docs --- README.txt => README.md | 36 +++++++++++++++++++++++++----------- pom.xml | 8 ++++---- 2 files changed, 29 insertions(+), 15 deletions(-) rename README.txt => README.md (89%) diff --git a/README.txt b/README.md similarity index 89% rename from README.txt rename to README.md index 50cc694f..54f7a4a7 100644 --- a/README.txt +++ b/README.md @@ -1,4 +1,6 @@ JsonSerde - a read/write SerDe for JSON Data +================================================ + AUTHOR: Roberto Congiu Serialization/Deserialization module for Apache Hadoop Hive @@ -12,22 +14,27 @@ Features: * nested data structures are also supported. COMPILE +--------- Use maven to compile the serde. +```bash $ mvn package -If you want to compile the serde against a different version of the cloudera libs, -use -D: -mvn -Dcdh.version=0.9.0-cdh3u4c-SNAPSHOT package +# If you want to compile the serde against a different +# version of the cloudera libs, use -D: +$ mvn -Dcdh.version=0.9.0-cdh3u4c-SNAPSHOT package +``` EXAMPLES +------------ Example scripts with simple sample data are in src/test/scripts. Here some excerpts: -* Query with complex fields like arrays +### Query with complex fields like arrays +```sql CREATE TABLE json_test1 ( one boolean, three array, @@ -41,11 +48,14 @@ hive> select three[1] from json_test1; gold yellow +``` -* Nested structures +### Nested structures You can also define nested structures: + +```sql add jar ../../../target/json-serde-1.0-SNAPSHOT-jar-with-dependencies.jar; CREATE TABLE json_nested_test ( @@ -61,8 +71,9 @@ LOAD DATA LOCAL INPATH 'nesteddata.txt' OVERWRITE INTO TABLE json_nested_test ; select * from json_nested_test; -- result: Switzerland ["German","French","Italian"] {"catholic":[10,20],"protestant":[40,50]} select languages[0] from json_nested_test; -- result: German select religions['catholic'][0] from json_nested_test; -- result: 10 +``` -* MALFORMED DATA +### MALFORMED DATA The default behavior on malformed data is throwing an exception. For example, for malformed json like @@ -72,12 +83,14 @@ you get: Failed with exception java.io.IOException:org.apache.hadoop.hive.serde2.SerDeException: Row is not a valid JSON Object - JSONException: Expected a ':' after a key at 32 [character 33 line 1] this may not be desirable if you have a few bad lines you wish to ignore. If so you can do: +```sql ALTER TABLE json_table SET SERDEPROPERTIES ( "ignore.malformed.json" = "true"); +``` it will not make the query fail, and the above record will be returned as NULL null null -* MAPPING HIVE KEYWORDS +### MAPPING HIVE KEYWORDS Sometimes it may happen that JSON data has attributes named like reserved words in hive. For instance, you may have a JSON attribute named 'timestamp', which is a reserved word @@ -95,7 +108,7 @@ Notice the "mapping.ts", that means: take the column 'ts' and read into it the JSON attribute named "timestamp" -# ARCHITECTURE +### ARCHITECTURE For the JSON encoding/decoding, I am using a modified version of Douglas Crockfords JSON library: https://github.com/douglascrockford/JSON-java @@ -115,12 +128,12 @@ match hive table declaration. More detailed explanation on my blog: http://www.congiu.com/articles/json_serde -# CONTRIBUTING +### CONTRIBUTING I am using gitflow for the release cycle. -* THANKS +### THANKS Thanks to Douglas Crockford for the liberal license for his JSON library, and thanks to my employer OpenX and my boss Michael Lum for letting me open source the code. @@ -134,7 +147,8 @@ Versions: 1.1.2 (2012/07/26): Fixed issue with columns that are not mapped into JSON, reported by Michael Phung 1.1.4 (2012/10/04): Fixed issue #13, problem with floats, Reported by Chuck Connell 1.1.6 (2013/07/10): Fixed issue #28, error after 'alter table add columns' -1.1.7 (TBD) : Fixed issue #25, timestamp support +1.1.7 (TBD) : Fixed issue #25, timestamp support, fix parametrized build, + Fixed issue #31 (static member shouldn't be static) diff --git a/pom.xml b/pom.xml index c02b1c8e..10467778 100644 --- a/pom.xml +++ b/pom.xml @@ -86,15 +86,15 @@ test - org.apache.hadoop.hive + org.apache.hive hive-serde - 0.8.0-cdh4a1-SNAPSHOT + ${cdh.version} provided - org.apache.hadoop.hive + org.apache.hive hive-exec - 0.8.0-cdh4a1-SNAPSHOT + ${cdh.version} provided From b2e6113840ec982ca7327ae0f072a48219f70df7 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 7 Aug 2013 07:45:12 -0700 Subject: [PATCH 09/11] updated docs --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 54f7a4a7..b9b3d9a8 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,9 @@ CREATE TABLE json_nested_test ( ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' STORED AS TEXTFILE; --- data : {"country":"Switzerland","languages":["German","French","Italian"],"religions":{"catholic":[10,20],"protestant":[40,50]}} +-- data : {"country":"Switzerland","languages":["German","French", +-- "Italian"],"religions":{"catholic":[10,20],"protestant":[40,50]}} + LOAD DATA LOCAL INPATH 'nesteddata.txt' OVERWRITE INTO TABLE json_nested_test ; select * from json_nested_test; -- result: Switzerland ["German","French","Italian"] {"catholic":[10,20],"protestant":[40,50]} From f40e1bc280c1c91c99becdd12dd349d7736b36fd Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Wed, 7 Aug 2013 07:46:45 -0700 Subject: [PATCH 10/11] updated docs --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b9b3d9a8..f54872f4 100644 --- a/README.md +++ b/README.md @@ -143,13 +143,13 @@ my employer OpenX and my boss Michael Lum for letting me open source the code. Versions: -1.0: initial release -1.1: fixed some string issues -1.1.1 (2012/07/03): fixed Map Adapter (get and put would call themselves...ooops) -1.1.2 (2012/07/26): Fixed issue with columns that are not mapped into JSON, reported by Michael Phung -1.1.4 (2012/10/04): Fixed issue #13, problem with floats, Reported by Chuck Connell -1.1.6 (2013/07/10): Fixed issue #28, error after 'alter table add columns' -1.1.7 (TBD) : Fixed issue #25, timestamp support, fix parametrized build, +* 1.0: initial release +* 1.1: fixed some string issues +* 1.1.1 (2012/07/03): fixed Map Adapter (get and put would call themselves...ooops) +* 1.1.2 (2012/07/26): Fixed issue with columns that are not mapped into JSON, reported by Michael Phung +* 1.1.4 (2012/10/04): Fixed issue #13, problem with floats, Reported by Chuck Connell +* 1.1.6 (2013/07/10): Fixed issue #28, error after 'alter table add columns' +* 1.1.7 (TBD) : Fixed issue #25, timestamp support, fix parametrized build, Fixed issue #31 (static member shouldn't be static) From 1a4a503ec901c962d1d6c225eb1997898b79d129 Mon Sep 17 00:00:00 2001 From: Roberto Congiu Date: Mon, 30 Sep 2013 09:25:02 -0700 Subject: [PATCH 11/11] release 1.1.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 10467778..f7ff815b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.openx.data json-serde - 1.1.7-SNAPSHOT + 1.1.7 jar openx-json-serde