Navigation Menu

Skip to content

Commit

Permalink
removed code lint, got rid of uncecked warnings and deprecation as we…
Browse files Browse the repository at this point in the history 
…ll. No code/functionality change
  • Loading branch information
@rcongiu
rcongiu committed Aug 28, 2016
1 parent bc7fe2a commit ca218a8
Show file tree
Hide file tree
Showing 14 changed files with 75 additions and 104 deletions.
13 changes: 7 additions & 6 deletions json-serde/src/main/java/org/openx/data/jsonserde/JsonSerDe.java
View file
Expand Up @@ -20,7 +20,8 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hive.serde2.SerDe;

import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
Expand Down Expand Up @@ -50,7 +51,7 @@

import javax.print.attribute.standard.DateTimeAtCompleted;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;

/**
Expand All @@ -60,7 +61,7 @@
*
* @author rcongiu
*/
public class JsonSerDe implements SerDe {
public class JsonSerDe extends AbstractSerDe {

public static final Log LOG = LogFactory.getLog(JsonSerDe.class);
List<String> columnNames;
Expand Down Expand Up @@ -94,8 +95,8 @@ public class JsonSerDe implements SerDe {
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
LOG.debug("Initializing SerDe");
// Get column names and sort order
String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS);
String columnTypeProperty = tbl.getProperty(Constants.LIST_COLUMN_TYPES);
String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);

LOG.debug("columns " + columnNameProperty + " types " + columnTypeProperty);

Expand Down Expand Up @@ -125,7 +126,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException
new JsonStructOIOptions(getMappings(tbl));

// Get the sort order
String columnSortOrder = tbl.getProperty(Constants.SERIALIZATION_SORT_ORDER);
String columnSortOrder = tbl.getProperty(serdeConstants.SERIALIZATION_SORT_ORDER);
columnSortOrderIsDesc = new boolean[columnNames.size()];
for (int i = 0; i < columnSortOrderIsDesc.length; i++) {
columnSortOrderIsDesc[i] = columnSortOrder != null &&
Expand Down
7 changes: 4 additions & 3 deletions json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JSONObjectMapAdapter.java
View file
Expand Up @@ -35,7 +35,7 @@
and won't work well for large maps. Good enough for now since
we seldom - if ever - use maps. */
public class JSONObjectMapAdapter implements Map {
HashMap cache;
HashMap<String,Object> cache;
JSONObject jsonObject;

public JSONObjectMapAdapter(JSONObject obj) {
Expand Down Expand Up @@ -69,7 +69,7 @@ public boolean isEmpty() {
}

protected final void initialize() {
if(cache==null) cache = new HashMap();
if(cache==null) cache = new HashMap<String,Object>();

for(Iterator<String> i = jsonObject.keys(); i.hasNext(); ) {
String o = i.next();
Expand Down Expand Up @@ -115,14 +115,15 @@ public Object get(Object key) {

@Override
public Object put(Object key, Object value) {
return cache.put(key,value);
return cache.put(key.toString(),value);
}

@Override
public Object remove(Object key) {
return cache.remove(key);
}

@SuppressWarnings("unchecked")
@Override
public void putAll(Map m) {
cache.putAll(m);
Expand Down
2 changes: 1 addition & 1 deletion ...serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonListObjectInspector.java
View file
Expand Up @@ -36,7 +36,7 @@ public List<?> getList(Object data) {
return null;
}
JSONArray array = (JSONArray) data;
List al = new ArrayList(array.length());
List<Object> al = new ArrayList<Object>(array.length());
for(int i =0; i< array.length(); i++) {
al.add(getListElement(data,i));
}
Expand Down
6 changes: 3 additions & 3 deletions json-serde/src/test/java/org/openx/data/jsonserde/GetJsonObjectTest.java
View file
Expand Up @@ -6,7 +6,7 @@

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.udf.UDFJson;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
Expand Down Expand Up @@ -38,9 +38,9 @@ static public void initialize() throws Exception {
Configuration conf = null;
Properties tbl = new Properties();
// from google video API
tbl.setProperty(Constants.LIST_COLUMNS, "kind,etag,pageInfo,v_items");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "kind,etag,pageInfo,v_items");
tbl.setProperty(
Constants.LIST_COLUMN_TYPES,
serdeConstants.LIST_COLUMN_TYPES,
("string,string," + "string,"
+ "ARRAY<STRUCT<kind:STRING,"
+ "etag:STRING,"
Expand Down
93 changes: 29 additions & 64 deletions json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTest.java
View file
Expand Up @@ -11,44 +11,26 @@
*======================================================================*/
package org.openx.data.jsonserde;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.Map;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import java.util.List;
import java.util.LinkedList;
import java.util.ArrayList;
import org.openx.data.jsonserde.json.JSONArray;
import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.junit.*;
import org.openx.data.jsonserde.json.JSONArray;
import org.openx.data.jsonserde.json.JSONException;
import org.openx.data.jsonserde.json.JSONObject;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBooleanObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.io.Writable;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.openx.data.jsonserde.objectinspector.primitive.*;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.*;

import static org.junit.Assert.*;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringBooleanObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringByteObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringDoubleObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringFloatObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringIntObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringLongObjectInspector;
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringShortObjectInspector;

/**
*
Expand Down Expand Up @@ -81,8 +63,8 @@ public void initialize(JsonSerDe instance) throws Exception {

Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "one,two,three,four");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string");

instance.initialize(conf, tbl);
}
Expand All @@ -92,8 +74,8 @@ public void initialize2(JsonSerDe instance) throws Exception {

Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "one,two,three,four,five");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,string");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,string");

instance.initialize(conf, tbl);
}
Expand Down Expand Up @@ -268,29 +250,14 @@ public void testGetSerializedClass() throws Exception {

/**
* Test of serialize method, of class JsonSerDe.
* @throws org.apache.hadoop.hive.serde2.SerDeException
* @throws org.openx.data.jsonserde.json.JSONException
*/
/* @Test
public void testSerialize() throws Exception {
System.out.println("serialize");
Object o = null;
ObjectInspector oi = null;
JsonSerDe instance = new JsonSerDe();
Writable expResult = null;
Writable result = instance.serialize(o, oi);
assertEquals(expResult, result);
}
*
*/
// @Test
public void testSerialize() throws SerDeException, JSONException, Exception {
System.out.println("serialize");

JsonSerDe instance = new JsonSerDe();
initialize(instance);

ArrayList row = new ArrayList(5);
ArrayList<Object> row = new ArrayList<Object>(5);

List<ObjectInspector> lOi = new LinkedList<ObjectInspector>();
List<String> fieldNames = new LinkedList<String>();
Expand Down Expand Up @@ -350,8 +317,8 @@ public JsonSerDe getMappedSerde() throws SerDeException {
JsonSerDe serde = new JsonSerDe();
Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "one,two,three,four,ts");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,int");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,ts");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,int");
// this means, we call it ts but in data it's 'timestamp'
tbl.setProperty("mapping.ts", "timestamp");

Expand All @@ -364,8 +331,8 @@ public JsonSerDe getNumericSerde() throws SerDeException {
JsonSerDe serde = new JsonSerDe();
Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "cboolean,ctinyint,csmallint,cint,cbigint,cfloat,cdouble");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,int,bigint,float,double");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "cboolean,ctinyint,csmallint,cint,cbigint,cfloat,cdouble");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,int,bigint,float,double");

serde.initialize(conf, tbl);
return serde;
Expand Down Expand Up @@ -470,9 +437,7 @@ public void testNegativeNumbers() throws SerDeException, JSONException {
}

/**
* Test scientific notation with
* @throws SerDeException
* @throws JSONException
* Test scientific notation
*/
@Test
public void testENotationNumbers() throws SerDeException, JSONException {
Expand Down Expand Up @@ -543,7 +508,7 @@ public void testSerializeWithMapping() throws SerDeException, JSONException {
JsonSerDe serde = getMappedSerde();

System.out.println("serialize");
ArrayList row = new ArrayList(5);
ArrayList<Object> row = new ArrayList<Object>(5);

List<ObjectInspector> lOi = new LinkedList<ObjectInspector>();
List<String> fieldNames = new LinkedList<String>();
Expand All @@ -553,12 +518,12 @@ public void testSerializeWithMapping() throws SerDeException, JSONException {
lOi.add(ObjectInspectorFactory.getReflectionObjectInspector(Boolean.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA));

row.add(new Float(43.2));
row.add(43.2f);
fieldNames.add("two");
lOi.add(ObjectInspectorFactory.getReflectionObjectInspector(Float.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA));

List<String> lst = new LinkedList<String>();
final List<String> lst = new LinkedList<String>();
row.add(lst);
fieldNames.add("three");
lOi.add(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory
Expand All @@ -569,7 +534,7 @@ public void testSerializeWithMapping() throws SerDeException, JSONException {
lOi.add(ObjectInspectorFactory.getReflectionObjectInspector(String.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA));

row.add(new Integer(7898));
row.add(7898);
fieldNames.add("ts");
lOi.add(ObjectInspectorFactory.getReflectionObjectInspector(Integer.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA));
Expand Down
8 changes: 3 additions & 5 deletions json-serde/src/test/java/org/openx/data/jsonserde/JsonSerDeTimeStampTest.java
View file
Expand Up @@ -14,7 +14,7 @@
package org.openx.data.jsonserde;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.junit.Before;
Expand Down Expand Up @@ -48,8 +48,8 @@ static public void initialize() throws Exception {
instance = new JsonSerDe();
Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "one,two,three,four,five");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,timestamp");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,timestamp");

instance.initialize(conf, tbl);
}
Expand Down Expand Up @@ -137,6 +137,4 @@ public static Timestamp getDate(String s) throws ParseException {
return ts;

}


}
6 changes: 3 additions & 3 deletions json-serde/src/test/java/org/openx/data/jsonserde/JsonUnionTest.java
View file
@@ -1,7 +1,7 @@
package org.openx.data.jsonserde;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
Expand Down Expand Up @@ -31,8 +31,8 @@ static public void initialize() throws Exception {
Configuration conf = null;
Properties tbl = new Properties();
// from google video API
tbl.setProperty(Constants.LIST_COLUMNS, "country,stuff");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "string,uniontype<int,double,array<string>,struct<a:int,b:string>,string>".toLowerCase());
tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,stuff");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,uniontype<int,double,array<string>,struct<a:int,b:string>,string>".toLowerCase());

instance.initialize(conf, tbl);
}
Expand Down
6 changes: 3 additions & 3 deletions json-serde/src/test/java/org/openx/data/jsonserde/NestedWithMappingTest.java
View file
Expand Up @@ -15,7 +15,7 @@
import java.util.List;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
Expand Down Expand Up @@ -70,8 +70,8 @@ WITH SERDEPROPERTIES ("mapping.useragent" = "User-Agent")
STORED AS TEXTFILE;
*/
tbl.setProperty(Constants.LIST_COLUMNS, "ts,t,request");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, ("string,int," +
tbl.setProperty(serdeConstants.LIST_COLUMNS, "ts,t,request");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ("string,int," +
"struct<path:string,ip:string,headers:struct<useragent:array<string>>>").
toLowerCase());
tbl.setProperty("mapping.useragent" , "User-Agent");
Expand Down
6 changes: 3 additions & 3 deletions json-serde/src/test/java/org/openx/data/jsonserde/ProtobufJsonTest.java
View file
Expand Up @@ -13,7 +13,7 @@
package org.openx.data.jsonserde;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.junit.Test;
Expand All @@ -36,8 +36,8 @@ public void initialize(JsonSerDe instance) throws Exception {

Configuration conf = null;
Properties tbl = new Properties();
tbl.setProperty(Constants.LIST_COLUMNS, "one");
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "string");
tbl.setProperty(serdeConstants.LIST_COLUMNS, "one");
tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string");

instance.initialize(conf, tbl);
}
Expand Down
6 changes: 6 additions & 0 deletions json-serde/src/test/resources/lo4j.properties
View file
@@ -0,0 +1,6 @@
log4j.rootLogger=INFO, A

log4j.appender.A=org.apache.log4j.ConsoleAppender log4j.appender.A.layout=org.apache.log4j.PatternLayout
log4j.appender.A.layout.ConversionPattern=%-5p [%t]: %m%n

log4j.logger.robertmaldon.moneymachine=DEBUG, B

3 comments on commit ca218a8

@prks280
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

json-serde-1.1.6-SNAPSHOT-jar-with-dependencies.jar does not exist

@prks280
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

plz help

@rcongiu
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1.1.6 is an old version.... use a more recent one from here : http://www.congiu.net/hive-json-serde/

Please sign in to comment.