Skip to content

Commit

Permalink
HIVE-15167 : remove SerDe interface; undeprecate Deserializer and Ser…
Browse files Browse the repository at this point in the history
…ializer (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
  • Loading branch information
sershe-apache committed Nov 15, 2016
1 parent 23bba11 commit 652ed7a
Show file tree
Hide file tree
Showing 60 changed files with 199 additions and 236 deletions.
Expand Up @@ -53,7 +53,7 @@
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
Expand Down Expand Up @@ -148,7 +148,7 @@ public void setConf(Configuration conf) {

@SuppressWarnings("deprecation")
@Override
public Class<? extends SerDe> getSerDeClass() {
public Class<? extends AbstractSerDe> getSerDeClass() {
return AccumuloSerDe.class;
}

Expand Down
Expand Up @@ -28,7 +28,7 @@
import org.apache.hadoop.hive.accumulo.LazyAccumuloRow;
import org.apache.hadoop.hive.accumulo.columns.ColumnMapping;
import org.apache.hadoop.hive.accumulo.columns.HiveAccumuloRowIdColumnMapping;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
Expand All @@ -45,7 +45,7 @@
* Deserialization from Accumulo to LazyAccumuloRow for Hive.
*
*/
public class AccumuloSerDe implements SerDe {
public class AccumuloSerDe extends AbstractSerDe {

private AccumuloSerDeParameters accumuloSerDeParameters;
private LazyAccumuloRow cachedRow;
Expand Down
Expand Up @@ -23,7 +23,7 @@

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
Expand All @@ -36,7 +36,7 @@
*/
public class TestRegexSerDe extends TestCase {

private SerDe createSerDe(String fieldNames, String fieldTypes,
private AbstractSerDe createSerDe(String fieldNames, String fieldTypes,
String inputRegex, String outputFormatString) throws Throwable {
Properties schema = new Properties();
schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
Expand All @@ -55,7 +55,7 @@ private SerDe createSerDe(String fieldNames, String fieldTypes,
public void testRegexSerDe() throws Throwable {
try {
// Create the SerDe
SerDe serDe = createSerDe(
AbstractSerDe serDe = createSerDe(
"host,identity,user,time,request,status,size,referer,agent",
"string,string,string,string,string,string,string,string,string",
"([^ ]*) ([^ ]*) ([^ ]*) (-|\\[[^\\]]*\\]) ([^ \"]*|\"[^\"]*\") "
Expand Down
Expand Up @@ -25,7 +25,7 @@
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
import org.slf4j.Logger;
Expand All @@ -50,7 +50,7 @@ public Class<? extends OutputFormat> getOutputFormatClass() {
}

@Override
public Class<? extends SerDe> getSerDeClass() {
public Class<? extends AbstractSerDe> getSerDeClass() {
return DruidSerDe.class;
}

Expand Down
Expand Up @@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.druid;

import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;

/**
* Storage handler for Druid to be used in tests. It cannot connect to
Expand All @@ -27,7 +27,7 @@
public class QTestDruidStorageHandler extends DruidStorageHandler {

@Override
public Class<? extends SerDe> getSerDeClass() {
public Class<? extends AbstractSerDe> getSerDeClass() {
return QTestDruidSerDe.class;
}

Expand Down
Expand Up @@ -30,7 +30,7 @@
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
Expand Down Expand Up @@ -117,7 +117,7 @@ public HBaseSerDe() throws SerDeException {

/**
* Initialize the SerDe given parameters.
* @see SerDe#initialize(Configuration, Properties)
* @see AbstractSerDe#initialize(Configuration, Properties)
*/
@Override
public void initialize(Configuration conf, Properties tbl)
Expand Down Expand Up @@ -268,7 +268,7 @@ public HBaseSerDeParameters getHBaseSerdeParam() {
* Deserialize a row from the HBase Result writable to a LazyObject
* @param result the HBase Result Writable containing the row
* @return the deserialized object
* @see SerDe#deserialize(Writable)
* @see AbstractSerDe#deserialize(Writable)
*/
@Override
public Object deserialize(Writable result) throws SerDeException {
Expand Down
Expand Up @@ -65,7 +65,7 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
Expand Down Expand Up @@ -315,7 +315,7 @@ public Class<? extends OutputFormat> getOutputFormatClass() {
}

@Override
public Class<? extends SerDe> getSerDeClass() {
public Class<? extends AbstractSerDe> getSerDeClass() {
return HBaseSerDe.class;
}

Expand Down
Expand Up @@ -27,7 +27,7 @@

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
Expand Down Expand Up @@ -56,7 +56,7 @@
@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS,
serdeConstants.LIST_COLUMN_TYPES})

public class HCatRecordSerDe implements SerDe {
public class HCatRecordSerDe extends AbstractSerDe {

private static final Logger LOG = LoggerFactory.getLogger(HCatRecordSerDe.class);

Expand Down
Expand Up @@ -38,7 +38,7 @@
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
Expand Down Expand Up @@ -89,7 +89,7 @@
serdeConstants.LIST_COLUMN_TYPES,
serdeConstants.TIMESTAMP_FORMATS})

public class JsonSerDe implements SerDe {
public class JsonSerDe extends AbstractSerDe {

private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class);
private List<String> columnNames;
Expand Down
Expand Up @@ -22,7 +22,7 @@
import java.io.IOException;

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Writable;
Expand All @@ -39,7 +39,7 @@
class DefaultRecordWriterContainer extends RecordWriterContainer {

private final HiveStorageHandler storageHandler;
private final SerDe serDe;
private final AbstractSerDe serDe;
private final OutputJobInfo jobInfo;
private final ObjectInspector hcatRecordOI;

Expand Down
Expand Up @@ -27,7 +27,7 @@

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Writable;
Expand Down Expand Up @@ -56,7 +56,7 @@ class DynamicPartitionFileRecordWriterContainer extends FileRecordWriterContaine
private int maxDynamicPartitions;

private final Map<String, RecordWriter<? super WritableComparable<?>, ? super Writable>> baseDynamicWriters;
private final Map<String, SerDe> baseDynamicSerDe;
private final Map<String, AbstractSerDe> baseDynamicSerDe;
private final Map<String, org.apache.hadoop.mapred.OutputCommitter> baseDynamicCommitters;
private final Map<String, org.apache.hadoop.mapred.TaskAttemptContext> dynamicContexts;
private final Map<String, ObjectInspector> dynamicObjectInspectors;
Expand All @@ -81,7 +81,7 @@ public DynamicPartitionFileRecordWriterContainer(
+ "HCatOutputFormat. Please make sure that method is called.");
}

this.baseDynamicSerDe = new HashMap<String, SerDe>();
this.baseDynamicSerDe = new HashMap<String, AbstractSerDe>();
this.baseDynamicWriters =
new HashMap<String, RecordWriter<? super WritableComparable<?>, ? super Writable>>();
this.baseDynamicCommitters = new HashMap<String, org.apache.hadoop.mapred.OutputCommitter>();
Expand Down Expand Up @@ -159,7 +159,7 @@ protected LocalFileWriter getLocalFileWriter(HCatRecord value) throws IOExceptio
localJobInfo = HCatBaseOutputFormat.getJobInfo(currTaskContext.getConfiguration());

// Setup serDe.
SerDe currSerDe =
AbstractSerDe currSerDe =
ReflectionUtils.newInstance(storageHandler.getSerDeClass(), currTaskContext.getJobConf());
try {
InternalUtil.initializeOutputSerDe(currSerDe, currTaskContext.getConfiguration(),
Expand Down
Expand Up @@ -29,7 +29,7 @@
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
Expand Down Expand Up @@ -82,8 +82,8 @@ public RecordWriter<WritableComparable<?>, HCatRecord> getRecordWriter(TaskAttem
StorerInfo storeInfo = jobInfo.getTableInfo().getStorerInfo();
HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(
context.getConfiguration(), storeInfo);
Class<? extends SerDe> serde = storageHandler.getSerDeClass();
SerDe sd = (SerDe) ReflectionUtils.newInstance(serde,
Class<? extends AbstractSerDe> serde = storageHandler.getSerDeClass();
AbstractSerDe sd = (AbstractSerDe) ReflectionUtils.newInstance(serde,
context.getConfiguration());
context.getConfiguration().set("mapred.output.value.class",
sd.getSerializedClass().getName());
Expand Down
Expand Up @@ -28,7 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.NullWritable;
Expand All @@ -54,7 +54,7 @@
abstract class FileRecordWriterContainer extends RecordWriterContainer {

protected final HiveStorageHandler storageHandler;
protected final SerDe serDe;
protected final AbstractSerDe serDe;
protected final ObjectInspector objectInspector;

private final List<Integer> partColsToDel;
Expand Down Expand Up @@ -110,7 +110,7 @@ public void write(WritableComparable<?> key, HCatRecord value) throws IOExceptio
LocalFileWriter localFileWriter = getLocalFileWriter(value);
RecordWriter localWriter = localFileWriter.getLocalWriter();
ObjectInspector localObjectInspector = localFileWriter.getLocalObjectInspector();
SerDe localSerDe = localFileWriter.getLocalSerDe();
AbstractSerDe localSerDe = localFileWriter.getLocalSerDe();
OutputJobInfo localJobInfo = localFileWriter.getLocalJobInfo();

for (Integer colToDel : partColsToDel) {
Expand All @@ -129,11 +129,11 @@ public void write(WritableComparable<?> key, HCatRecord value) throws IOExceptio
class LocalFileWriter {
private RecordWriter localWriter;
private ObjectInspector localObjectInspector;
private SerDe localSerDe;
private AbstractSerDe localSerDe;
private OutputJobInfo localJobInfo;

public LocalFileWriter(RecordWriter localWriter, ObjectInspector localObjectInspector,
SerDe localSerDe, OutputJobInfo localJobInfo) {
AbstractSerDe localSerDe, OutputJobInfo localJobInfo) {
this.localWriter = localWriter;
this.localObjectInspector = localObjectInspector;
this.localSerDe = localSerDe;
Expand All @@ -148,7 +148,7 @@ public ObjectInspector getLocalObjectInspector() {
return localObjectInspector;
}

public SerDe getLocalSerDe() {
public AbstractSerDe getLocalSerDe() {
return localSerDe;
}

Expand Down
Expand Up @@ -31,7 +31,7 @@
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
Expand Down Expand Up @@ -62,17 +62,17 @@ public class FosterStorageHandler extends DefaultStorageHandler {

private Class<? extends InputFormat> ifClass;
private Class<? extends OutputFormat> ofClass;
private Class<? extends SerDe> serDeClass;
private Class<? extends AbstractSerDe> serDeClass;

public FosterStorageHandler(String ifName, String ofName, String serdeName) throws ClassNotFoundException {
this((Class<? extends InputFormat>) JavaUtils.loadClass(ifName),
(Class<? extends OutputFormat>) JavaUtils.loadClass(ofName),
(Class<? extends SerDe>) JavaUtils.loadClass(serdeName));
(Class<? extends AbstractSerDe>) JavaUtils.loadClass(serdeName));
}

public FosterStorageHandler(Class<? extends InputFormat> ifClass,
Class<? extends OutputFormat> ofClass,
Class<? extends SerDe> serDeClass) {
Class<? extends AbstractSerDe> serDeClass) {
this.ifClass = ifClass;
this.ofClass = ofClass;
this.serDeClass = serDeClass;
Expand All @@ -89,7 +89,7 @@ public Class<? extends OutputFormat> getOutputFormatClass() {
}

@Override
public Class<? extends SerDe> getSerDeClass() {
public Class<? extends AbstractSerDe> getSerDeClass() {
return serDeClass; //To change body of implemented methods use File | Settings | File Templates.
}

Expand Down
Expand Up @@ -26,7 +26,7 @@
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
Expand Down Expand Up @@ -141,7 +141,7 @@ private static ObjectInspector getObjectInspector(TypeInfo type) throws IOExcept

//TODO this has to find a better home, it's also hardcoded as default in hive would be nice
// if the default was decided by the serde
static void initializeOutputSerDe(SerDe serDe, Configuration conf, OutputJobInfo jobInfo)
static void initializeOutputSerDe(AbstractSerDe serDe, Configuration conf, OutputJobInfo jobInfo)
throws SerDeException {
SerDeUtils.initializeSerDe(serDe, conf,
getSerdeProperties(jobInfo.getTableInfo(),
Expand Down
Expand Up @@ -32,7 +32,7 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.RecordUpdater;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
Expand Down Expand Up @@ -152,7 +152,7 @@ private List<Integer> getBucketColIDs(List<String> bucketCols, List<FieldSchema>
* @return serde
* @throws SerializationError
*/
public abstract SerDe getSerde() throws SerializationError;
public abstract AbstractSerDe getSerde() throws SerializationError;

/**
* Encode a record as an Object that Hive can read with the ObjectInspector associated with the
Expand Down
Expand Up @@ -27,7 +27,7 @@
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
Expand Down Expand Up @@ -270,7 +270,7 @@ public void write(long transactionId, byte[] record)
}

@Override
public SerDe getSerde() {
public AbstractSerDe getSerde() {
return serde;
}

Expand Down

0 comments on commit 652ed7a

Please sign in to comment.