From e375b21a82e7d294f2dcdc7a3e9a351602488c1c Mon Sep 17 00:00:00 2001 From: Viraj Bhat Date: Fri, 26 Apr 2013 15:36:38 -0700 Subject: [PATCH] Intial Patch for HIVE-4331 --- build.properties | 2 +- hbase-handler/ivy.xml | 47 +- .../hive/hbase/HBaseStorageHandler.java | 101 ++- .../hbase/HiveHBaseTableOutputFormat.java | 76 +-- .../src/test/queries/positive/hbase_bulk.m | 4 +- .../results/positive/external_table_ppd.q.out | 2 +- .../hbase_binary_storage_queries.q.out | 4 +- hcatalog/build.properties | 5 +- hcatalog/build.xml | 14 - .../cli/SemanticAnalysis/CreateTableHook.java | 4 +- .../org/apache/hcatalog/common/HCatUtil.java | 37 +- .../DefaultRecordWriterContainer.java | 3 +- .../FileOutputCommitterContainer.java | 3 +- .../mapreduce/FileOutputFormatContainer.java | 3 +- .../mapreduce/FileRecordWriterContainer.java | 5 +- .../mapreduce/FosterStorageHandler.java | 15 +- .../mapreduce/HCatBaseInputFormat.java | 5 +- .../mapreduce/HCatBaseOutputFormat.java | 13 +- .../hcatalog/mapreduce/HCatOutputFormat.java | 3 +- .../hcatalog/mapreduce/HCatRecordReader.java | 9 +- .../mapreduce/HCatStorageHandler.java | 4 +- .../hcatalog/mapreduce/InitializeInput.java | 3 +- .../apache/hcatalog/mapreduce/PartInfo.java | 3 +- ...torageDelegationAuthorizationProvider.java | 5 +- .../hcatalog/cli/DummyStorageHandler.java | 289 -------- .../org/apache/hcatalog/pig/HCatLoader.java | 1 + hcatalog/pom.xml | 4 +- .../hadoop/mapred/TempletonJobTracker.java | 10 +- hcatalog/storage-handlers/hbase/build.xml | 76 +-- .../hbase/conf/revision-manager-site.xml | 24 - .../hbase/if/transaction.thrift | 30 - hcatalog/storage-handlers/hbase/pom.xml | 141 +++- .../thrift/StoreFamilyRevision.java | 416 ------------ .../thrift/StoreFamilyRevisionList.java | 369 ---------- .../hbase/HBaseAuthorizationProvider.java | 144 ---- .../hcatalog/hbase/HBaseBaseOutputFormat.java | 76 --- .../hcatalog/hbase/HBaseBulkOutputFormat.java | 221 ------ .../apache/hcatalog/hbase/HBaseConstants.java | 41 -- .../hbase/HBaseDirectOutputFormat.java | 167 ----- .../hbase/HBaseHCatStorageHandler.java | 610 ----------------- .../hcatalog/hbase/HBaseInputFormat.java | 126 ---- .../hbase/HBaseRevisionManagerUtil.java | 257 ------- .../org/apache/hcatalog/hbase/HBaseUtil.java | 159 ----- .../hcatalog/hbase/HCatTableSnapshot.java | 92 --- .../hbase/HbaseSnapshotRecordReader.java | 255 ------- .../hcatalog/hbase/ImportSequenceFile.java | 251 ------- .../hcatalog/hbase/ResultConverter.java | 58 -- .../hbase/snapshot/FamilyRevision.java | 71 -- .../hcatalog/hbase/snapshot/IDGenerator.java | 145 ---- .../hcatalog/hbase/snapshot/PathUtil.java | 132 ---- .../hcatalog/hbase/snapshot/RMConstants.java | 30 - .../hbase/snapshot/RevisionManager.java | 148 ---- .../RevisionManagerConfiguration.java | 59 -- .../snapshot/RevisionManagerEndpoint.java | 141 ---- .../RevisionManagerEndpointClient.java | 125 ---- .../snapshot/RevisionManagerFactory.java | 105 --- .../snapshot/RevisionManagerProtocol.java | 30 - .../hbase/snapshot/TableSnapshot.java | 90 --- .../hcatalog/hbase/snapshot/Transaction.java | 116 ---- .../snapshot/ZKBasedRevisionManager.java | 461 ------------- .../hcatalog/hbase/snapshot/ZKUtil.java | 519 -------------- .../hbase/snapshot/lock/LockListener.java | 41 -- .../hbase/snapshot/lock/ProtocolSupport.java | 195 ------ .../hbase/snapshot/lock/WriteLock.java | 303 --------- .../hbase/snapshot/lock/ZNodeName.java | 113 ---- .../snapshot/lock/ZooKeeperOperation.java | 41 -- .../hcatalog/hbase/snapshot/package-info.java | 28 - .../resources/revision-manager-default.xml | 46 -- .../hcatalog/hbase/SkeletonHBaseTest.java | 2 - .../hbase/TestHBaseBulkOutputFormat.java | 631 ------------------ .../hbase/TestHBaseDirectOutputFormat.java | 501 -------------- .../hcatalog/hbase/TestHBaseInputFormat.java | 398 ++--------- ....java => TestHiveHBaseStorageHandler.java} | 105 ++- .../hbase/TestHiveHBaseTableOutputFormat.java | 352 ++++++++++ .../hbase/TestPigHBaseStorageHandler.java | 370 ++++++++++ .../apache/hcatalog/hbase/TestSnapshots.java | 141 ---- .../hcatalog/hbase/snapshot/IDGenClient.java | 72 -- .../hbase/snapshot/TestIDGenerator.java | 99 --- .../hbase/snapshot/TestRevisionManager.java | 260 -------- .../TestRevisionManagerConfiguration.java | 34 - .../snapshot/TestRevisionManagerEndpoint.java | 206 ------ .../snapshot/TestThriftSerialization.java | 85 --- .../hbase/snapshot/TestZNodeSetUp.java | 120 ---- .../hbase/snapshot/lock/WriteLockTest.java | 161 ----- .../hbase/snapshot/lock/ZNodeNameTest.java | 62 -- ivy/libraries.properties | 2 +- .../hadoop/hive/ql/exec/FileSinkOperator.java | 17 +- .../hive/ql/io/HiveFileFormatUtils.java | 58 +- .../ql/io/HivePassThroughOutputFormat.java | 121 ++++ .../ql/io/HivePassThroughRecordWriter.java | 47 ++ .../hadoop/hive/ql/metadata/Partition.java | 4 +- .../apache/hadoop/hive/ql/metadata/Table.java | 12 +- .../hadoop/hive/ql/plan/CreateTableDesc.java | 2 +- .../hadoop/hive/ql/plan/PartitionDesc.java | 4 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 7 + .../apache/hadoop/hive/ql/plan/TableDesc.java | 12 +- 96 files changed, 1492 insertions(+), 9519 deletions(-) delete mode 100644 hcatalog/storage-handlers/hbase/conf/revision-manager-site.xml delete mode 100644 hcatalog/storage-handlers/hbase/if/transaction.thrift delete mode 100644 hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java delete mode 100644 hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java delete mode 100644 hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java delete mode 100644 hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java rename hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/{TestHBaseHCatStorageHandler.java => TestHiveHBaseStorageHandler.java} (66%) create mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseTableOutputFormat.java create mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestPigHBaseStorageHandler.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/WriteLockTest.java delete mode 100644 hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/ZNodeNameTest.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java diff --git a/build.properties b/build.properties index 70b8a12a561b..5b374aa3e716 100644 --- a/build.properties +++ b/build.properties @@ -29,7 +29,7 @@ javac.args.warnings= hadoop-0.20.version=0.20.2 hadoop-0.20S.version=1.1.2 -hadoop-0.23.version=2.0.3-alpha +hadoop-0.23.version=0.23.6 hadoop.version=${hadoop-0.20.version} hadoop.security.version=${hadoop-0.20S.version} # Used to determine which set of Hadoop artifacts we depend on. diff --git a/hbase-handler/ivy.xml b/hbase-handler/ivy.xml index 7be86492d412..b626235bd61c 100644 --- a/hbase-handler/ivy.xml +++ b/hbase-handler/ivy.xml @@ -28,18 +28,59 @@ + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index 77ff8240186e..ec37104cfcf1 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; @@ -34,7 +35,9 @@ import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -45,6 +48,7 @@ import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer; import org.apache.hadoop.hive.ql.index.IndexSearchCondition; import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; @@ -55,6 +59,11 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.TBase; +import org.apache.zookeeper.ZooKeeper; + +import com.facebook.fb303.FacebookBase; +import com.google.common.util.concurrent.ThreadFactoryBuilder; /** * HBaseStorageHandler provides a HiveStorageHandler implementation for @@ -65,6 +74,10 @@ public class HBaseStorageHandler extends DefaultStorageHandler final static public String DEFAULT_PREFIX = "default."; + //Check if the configure job properties is called from input or output for setting asymmetric properties + String CONFIGURE_JOB_PROPERTIES = "INPUT"; + + private Configuration jobConf; private Configuration hbaseConf; private HBaseAdmin admin; @@ -85,11 +98,16 @@ private String getHBaseTableName(Table tbl) { // for backwards compatibility with the original specs). String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME); if (tableName == null) { + //convert to lower case in case we are getting from serde tableName = tbl.getSd().getSerdeInfo().getParameters().get( HBaseSerDe.HBASE_TABLE_NAME); + //standardize to lower case + if (tableName != null) { + tableName = tableName.toLowerCase(); + } } if (tableName == null) { - tableName = tbl.getDbName() + "." + tbl.getTableName(); + tableName = (tbl.getDbName() + "." + tbl.getTableName()).toLowerCase(); if (tableName.startsWith(DEFAULT_PREFIX)) { tableName = tableName.substring(DEFAULT_PREFIX.length()); } @@ -230,8 +248,13 @@ public Configuration getConf() { return hbaseConf; } + public Configuration getJobConf() { + return jobConf; + } + @Override public void setConf(Configuration conf) { + jobConf = conf; hbaseConf = HBaseConfiguration.create(conf); } @@ -242,7 +265,7 @@ public Class getInputFormatClass() { @Override public Class getOutputFormatClass() { - return HiveHBaseTableOutputFormat.class; + return org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat.class; } @Override @@ -257,16 +280,18 @@ public HiveMetaHook getMetaHook() { @Override public void configureInputJobProperties( - TableDesc tableDesc, - Map jobProperties) { - configureTableJobProperties(tableDesc, jobProperties); + TableDesc tableDesc, + Map jobProperties) { + CONFIGURE_JOB_PROPERTIES = "INPUT"; + configureTableJobProperties(tableDesc, jobProperties); } @Override public void configureOutputJobProperties( - TableDesc tableDesc, - Map jobProperties) { - configureTableJobProperties(tableDesc, jobProperties); + TableDesc tableDesc, + Map jobProperties) { + CONFIGURE_JOB_PROPERTIES = "OUTPUT"; + configureTableJobProperties(tableDesc, jobProperties); } @Override @@ -287,14 +312,70 @@ public void configureTableJobProperties( if (tableName == null) { tableName = tableProperties.getProperty(hive_metastoreConstants.META_TABLE_NAME); + tableName = tableName.toLowerCase(); if (tableName.startsWith(DEFAULT_PREFIX)) { tableName = tableName.substring(DEFAULT_PREFIX.length()); } } jobProperties.put(HBaseSerDe.HBASE_TABLE_NAME, tableName); + + // do this for reconciling HBaseStorageHandler for use in HCatalog + // check to see if this an input job or an outputjob + if (CONFIGURE_JOB_PROPERTIES == "INPUT") { + try { + Configuration jobConf = getJobConf(); + addResources(jobConf, jobProperties); + JobConf copyOfConf = new JobConf(jobConf); + HBaseConfiguration.addHbaseResources(copyOfConf); + //Getting hbase delegation token in getInputSplits does not work with PIG. + if (jobConf instanceof JobConf) { //Should be the case + addHBaseDelegationToken(copyOfConf); + ((JobConf)jobConf).getCredentials().addAll(copyOfConf.getCredentials()); + } + }//try + catch (IOException e) { + throw new IllegalStateException("Error while configuring input job properties", e); + } + } + else { + //find out a method of populating HCat specific info here + Map tableJobProperties = tableDesc.getJobProperties(); + Configuration jobConf = getJobConf(); + addResources(jobConf, jobProperties); + JobConf copyOfConf = new JobConf(jobConf); + HBaseConfiguration.addHbaseResources(copyOfConf); + jobProperties.put(TableOutputFormat.OUTPUT_TABLE, tableName); + } } - @Override + /** + * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map + * if they are not already present in the jobConf. + * @param jobConf Job configuration + * @param newJobProperties Map to which new properties should be added + */ + private void addResources(Configuration jobConf, + Map newJobProperties) { + Configuration conf = new Configuration(false); + HBaseConfiguration.addHbaseResources(conf); + for (Entry entry : conf) { + if (jobConf.get(entry.getKey()) == null) { + newJobProperties.put(entry.getKey(), entry.getValue()); + } + } + } + + private void addHBaseDelegationToken(JobConf job) throws IOException { + if (User.isHBaseSecurityEnabled(job)) { + try { + User.getCurrent().obtainAuthTokenForJob(job); + } catch (InterruptedException e) { + throw new IOException("Error while obtaining hbase delegation token", e); + } + } + } + +@Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { try { TableMapReduceUtil.addDependencyJars(jobConf); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java index 2e2a80ed4008..3d152062e670 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java @@ -19,27 +19,26 @@ package org.apache.hadoop.hive.hbase; import java.io.IOException; -import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; +import org.apache.hadoop.hbase.mapreduce.TableOutputCommitter; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; -import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.util.Progressable; /** @@ -49,7 +48,7 @@ */ public class HiveHBaseTableOutputFormat extends TableOutputFormat implements - HiveOutputFormat, + //HiveOutputFormat, OutputFormat { static final Log LOG = LogFactory.getLog(HiveHBaseTableOutputFormat.class); @@ -66,39 +65,7 @@ public class HiveHBaseTableOutputFormat extends * @param progress progress used for status report * @return the RecordWriter for the output file */ - @Override - public RecordWriter getHiveRecordWriter( - JobConf jc, - Path finalOutPath, - Class valueClass, - boolean isCompressed, - Properties tableProperties, - final Progressable progressable) throws IOException { - String hbaseTableName = jc.get(HBaseSerDe.HBASE_TABLE_NAME); - jc.set(TableOutputFormat.OUTPUT_TABLE, hbaseTableName); - final boolean walEnabled = HiveConf.getBoolVar( - jc, HiveConf.ConfVars.HIVE_HBASE_WAL_ENABLED); - final HTable table = new HTable(HBaseConfiguration.create(jc), hbaseTableName); - table.setAutoFlush(false); - - return new RecordWriter() { - - @Override - public void close(boolean abort) throws IOException { - if (!abort) { - table.flushCommits(); - } - } - - @Override - public void write(Writable w) throws IOException { - Put put = (Put) w; - put.setWriteToWAL(walEnabled); - table.put(put); - } - }; - } @Override public void checkOutputSpecs(FileSystem fs, JobConf jc) throws IOException { @@ -127,6 +94,37 @@ public void checkOutputSpecs(FileSystem fs, JobConf jc) throws IOException { String name, Progressable progressable) throws IOException { - throw new RuntimeException("Error: Hive should not invoke this method."); + String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); + jobConf.set(TableOutputFormat.OUTPUT_TABLE, hbaseTableName); + final boolean walEnabled = HiveConf.getBoolVar( + jobConf, HiveConf.ConfVars.HIVE_HBASE_WAL_ENABLED); + final HTable table = new HTable(HBaseConfiguration.create(jobConf), hbaseTableName); + table.setAutoFlush(false); + return new MyRecordWriter(table); + } + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException, + InterruptedException { + return new TableOutputCommitter(); +} + + + static private class MyRecordWriter implements org.apache.hadoop.mapred.RecordWriter { + private final HTable m_table; + + public MyRecordWriter(HTable table) { + m_table = table; + } + + public void close(Reporter reporter) + throws IOException { + m_table.close(); + } + + public void write(ImmutableBytesWritable key, + Put value) throws IOException { + m_table.put(new Put(value)); + } } } diff --git a/hbase-handler/src/test/queries/positive/hbase_bulk.m b/hbase-handler/src/test/queries/positive/hbase_bulk.m index 764d9245f54e..4c2bceb01764 100644 --- a/hbase-handler/src/test/queries/positive/hbase_bulk.m +++ b/hbase-handler/src/test/queries/positive/hbase_bulk.m @@ -35,8 +35,8 @@ create table hbpartition(part_break string) set mapred.reduce.tasks=3; set hive.mapred.partitioner=org.apache.hadoop.mapred.lib.TotalOrderPartitioner; -set total.order.partitioner.natural.order=false; -set total.order.partitioner.path=/tmp/hbpartition.lst; +set mapreduce.totalorderpartitioner.naturalorder=false; +set mapreduce.totalorderpartitioner.path=/tmp/hbpartition.lst; -- this should produce three files in /tmp/hbsort/cf -- include some trailing blanks and nulls to make sure we handle them correctly diff --git a/hbase-handler/src/test/results/positive/external_table_ppd.q.out b/hbase-handler/src/test/results/positive/external_table_ppd.q.out index 38d9175e8970..3cac34857e2a 100644 --- a/hbase-handler/src/test/results/positive/external_table_ppd.q.out +++ b/hbase-handler/src/test/results/positive/external_table_ppd.q.out @@ -60,7 +60,7 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat Compressed: No Num Buckets: -1 Bucket Columns: [] diff --git a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out index 306e66665853..12fcaf7e655e 100644 --- a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out @@ -60,7 +60,7 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat Compressed: No Num Buckets: -1 Bucket Columns: [] @@ -231,7 +231,7 @@ Table Parameters: # Storage Information SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe InputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat -OutputFormat: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat Compressed: No Num Buckets: -1 Bucket Columns: [] diff --git a/hcatalog/build.properties b/hcatalog/build.properties index fe2de62940b6..f9b686861e6a 100644 --- a/hcatalog/build.properties +++ b/hcatalog/build.properties @@ -37,7 +37,8 @@ test.classes=${test.dir}/classes test.logs=${test.dir}/logs test.timeout=2700000 test.warehouse.dir=${test.dir}/hcat_junit_warehouse -mvnrepo=http://repo2.maven.org/maven2 +#mvnrepo=http://repo2.maven.org/maven2 +mvnrepo=http://ymaven.corp.yahoo.com:9999/repository/public test.src.dir=${basedir}/src/test test.junit.output.format=plain test.output=no @@ -91,5 +92,5 @@ mvn.deploy.repo.id=apache.snapshots.https mvn.deploy.repo.url=https://repository.apache.org/content/repositories/snapshots maven-ant-tasks.version=2.1.3 mvn.local.repo=${user.home}/.m2/repository -mvn.hadoop.profile=hadoop20 +mvn.hadoop.profile=hadoop23 diff --git a/hcatalog/build.xml b/hcatalog/build.xml index f7112881d98d..ee612aaed48b 100644 --- a/hcatalog/build.xml +++ b/hcatalog/build.xml @@ -77,9 +77,6 @@ - - - - - - - - - - + + - - - - + @@ -78,45 +77,18 @@ - + - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hcatalog/storage-handlers/hbase/conf/revision-manager-site.xml b/hcatalog/storage-handlers/hbase/conf/revision-manager-site.xml deleted file mode 100644 index 12ceb9b0127d..000000000000 --- a/hcatalog/storage-handlers/hbase/conf/revision-manager-site.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - diff --git a/hcatalog/storage-handlers/hbase/if/transaction.thrift b/hcatalog/storage-handlers/hbase/if/transaction.thrift deleted file mode 100644 index a74ac3d6f7e4..000000000000 --- a/hcatalog/storage-handlers/hbase/if/transaction.thrift +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -namespace java org.apache.hcatalog.hbase.snapshot.transaction.thrift -namespace cpp Apache.HCatalog.HBase - -struct StoreFamilyRevision { - 1: i64 revision, - 2: i64 timestamp -} - -struct StoreFamilyRevisionList { - 1: list revisionList -} diff --git a/hcatalog/storage-handlers/hbase/pom.xml b/hcatalog/storage-handlers/hbase/pom.xml index 7806c8d89cf5..6bbae2694bcf 100644 --- a/hcatalog/storage-handlers/hbase/pom.xml +++ b/hcatalog/storage-handlers/hbase/pom.xml @@ -35,6 +35,116 @@ hbase-storage-handler http://maven.apache.org + + + hadoop20 + + + org.apache.hadoop + hadoop-tools + ${hadoop20.version} + compile + + + org.apache.hadoop + hadoop-test + ${hadoop20.version} + test + + + org.apache.pig + pig + ${pig.version} + compile + + + + + hadoop23 + + + org.apache.hadoop + hadoop-archives + ${hadoop23.version} + compile + + + org.apache.hadoop + hadoop-mapreduce-client-app + ${hadoop23.version} + test + + + + org.apache.hadoop + hadoop-yarn-server-tests + ${hadoop23.version} + test + test-jar + + + + org.apache.hadoop + hadoop-common + ${hadoop23.version} + compile + + + org.apache.hadoop + hadoop-common + ${hadoop23.version} + test + test-jar + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop23.version} + compile + + + org.apache.hadoop + hadoop-hdfs + ${hadoop23.version} + test + test-jar + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop23.version} + compile + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop23.version} + test + test-jar + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop23.version} + test + test-jar + + + + org.apache.pig + pig + ${pig.version} + compile + h2 + + + + + + org.apache.hive @@ -55,15 +165,15 @@ compile - org.apache.zookeeper - zookeeper - ${zookeeper.version} + org.apache.hcatalog + hcatalog-pig-adapter + ${hcatalog.version} compile - org.apache.hbase - hbase - ${hbase.version} + org.apache.zookeeper + zookeeper + ${zookeeper.version} compile @@ -78,7 +188,6 @@ org.apache.hbase hbase ${hbase.version} - tests test @@ -88,16 +197,24 @@ - org.apache.zookeeper - zookeeper - ${zookeeper.version} - tests - test + org.apache.hbase + hbase + ${hbase.version} + test + test-jar com.yammer.metrics metrics-core ${metrics-core.version} + tests + test + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + tests test diff --git a/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java b/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java deleted file mode 100644 index a5d821343cc5..000000000000 --- a/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevision.java +++ /dev/null @@ -1,416 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This class is used to store the revision and timestamp of a column family - * in a transaction. - * - */ -/** - * Autogenerated by Thrift Compiler (0.7.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - */ -package org.apache.hcatalog.hbase.snapshot.transaction.thrift; - -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; - -public class StoreFamilyRevision implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevision"); - - private static final org.apache.thrift.protocol.TField REVISION_FIELD_DESC = new org.apache.thrift.protocol.TField("revision", org.apache.thrift.protocol.TType.I64, (short) 1); - private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short) 2); - - public long revision; // required - public long timestamp; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - REVISION((short) 1, "revision"), - TIMESTAMP((short) 2, "timestamp"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch (fieldId) { - case 1: // REVISION - return REVISION; - case 2: // TIMESTAMP - return TIMESTAMP; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - private static final int __REVISION_ISSET_ID = 0; - private static final int __TIMESTAMP_ISSET_ID = 1; - private BitSet __isset_bit_vector = new BitSet(2); - - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.REVISION, new org.apache.thrift.meta_data.FieldMetaData("revision", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); - tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevision.class, metaDataMap); - } - - public StoreFamilyRevision() { - } - - public StoreFamilyRevision( - long revision, - long timestamp) { - this(); - this.revision = revision; - setRevisionIsSet(true); - this.timestamp = timestamp; - setTimestampIsSet(true); - } - - /** - * Performs a deep copy on other. - */ - public StoreFamilyRevision(StoreFamilyRevision other) { - __isset_bit_vector.clear(); - __isset_bit_vector.or(other.__isset_bit_vector); - this.revision = other.revision; - this.timestamp = other.timestamp; - } - - public StoreFamilyRevision deepCopy() { - return new StoreFamilyRevision(this); - } - - @Override - public void clear() { - setRevisionIsSet(false); - this.revision = 0; - setTimestampIsSet(false); - this.timestamp = 0; - } - - public long getRevision() { - return this.revision; - } - - public StoreFamilyRevision setRevision(long revision) { - this.revision = revision; - setRevisionIsSet(true); - return this; - } - - public void unsetRevision() { - __isset_bit_vector.clear(__REVISION_ISSET_ID); - } - - /** Returns true if field revision is set (has been assigned a value) and false otherwise */ - public boolean isSetRevision() { - return __isset_bit_vector.get(__REVISION_ISSET_ID); - } - - public void setRevisionIsSet(boolean value) { - __isset_bit_vector.set(__REVISION_ISSET_ID, value); - } - - public long getTimestamp() { - return this.timestamp; - } - - public StoreFamilyRevision setTimestamp(long timestamp) { - this.timestamp = timestamp; - setTimestampIsSet(true); - return this; - } - - public void unsetTimestamp() { - __isset_bit_vector.clear(__TIMESTAMP_ISSET_ID); - } - - /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ - public boolean isSetTimestamp() { - return __isset_bit_vector.get(__TIMESTAMP_ISSET_ID); - } - - public void setTimestampIsSet(boolean value) { - __isset_bit_vector.set(__TIMESTAMP_ISSET_ID, value); - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case REVISION: - if (value == null) { - unsetRevision(); - } else { - setRevision((Long) value); - } - break; - - case TIMESTAMP: - if (value == null) { - unsetTimestamp(); - } else { - setTimestamp((Long) value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case REVISION: - return Long.valueOf(getRevision()); - - case TIMESTAMP: - return Long.valueOf(getTimestamp()); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case REVISION: - return isSetRevision(); - case TIMESTAMP: - return isSetTimestamp(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof StoreFamilyRevision) - return this.equals((StoreFamilyRevision) that); - return false; - } - - public boolean equals(StoreFamilyRevision that) { - if (that == null) - return false; - - boolean this_present_revision = true; - boolean that_present_revision = true; - if (this_present_revision || that_present_revision) { - if (!(this_present_revision && that_present_revision)) - return false; - if (this.revision != that.revision) - return false; - } - - boolean this_present_timestamp = true; - boolean that_present_timestamp = true; - if (this_present_timestamp || that_present_timestamp) { - if (!(this_present_timestamp && that_present_timestamp)) - return false; - if (this.timestamp != that.timestamp) - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - public int compareTo(StoreFamilyRevision other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - StoreFamilyRevision typedOther = (StoreFamilyRevision) other; - - lastComparison = Boolean.valueOf(isSetRevision()).compareTo(typedOther.isSetRevision()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetRevision()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revision, typedOther.revision); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = Boolean.valueOf(isSetTimestamp()).compareTo(typedOther.isSetTimestamp()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTimestamp()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, typedOther.timestamp); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField field; - iprot.readStructBegin(); - while (true) { - field = iprot.readFieldBegin(); - if (field.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (field.id) { - case 1: // REVISION - if (field.type == org.apache.thrift.protocol.TType.I64) { - this.revision = iprot.readI64(); - setRevisionIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - case 2: // TIMESTAMP - if (field.type == org.apache.thrift.protocol.TType.I64) { - this.timestamp = iprot.readI64(); - setTimestampIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - validate(); - - oprot.writeStructBegin(STRUCT_DESC); - oprot.writeFieldBegin(REVISION_FIELD_DESC); - oprot.writeI64(this.revision); - oprot.writeFieldEnd(); - oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); - oprot.writeI64(this.timestamp); - oprot.writeFieldEnd(); - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("StoreFamilyRevision("); - boolean first = true; - - sb.append("revision:"); - sb.append(this.revision); - first = false; - if (!first) sb.append(", "); - sb.append("timestamp:"); - sb.append(this.timestamp); - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. - __isset_bit_vector = new BitSet(1); - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - -} - diff --git a/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java b/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java deleted file mode 100644 index 0f661cb80677..000000000000 --- a/hcatalog/storage-handlers/hbase/src/gen-java/org/apache/hcatalog/hbase/snapshot/transaction/thrift/StoreFamilyRevisionList.java +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This class is used to store a list of StoreFamilyRevision for a column - * family in zookeeper. - * - */ -/** - * Autogenerated by Thrift Compiler (0.7.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - */ -package org.apache.hcatalog.hbase.snapshot.transaction.thrift; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class StoreFamilyRevisionList implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StoreFamilyRevisionList"); - - private static final org.apache.thrift.protocol.TField REVISION_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("revisionList", org.apache.thrift.protocol.TType.LIST, (short) 1); - - public List revisionList; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - REVISION_LIST((short) 1, "revisionList"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch (fieldId) { - case 1: // REVISION_LIST - return REVISION_LIST; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.REVISION_LIST, new org.apache.thrift.meta_data.FieldMetaData("revisionList", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, StoreFamilyRevision.class)))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StoreFamilyRevisionList.class, metaDataMap); - } - - public StoreFamilyRevisionList() { - } - - public StoreFamilyRevisionList( - List revisionList) { - this(); - this.revisionList = revisionList; - } - - /** - * Performs a deep copy on other. - */ - public StoreFamilyRevisionList(StoreFamilyRevisionList other) { - if (other.isSetRevisionList()) { - List __this__revisionList = new ArrayList(); - for (StoreFamilyRevision other_element : other.revisionList) { - __this__revisionList.add(new StoreFamilyRevision(other_element)); - } - this.revisionList = __this__revisionList; - } - } - - public StoreFamilyRevisionList deepCopy() { - return new StoreFamilyRevisionList(this); - } - - @Override - public void clear() { - this.revisionList = null; - } - - public int getRevisionListSize() { - return (this.revisionList == null) ? 0 : this.revisionList.size(); - } - - public java.util.Iterator getRevisionListIterator() { - return (this.revisionList == null) ? null : this.revisionList.iterator(); - } - - public void addToRevisionList(StoreFamilyRevision elem) { - if (this.revisionList == null) { - this.revisionList = new ArrayList(); - } - this.revisionList.add(elem); - } - - public List getRevisionList() { - return this.revisionList; - } - - public StoreFamilyRevisionList setRevisionList(List revisionList) { - this.revisionList = revisionList; - return this; - } - - public void unsetRevisionList() { - this.revisionList = null; - } - - /** Returns true if field revisionList is set (has been assigned a value) and false otherwise */ - public boolean isSetRevisionList() { - return this.revisionList != null; - } - - public void setRevisionListIsSet(boolean value) { - if (!value) { - this.revisionList = null; - } - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case REVISION_LIST: - if (value == null) { - unsetRevisionList(); - } else { - setRevisionList((List) value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case REVISION_LIST: - return getRevisionList(); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case REVISION_LIST: - return isSetRevisionList(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof StoreFamilyRevisionList) - return this.equals((StoreFamilyRevisionList) that); - return false; - } - - public boolean equals(StoreFamilyRevisionList that) { - if (that == null) - return false; - - boolean this_present_revisionList = true && this.isSetRevisionList(); - boolean that_present_revisionList = true && that.isSetRevisionList(); - if (this_present_revisionList || that_present_revisionList) { - if (!(this_present_revisionList && that_present_revisionList)) - return false; - if (!this.revisionList.equals(that.revisionList)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - public int compareTo(StoreFamilyRevisionList other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - StoreFamilyRevisionList typedOther = (StoreFamilyRevisionList) other; - - lastComparison = Boolean.valueOf(isSetRevisionList()).compareTo(typedOther.isSetRevisionList()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetRevisionList()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.revisionList, typedOther.revisionList); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField field; - iprot.readStructBegin(); - while (true) { - field = iprot.readFieldBegin(); - if (field.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (field.id) { - case 1: // REVISION_LIST - if (field.type == org.apache.thrift.protocol.TType.LIST) { - { - org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); - this.revisionList = new ArrayList(_list0.size); - for (int _i1 = 0; _i1 < _list0.size; ++_i1) { - StoreFamilyRevision _elem2; // required - _elem2 = new StoreFamilyRevision(); - _elem2.read(iprot); - this.revisionList.add(_elem2); - } - iprot.readListEnd(); - } - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (this.revisionList != null) { - oprot.writeFieldBegin(REVISION_LIST_FIELD_DESC); - { - oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.revisionList.size())); - for (StoreFamilyRevision _iter3 : this.revisionList) { - _iter3.write(oprot); - } - oprot.writeListEnd(); - } - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("StoreFamilyRevisionList("); - boolean first = true; - - sb.append("revisionList:"); - if (this.revisionList == null) { - sb.append("null"); - } else { - sb.append(this.revisionList); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - -} - diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java deleted file mode 100644 index ee803897fa24..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseAuthorizationProvider.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.ql.metadata.AuthorizationException; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; -import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.ql.security.authorization.Privilege; - -/** - * This class is an implementation of HiveAuthorizationProvider to provide - * authorization functionality for HBase tables. - */ -class HBaseAuthorizationProvider implements HiveAuthorizationProvider { - - @Override - public Configuration getConf() { - return null; - } - - @Override - public void setConf(Configuration conf) { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #init(org.apache.hadoop.conf.Configuration) - */ - @Override - public void init(Configuration conf) throws HiveException { - } - - @Override - public HiveAuthenticationProvider getAuthenticator() { - return null; - } - - @Override - public void setAuthenticator(HiveAuthenticationProvider authenticator) { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.metastore.api.Database, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Database db, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Table, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Table table, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Partition, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Partition part, Privilege[] readRequiredPriv, - Privilege[] writeRequiredPriv) throws HiveException, - AuthorizationException { - } - - /* - * (non-Javadoc) - * - * @see - * org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider - * #authorize(org.apache.hadoop.hive.ql.metadata.Table, - * org.apache.hadoop.hive.ql.metadata.Partition, java.util.List, - * org.apache.hadoop.hive.ql.security.authorization.Privilege[], - * org.apache.hadoop.hive.ql.security.authorization.Privilege[]) - */ - @Override - public void authorize(Table table, Partition part, List columns, - Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) - throws HiveException, AuthorizationException { - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java deleted file mode 100644 index 7332af4af778..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBaseOutputFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.Properties; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hive.ql.io.HiveOutputFormat; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.util.Progressable; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.mapreduce.OutputJobInfo; - -public class HBaseBaseOutputFormat implements OutputFormat, Put>, - HiveOutputFormat, Put> { - - @Override - public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( - JobConf jc, Path finalOutPath, - Class valueClass, boolean isCompressed, - Properties tableProperties, Progressable progress) - throws IOException { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { - OutputFormat, Put> outputFormat = getOutputFormat(job); - outputFormat.checkOutputSpecs(ignored, job); - } - - @Override - public RecordWriter, Put> getRecordWriter(FileSystem ignored, - JobConf job, String name, Progressable progress) throws IOException { - OutputFormat, Put> outputFormat = getOutputFormat(job); - return outputFormat.getRecordWriter(ignored, job, name, progress); - } - - private OutputFormat, Put> getOutputFormat(JobConf job) - throws IOException { - String outputInfo = job.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(outputInfo); - OutputFormat, Put> outputFormat = null; - if (HBaseHCatStorageHandler.isBulkMode(outputJobInfo)) { - outputFormat = new HBaseBulkOutputFormat(); - } else { - outputFormat = new HBaseDirectOutputFormat(); - } - return outputFormat; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java deleted file mode 100644 index 4a188e02af57..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.FileOutputCommitter; -import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.OutputCommitter; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.SequenceFileOutputFormat; -import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.util.Progressable; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; - -/** - * Class which imports data into HBase via it's "bulk load" feature. Wherein - * regions are created by the MR job using HFileOutputFormat and then later - * "moved" into the appropriate region server. - */ -class HBaseBulkOutputFormat extends HBaseBaseOutputFormat { - - private final static ImmutableBytesWritable EMPTY_LIST = new ImmutableBytesWritable( - new byte[0]); - private SequenceFileOutputFormat, Put> baseOutputFormat; - - public HBaseBulkOutputFormat() { - baseOutputFormat = new SequenceFileOutputFormat, Put>(); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) - throws IOException { - baseOutputFormat.checkOutputSpecs(ignored, job); - HBaseUtil.addHBaseDelegationToken(job); - addJTDelegationToken(job); - } - - @Override - public RecordWriter, Put> getRecordWriter( - FileSystem ignored, JobConf job, String name, Progressable progress) - throws IOException { - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Put.class); - long version = HBaseRevisionManagerUtil.getOutputRevision(job); - return new HBaseBulkRecordWriter(baseOutputFormat.getRecordWriter( - ignored, job, name, progress), version); - } - - private void addJTDelegationToken(JobConf job) throws IOException { - // Get jobTracker delegation token if security is enabled - // we need to launch the ImportSequenceFile job - if (User.isSecurityEnabled()) { - JobClient jobClient = new JobClient(new JobConf(job)); - try { - job.getCredentials().addToken(new Text("my mr token"), - jobClient.getDelegationToken(null)); - } catch (InterruptedException e) { - throw new IOException("Error while getting JT delegation token", e); - } - } - } - - private static class HBaseBulkRecordWriter implements - RecordWriter, Put> { - - private RecordWriter, Put> baseWriter; - private final Long outputVersion; - - public HBaseBulkRecordWriter( - RecordWriter, Put> baseWriter, - Long outputVersion) { - this.baseWriter = baseWriter; - this.outputVersion = outputVersion; - } - - @Override - public void write(WritableComparable key, Put value) - throws IOException { - Put put = value; - if (outputVersion != null) { - put = new Put(value.getRow(), outputVersion.longValue()); - for (List row : value.getFamilyMap().values()) { - for (KeyValue el : row) { - put.add(el.getFamily(), el.getQualifier(), el.getValue()); - } - } - } - // we ignore the key - baseWriter.write(EMPTY_LIST, put); - } - - @Override - public void close(Reporter reporter) throws IOException { - baseWriter.close(reporter); - } - } - - public static class HBaseBulkOutputCommitter extends OutputCommitter { - - private final OutputCommitter baseOutputCommitter; - - public HBaseBulkOutputCommitter() { - baseOutputCommitter = new FileOutputCommitter(); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) - throws IOException { - baseOutputCommitter.abortTask(taskContext); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) - throws IOException { - // baseOutputCommitter.commitTask(taskContext); - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) - throws IOException { - return baseOutputCommitter.needsTaskCommit(taskContext); - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - baseOutputCommitter.setupJob(jobContext); - } - - @Override - public void setupTask(TaskAttemptContext taskContext) - throws IOException { - baseOutputCommitter.setupTask(taskContext); - } - - @Override - public void abortJob(JobContext jobContext, int status) - throws IOException { - baseOutputCommitter.abortJob(jobContext, status); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - rm.abortWriteTransaction(HBaseRevisionManagerUtil - .getWriteTransaction(jobContext.getConfiguration())); - } finally { - cleanIntermediate(jobContext); - if (rm != null) - rm.close(); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - baseOutputCommitter.commitJob(jobContext); - RevisionManager rm = null; - try { - Configuration conf = jobContext.getConfiguration(); - Path srcPath = FileOutputFormat.getOutputPath(jobContext.getJobConf()); - if (!FileSystem.get(conf).exists(srcPath)) { - throw new IOException("Failed to bulk import hfiles. " + - "Intermediate data directory is cleaned up or missing. " + - "Please look at the bulk import job if it exists for failure reason"); - } - Path destPath = new Path(srcPath.getParent(), srcPath.getName() + "_hfiles"); - boolean success = ImportSequenceFile.runJob(jobContext, - conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY), - srcPath, - destPath); - if (!success) { - cleanIntermediate(jobContext); - throw new IOException("Failed to bulk import hfiles." + - " Please look at the bulk import job for failure reason"); - } - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(conf)); - cleanIntermediate(jobContext); - } finally { - if (rm != null) - rm.close(); - } - } - - private void cleanIntermediate(JobContext jobContext) - throws IOException { - FileSystem fs = FileSystem.get(jobContext.getConfiguration()); - fs.delete(FileOutputFormat.getOutputPath(jobContext.getJobConf()), true); - } - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java deleted file mode 100644 index 41e62c37af6f..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseConstants.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import org.apache.hcatalog.common.HCatConstants; - -/** - * Constants class for constants used in HBase storage handler. - */ -class HBaseConstants { - - /** key used to store write transaction object */ - public static final String PROPERTY_WRITE_TXN_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.writeTxn"; - - /** key used to define the name of the table to write to */ - public static final String PROPERTY_OUTPUT_TABLE_NAME_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.mapreduce.outputTableName"; - - /** key used to define whether bulk storage output format will be used or not */ - public static final String PROPERTY_BULK_OUTPUT_MODE_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + ".hbase.output.bulkMode"; - - /** key used to define the hbase table snapshot. */ - public static final String PROPERTY_TABLE_SNAPSHOT_KEY = HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX + "hbase.table.snapshot"; - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java deleted file mode 100644 index b7537d4dbba6..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.OutputCommitter; -import org.apache.hadoop.mapred.RecordWriter; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.util.Progressable; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.Transaction; - -/** - * "Direct" implementation of OutputFormat for HBase. Uses HTable client's put - * API to write each row to HBase one a time. Presently it is just using - * TableOutputFormat as the underlying implementation in the future we can tune - * this to make the writes faster such as permanently disabling WAL, caching, - * etc. - */ -class HBaseDirectOutputFormat extends HBaseBaseOutputFormat { - - private TableOutputFormat outputFormat; - - public HBaseDirectOutputFormat() { - this.outputFormat = new TableOutputFormat(); - } - - @Override - public RecordWriter, Put> getRecordWriter(FileSystem ignored, - JobConf job, String name, Progressable progress) - throws IOException { - long version = HBaseRevisionManagerUtil.getOutputRevision(job); - return new HBaseDirectRecordWriter(outputFormat.getRecordWriter(ignored, job, name, - progress), version); - } - - @Override - public void checkOutputSpecs(FileSystem ignored, JobConf job) - throws IOException { - outputFormat.checkOutputSpecs(ignored, job); - HBaseUtil.addHBaseDelegationToken(job); - } - - private static class HBaseDirectRecordWriter implements - RecordWriter, Put> { - - private RecordWriter, Put> baseWriter; - private final Long outputVersion; - - public HBaseDirectRecordWriter( - RecordWriter, Put> baseWriter, - Long outputVersion) { - this.baseWriter = baseWriter; - this.outputVersion = outputVersion; - } - - @Override - public void write(WritableComparable key, Put value) - throws IOException { - Put put = value; - if (outputVersion != null) { - put = new Put(value.getRow(), outputVersion.longValue()); - for (List row : value.getFamilyMap().values()) { - for (KeyValue el : row) { - put.add(el.getFamily(), el.getQualifier(), el.getValue()); - } - } - } - baseWriter.write(key, put); - } - - @Override - public void close(Reporter reporter) throws IOException { - baseWriter.close(reporter); - } - - } - - public static class HBaseDirectOutputCommitter extends OutputCommitter { - - public HBaseDirectOutputCommitter() throws IOException { - } - - @Override - public void abortTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public void commitTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) - throws IOException { - return false; - } - - @Override - public void setupJob(JobContext jobContext) throws IOException { - } - - @Override - public void setupTask(TaskAttemptContext taskContext) - throws IOException { - } - - @Override - public void abortJob(JobContext jobContext, int status) - throws IOException { - super.abortJob(jobContext, status); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - Transaction writeTransaction = HBaseRevisionManagerUtil - .getWriteTransaction(jobContext.getConfiguration()); - rm.abortWriteTransaction(writeTransaction); - } finally { - if (rm != null) - rm.close(); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil - .getOpenedRevisionManager(jobContext.getConfiguration()); - rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(jobContext - .getConfiguration())); - } finally { - if (rm != null) - rm.close(); - } - } - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java deleted file mode 100644 index 31b7741f9ef0..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java +++ /dev/null @@ -1,610 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.hbase.mapreduce.TableInputFormat; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hadoop.hive.metastore.HiveMetaHook; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputFormat; -import org.apache.hadoop.util.StringUtils; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; -import org.apache.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hcatalog.hbase.snapshot.Transaction; -import org.apache.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hcatalog.mapreduce.HCatTableInfo; -import org.apache.hcatalog.mapreduce.InputJobInfo; -import org.apache.hcatalog.mapreduce.OutputJobInfo; -import org.apache.hcatalog.mapreduce.HCatStorageHandler; -import org.apache.thrift.TBase; -import org.apache.zookeeper.ZooKeeper; - -import com.facebook.fb303.FacebookBase; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * This class HBaseHCatStorageHandler provides functionality to create HBase - * tables through HCatalog. The implementation is very similar to the - * HiveHBaseStorageHandler, with more details to suit HCatalog. - */ -public class HBaseHCatStorageHandler extends HCatStorageHandler implements HiveMetaHook, Configurable { - - public final static String DEFAULT_PREFIX = "default."; - private final static String PROPERTY_INT_OUTPUT_LOCATION = "hcat.hbase.mapreduce.intermediateOutputLocation"; - - private Configuration hbaseConf; - private Configuration jobConf; - private HBaseAdmin admin; - - @Override - public void configureInputJobProperties(TableDesc tableDesc, Map jobProperties) { - // Populate jobProperties with input table name, table columns, RM snapshot, - // hbase-default.xml and hbase-site.xml - Map tableJobProperties = tableDesc.getJobProperties(); - String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_JOB_INFO); - try { - InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); - HCatTableInfo tableInfo = inputJobInfo.getTableInfo(); - String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); - jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName); - - Configuration jobConf = getJobConf(); - addResources(jobConf, jobProperties); - JobConf copyOfConf = new JobConf(jobConf); - HBaseConfiguration.addHbaseResources(copyOfConf); - //Getting hbase delegation token in getInputSplits does not work with PIG. So need to - //do it here - if (jobConf instanceof JobConf) { //Should be the case - HBaseUtil.addHBaseDelegationToken(copyOfConf); - ((JobConf) jobConf).getCredentials().addAll(copyOfConf.getCredentials()); - } - - String outputSchema = jobConf.get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA); - jobProperties.put(TableInputFormat.SCAN_COLUMNS, getScanColumns(tableInfo, outputSchema)); - - String serSnapshot = (String) inputJobInfo.getProperties().get( - HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); - if (serSnapshot == null) { - HCatTableSnapshot snapshot = - HBaseRevisionManagerUtil.createSnapshot( - RevisionManagerConfiguration.create(copyOfConf), - qualifiedTableName, tableInfo); - jobProperties.put(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, - HCatUtil.serialize(snapshot)); - } - - //This adds it directly to the jobConf. Setting in jobProperties does not get propagated - //to JobConf as of now as the jobProperties is maintained per partition - //TODO: Remove when HCAT-308 is fixed - addOutputDependencyJars(jobConf); - jobProperties.put("tmpjars", jobConf.get("tmpjars")); - - } catch (IOException e) { - throw new IllegalStateException("Error while configuring job properties", e); - } - } - - @Override - public void configureOutputJobProperties(TableDesc tableDesc, Map jobProperties) { - // Populate jobProperties with output table name, hbase-default.xml, hbase-site.xml, OutputJobInfo - // Populate RM transaction in OutputJobInfo - // In case of bulk mode, populate intermediate output location - Map tableJobProperties = tableDesc.getJobProperties(); - String jobString = tableJobProperties.get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - try { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); - HCatTableInfo tableInfo = outputJobInfo.getTableInfo(); - String qualifiedTableName = HBaseHCatStorageHandler.getFullyQualifiedHBaseTableName(tableInfo); - jobProperties.put(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, qualifiedTableName); - jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName); - - Configuration jobConf = getJobConf(); - addResources(jobConf, jobProperties); - - Configuration copyOfConf = new Configuration(jobConf); - HBaseConfiguration.addHbaseResources(copyOfConf); - - String txnString = outputJobInfo.getProperties().getProperty( - HBaseConstants.PROPERTY_WRITE_TXN_KEY); - Transaction txn = null; - if (txnString == null) { - txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo, - RevisionManagerConfiguration.create(copyOfConf)); - String serializedTxn = HCatUtil.serialize(txn); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - serializedTxn); - } else { - txn = (Transaction) HCatUtil.deserialize(txnString); - } - if (isBulkMode(outputJobInfo)) { - String tableLocation = tableInfo.getTableLocation(); - String location = new Path(tableLocation, "REVISION_" + txn.getRevisionNumber()) - .toString(); - outputJobInfo.getProperties().setProperty(PROPERTY_INT_OUTPUT_LOCATION, location); - // We are writing out an intermediate sequenceFile hence - // location is not passed in OutputJobInfo.getLocation() - // TODO replace this with a mapreduce constant when available - jobProperties.put("mapred.output.dir", location); - jobProperties.put("mapred.output.committer.class", HBaseBulkOutputCommitter.class.getName()); - } else { - jobProperties.put("mapred.output.committer.class", HBaseDirectOutputCommitter.class.getName()); - } - - jobProperties.put(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); - addOutputDependencyJars(jobConf); - jobProperties.put("tmpjars", jobConf.get("tmpjars")); - - } catch (IOException e) { - throw new IllegalStateException("Error while configuring job properties", e); - } - } - - /* - * @return instance of HiveAuthorizationProvider - * - * @throws HiveException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler# - * getAuthorizationProvider() - */ - @Override - public HiveAuthorizationProvider getAuthorizationProvider() - throws HiveException { - - HBaseAuthorizationProvider hbaseAuth = new HBaseAuthorizationProvider(); - hbaseAuth.init(getConf()); - return hbaseAuth; - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #commitCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void commitCreateTable(Table table) throws MetaException { - } - - /* - * @param instance of table - * - * @param deleteData - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #commitDropTable(org.apache.hadoop.hive.metastore.api.Table, boolean) - */ - @Override - public void commitDropTable(Table tbl, boolean deleteData) - throws MetaException { - checkDeleteTable(tbl); - - } - - /* - * @param instance of table - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #preCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void preCreateTable(Table tbl) throws MetaException { - boolean isExternal = MetaStoreUtils.isExternalTable(tbl); - - hbaseConf = getConf(); - - if (tbl.getSd().getLocation() != null) { - throw new MetaException("LOCATION may not be specified for HBase."); - } - - try { - String tableName = getFullyQualifiedHBaseTableName(tbl); - String hbaseColumnsMapping = tbl.getParameters().get( - HBaseSerDe.HBASE_COLUMNS_MAPPING); - - if (hbaseColumnsMapping == null) { - throw new MetaException( - "No hbase.columns.mapping defined in table" - + " properties."); - } - - List hbaseColumnFamilies = new ArrayList(); - List hbaseColumnQualifiers = new ArrayList(); - List hbaseColumnFamiliesBytes = new ArrayList(); - int iKey = HBaseUtil.parseColumnMapping(hbaseColumnsMapping, - hbaseColumnFamilies, hbaseColumnFamiliesBytes, - hbaseColumnQualifiers, null); - - HTableDescriptor tableDesc; - Set uniqueColumnFamilies = new HashSet(); - if (!getHBaseAdmin().tableExists(tableName)) { - // if it is not an external table then create one - if (!isExternal) { - // Create the column descriptors - tableDesc = new HTableDescriptor(tableName); - uniqueColumnFamilies.addAll(hbaseColumnFamilies); - uniqueColumnFamilies.remove(hbaseColumnFamilies.get(iKey)); - - for (String columnFamily : uniqueColumnFamilies) { - HColumnDescriptor familyDesc = new HColumnDescriptor(Bytes - .toBytes(columnFamily)); - familyDesc.setMaxVersions(Integer.MAX_VALUE); - tableDesc.addFamily(familyDesc); - } - - getHBaseAdmin().createTable(tableDesc); - } else { - // an external table - throw new MetaException("HBase table " + tableName - + " doesn't exist while the table is " - + "declared as an external table."); - } - - } else { - if (!isExternal) { - throw new MetaException("Table " + tableName - + " already exists within HBase." - + " Use CREATE EXTERNAL TABLE instead to" - + " register it in HCatalog."); - } - // make sure the schema mapping is right - tableDesc = getHBaseAdmin().getTableDescriptor( - Bytes.toBytes(tableName)); - - for (int i = 0; i < hbaseColumnFamilies.size(); i++) { - if (i == iKey) { - continue; - } - - if (!tableDesc.hasFamily(hbaseColumnFamiliesBytes.get(i))) { - throw new MetaException("Column Family " - + hbaseColumnFamilies.get(i) - + " is not defined in hbase table " + tableName); - } - } - } - - // ensure the table is online - new HTable(hbaseConf, tableDesc.getName()); - - //Set up table in revision manager. - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); - rm.createTable(tableName, new ArrayList(uniqueColumnFamilies)); - - } catch (MasterNotRunningException mnre) { - throw new MetaException(StringUtils.stringifyException(mnre)); - } catch (IOException ie) { - throw new MetaException(StringUtils.stringifyException(ie)); - } catch (IllegalArgumentException iae) { - throw new MetaException(StringUtils.stringifyException(iae)); - } - - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #preDropTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void preDropTable(Table table) throws MetaException { - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void rollbackCreateTable(Table table) throws MetaException { - checkDeleteTable(table); - } - - /* - * @param table - * - * @throws MetaException - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler - * #rollbackDropTable(org.apache.hadoop.hive.metastore.api.Table) - */ - @Override - public void rollbackDropTable(Table table) throws MetaException { - } - - /* - * @return instance of HiveMetaHook - * - * @see org.apache.hcatalog.storagehandler.HCatStorageHandler#getMetaHook() - */ - @Override - public HiveMetaHook getMetaHook() { - return this; - } - - private HBaseAdmin getHBaseAdmin() throws MetaException { - try { - if (admin == null) { - admin = new HBaseAdmin(this.getConf()); - } - return admin; - } catch (MasterNotRunningException mnre) { - throw new MetaException(StringUtils.stringifyException(mnre)); - } catch (ZooKeeperConnectionException zkce) { - throw new MetaException(StringUtils.stringifyException(zkce)); - } - } - - private String getFullyQualifiedHBaseTableName(Table tbl) { - String tableName = tbl.getParameters().get(HBaseSerDe.HBASE_TABLE_NAME); - if (tableName == null) { - tableName = tbl.getSd().getSerdeInfo().getParameters() - .get(HBaseSerDe.HBASE_TABLE_NAME); - } - if (tableName == null) { - if (tbl.getDbName().equals(MetaStoreUtils.DEFAULT_DATABASE_NAME)) { - tableName = tbl.getTableName(); - } else { - tableName = tbl.getDbName() + "." + tbl.getTableName(); - } - tableName = tableName.toLowerCase(); - } - return tableName; - } - - static String getFullyQualifiedHBaseTableName(HCatTableInfo tableInfo) { - String qualifiedName = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_TABLE_NAME); - if (qualifiedName == null) { - String databaseName = tableInfo.getDatabaseName(); - String tableName = tableInfo.getTableName(); - if ((databaseName == null) - || (databaseName.equals(MetaStoreUtils.DEFAULT_DATABASE_NAME))) { - qualifiedName = tableName; - } else { - qualifiedName = databaseName + "." + tableName; - } - qualifiedName = qualifiedName.toLowerCase(); - } - return qualifiedName; - } - - @Override - public Class getInputFormatClass() { - return HBaseInputFormat.class; - } - - @Override - public Class getOutputFormatClass() { - return HBaseBaseOutputFormat.class; - } - - /* - * @return subclass of SerDe - * - * @throws UnsupportedOperationException - * - * @see - * org.apache.hcatalog.storagehandler.HCatStorageHandler#getSerDeClass() - */ - @Override - public Class getSerDeClass() - throws UnsupportedOperationException { - return HBaseSerDe.class; - } - - public Configuration getJobConf() { - return jobConf; - } - - @Override - public Configuration getConf() { - - if (hbaseConf == null) { - hbaseConf = HBaseConfiguration.create(); - } - return hbaseConf; - } - - @Override - public void setConf(Configuration conf) { - //setConf is called both during DDL operations and mapred read/write jobs. - //Creating a copy of conf for DDL and adding hbase-default and hbase-site.xml to it. - //For jobs, maintaining a reference instead of cloning as we need to - // 1) add hbase delegation token to the Credentials. - // 2) set tmpjars on it. Putting in jobProperties does not get propagated to JobConf - // in case of InputFormat as they are maintained per partition. - //Not adding hbase-default.xml and hbase-site.xml to jobConf as it will override any - //hbase properties set in the JobConf by the user. In configureInputJobProperties and - //configureOutputJobProperties, we take care of adding the default properties - //that are not already present. TODO: Change to a copy for jobs after HCAT-308 is fixed. - jobConf = conf; - hbaseConf = RevisionManagerConfiguration.create(HBaseConfiguration.create(conf)); - } - - private void checkDeleteTable(Table table) throws MetaException { - boolean isExternal = MetaStoreUtils.isExternalTable(table); - String tableName = getFullyQualifiedHBaseTableName(table); - RevisionManager rm = null; - try { - if (!isExternal && getHBaseAdmin().tableExists(tableName)) { - // we have created an HBase table, so we delete it to roll back; - if (getHBaseAdmin().isTableEnabled(tableName)) { - getHBaseAdmin().disableTable(tableName); - } - getHBaseAdmin().deleteTable(tableName); - - //Drop table in revision manager. - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf); - rm.dropTable(tableName); - } - } catch (IOException ie) { - throw new MetaException(StringUtils.stringifyException(ie)); - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - } - - /** - * Helper method for users to add the required depedency jars to distributed cache. - * @param conf - * @throws IOException - */ - private void addOutputDependencyJars(Configuration conf) throws IOException { - TableMapReduceUtil.addDependencyJars(conf, - //ZK - ZooKeeper.class, - //HBase - HTable.class, - //Hive - HiveException.class, - //HCatalog jar - HCatOutputFormat.class, - //hcat hbase storage handler jar - HBaseHCatStorageHandler.class, - //hive hbase storage handler jar - HBaseSerDe.class, - //hive jar - Table.class, - //libthrift jar - TBase.class, - //hbase jar - Bytes.class, - //thrift-fb303 .jar - FacebookBase.class, - //guava jar - ThreadFactoryBuilder.class); - } - - /** - * Utility method to add hbase-default.xml and hbase-site.xml properties to a new map - * if they are not already present in the jobConf. - * @param jobConf Job configuration - * @param newJobProperties Map to which new properties should be added - */ - private void addResources(Configuration jobConf, - Map newJobProperties) { - Configuration conf = new Configuration(false); - HBaseConfiguration.addHbaseResources(conf); - RevisionManagerConfiguration.addResources(conf); - for (Entry entry : conf) { - if (jobConf.get(entry.getKey()) == null) - newJobProperties.put(entry.getKey(), entry.getValue()); - } - } - - public static boolean isBulkMode(OutputJobInfo outputJobInfo) { - //Default is false - String bulkMode = outputJobInfo.getTableInfo().getStorerInfo().getProperties() - .getProperty(HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY, - "false"); - return "true".equals(bulkMode); - } - - private String getScanColumns(HCatTableInfo tableInfo, String outputColSchema) throws IOException { - StringBuilder builder = new StringBuilder(); - String hbaseColumnMapping = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); - if (outputColSchema == null) { - String[] splits = hbaseColumnMapping.split("[,]"); - for (int i = 0; i < splits.length; i++) { - if (!splits[i].equals(HBaseSerDe.HBASE_KEY_COL)) - builder.append(splits[i]).append(" "); - } - } else { - HCatSchema outputSchema = (HCatSchema) HCatUtil.deserialize(outputColSchema); - HCatSchema tableSchema = tableInfo.getDataColumns(); - List outputFieldNames = outputSchema.getFieldNames(); - List outputColumnMapping = new ArrayList(); - for (String fieldName : outputFieldNames) { - int position = tableSchema.getPosition(fieldName); - outputColumnMapping.add(position); - } - List columnFamilies = new ArrayList(); - List columnQualifiers = new ArrayList(); - HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, null, - columnQualifiers, null); - for (int i = 0; i < outputColumnMapping.size(); i++) { - int cfIndex = outputColumnMapping.get(i); - String cf = columnFamilies.get(cfIndex); - // We skip the key column. - if (cf.equals(HBaseSerDe.HBASE_KEY_COL) == false) { - String qualifier = columnQualifiers.get(i); - builder.append(cf); - builder.append(":"); - if (qualifier != null) { - builder.append(qualifier); - } - builder.append(" "); - } - } - } - //Remove the extra space delimiter - builder.deleteCharAt(builder.length() - 1); - return builder.toString(); - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java deleted file mode 100644 index f4f9ed11ae4e..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapred.TableSplit; -import org.apache.hadoop.hbase.mapreduce.TableInputFormat; -import org.apache.hadoop.mapred.HCatMapRedUtil; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.mapreduce.InputJobInfo; - -/** - * This class HBaseInputFormat is a wrapper class of TableInputFormat in HBase. - */ -class HBaseInputFormat implements InputFormat { - - private final TableInputFormat inputFormat; - - public HBaseInputFormat() { - inputFormat = new TableInputFormat(); - } - - /* - * @param instance of InputSplit - * - * @param instance of TaskAttemptContext - * - * @return RecordReader - * - * @throws IOException - * - * @throws InterruptedException - * - * @see - * org.apache.hadoop.mapreduce.InputFormat#createRecordReader(org.apache - * .hadoop.mapreduce.InputSplit, - * org.apache.hadoop.mapreduce.TaskAttemptContext) - */ - @Override - public RecordReader getRecordReader( - InputSplit split, JobConf job, Reporter reporter) - throws IOException { - String jobString = job.get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobString); - - String tableName = job.get(TableInputFormat.INPUT_TABLE); - TableSplit tSplit = (TableSplit) split; - HbaseSnapshotRecordReader recordReader = new HbaseSnapshotRecordReader(inputJobInfo, job); - inputFormat.setConf(job); - Scan inputScan = inputFormat.getScan(); - // TODO: Make the caching configurable by the user - inputScan.setCaching(200); - inputScan.setCacheBlocks(false); - Scan sc = new Scan(inputScan); - sc.setStartRow(tSplit.getStartRow()); - sc.setStopRow(tSplit.getEndRow()); - recordReader.setScan(sc); - recordReader.setHTable(new HTable(job, tableName)); - recordReader.init(); - return recordReader; - } - - /* - * @param jobContext - * - * @return List of InputSplit - * - * @throws IOException - * - * @throws InterruptedException - * - * @see - * org.apache.hadoop.mapreduce.InputFormat#getSplits(org.apache.hadoop.mapreduce - * .JobContext) - */ - @Override - public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) - throws IOException { - inputFormat.setConf(job); - return convertSplits(inputFormat.getSplits(HCatMapRedUtil.createJobContext(job, null, - Reporter.NULL))); - } - - private InputSplit[] convertSplits(List splits) { - InputSplit[] converted = new InputSplit[splits.size()]; - for (int i = 0; i < splits.size(); i++) { - org.apache.hadoop.hbase.mapreduce.TableSplit tableSplit = - (org.apache.hadoop.hbase.mapreduce.TableSplit) splits.get(i); - TableSplit newTableSplit = new TableSplit(tableSplit.getTableName(), - tableSplit.getStartRow(), - tableSplit.getEndRow(), tableSplit.getRegionLocation()); - converted[i] = newTableSplit; - } - return converted; - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java deleted file mode 100644 index c2dafda91b4c..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory; -import org.apache.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hcatalog.hbase.snapshot.Transaction; -import org.apache.hcatalog.mapreduce.HCatTableInfo; -import org.apache.hcatalog.mapreduce.InputJobInfo; -import org.apache.hcatalog.mapreduce.OutputJobInfo; -import org.apache.hcatalog.mapreduce.StorerInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * The Class HBaseRevisionManagerUtil has utility methods to interact with Revision Manager - * - */ -class HBaseRevisionManagerUtil { - - private final static Logger LOG = LoggerFactory.getLogger(HBaseRevisionManagerUtil.class); - - private HBaseRevisionManagerUtil() { - } - - /** - * Creates the latest snapshot of the table. - * - * @param jobConf The job configuration. - * @param hbaseTableName The fully qualified name of the HBase table. - * @param tableInfo HCat table information - * @return An instance of HCatTableSnapshot - * @throws IOException Signals that an I/O exception has occurred. - */ - static HCatTableSnapshot createSnapshot(Configuration jobConf, - String hbaseTableName, HCatTableInfo tableInfo) throws IOException { - - RevisionManager rm = null; - TableSnapshot snpt; - try { - rm = getOpenedRevisionManager(jobConf); - snpt = rm.createSnapshot(hbaseTableName); - } finally { - closeRevisionManagerQuietly(rm); - } - - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(snpt, tableInfo); - return hcatSnapshot; - } - - /** - * Creates the snapshot using the revision specified by the user. - * - * @param jobConf The job configuration. - * @param tableName The fully qualified name of the table whose snapshot is being taken. - * @param revision The revision number to use for the snapshot. - * @return An instance of HCatTableSnapshot. - * @throws IOException Signals that an I/O exception has occurred. - */ - static HCatTableSnapshot createSnapshot(Configuration jobConf, - String tableName, long revision) - throws IOException { - - TableSnapshot snpt; - RevisionManager rm = null; - try { - rm = getOpenedRevisionManager(jobConf); - snpt = rm.createSnapshot(tableName, revision); - } finally { - closeRevisionManagerQuietly(rm); - } - - String inputJobString = jobConf.get(HCatConstants.HCAT_KEY_JOB_INFO); - if (inputJobString == null) { - throw new IOException( - "InputJobInfo information not found in JobContext. " - + "HCatInputFormat.setInput() not called?"); - } - InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(inputJobString); - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil - .convertSnapshot(snpt, inputInfo.getTableInfo()); - - return hcatSnapshot; - } - - /** - * Gets an instance of revision manager which is opened. - * - * @param jobConf The job configuration. - * @return RevisionManager An instance of revision manager. - * @throws IOException - */ - static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException { - return RevisionManagerFactory.getOpenedRevisionManager(jobConf); - } - - static void closeRevisionManagerQuietly(RevisionManager rm) { - if (rm != null) { - try { - rm.close(); - } catch (IOException e) { - LOG.warn("Error while trying to close revision manager", e); - } - } - } - - - static HCatTableSnapshot convertSnapshot(TableSnapshot hbaseSnapshot, - HCatTableInfo hcatTableInfo) throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); - HashMap revisionMap = new HashMap(); - - for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { - if (hcatHbaseColMap.containsKey(fSchema.getName())) { - String colFamily = hcatHbaseColMap.get(fSchema.getName()); - long revisionID = hbaseSnapshot.getRevision(colFamily); - revisionMap.put(fSchema.getName(), revisionID); - } - } - - HCatTableSnapshot hcatSnapshot = new HCatTableSnapshot( - hcatTableInfo.getDatabaseName(), hcatTableInfo.getTableName(), revisionMap, hbaseSnapshot.getLatestRevision()); - return hcatSnapshot; - } - - static TableSnapshot convertSnapshot(HCatTableSnapshot hcatSnapshot, - HCatTableInfo hcatTableInfo) throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - Map revisionMap = new HashMap(); - Map hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo); - for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) { - String colFamily = hcatHbaseColMap.get(fSchema.getName()); - if (hcatSnapshot.containsColumn(fSchema.getName())) { - long revision = hcatSnapshot.getRevision(fSchema.getName()); - revisionMap.put(colFamily, revision); - } - } - - String fullyQualifiedName = hcatSnapshot.getDatabaseName() + "." - + hcatSnapshot.getTableName(); - return new TableSnapshot(fullyQualifiedName, revisionMap, hcatSnapshot.getLatestRevision()); - - } - - /** - * Begins a transaction in the revision manager for the given table. - * @param qualifiedTableName Name of the table - * @param tableInfo HCat Table information - * @param jobConf Job Configuration - * @return The new transaction in revision manager - * @throws IOException - */ - static Transaction beginWriteTransaction(String qualifiedTableName, - HCatTableInfo tableInfo, Configuration jobConf) throws IOException { - Transaction txn; - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(jobConf); - String hBaseColumns = tableInfo.getStorerInfo().getProperties() - .getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING); - String[] splits = hBaseColumns.split("[,:]"); - Set families = new HashSet(); - for (int i = 0; i < splits.length; i += 2) { - if (!splits[i].isEmpty()) - families.add(splits[i]); - } - txn = rm.beginWriteTransaction(qualifiedTableName, new ArrayList(families)); - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - return txn; - } - - static Transaction getWriteTransaction(Configuration conf) throws IOException { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - return (Transaction) HCatUtil.deserialize(outputJobInfo.getProperties() - .getProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); - } - - static void setWriteTransaction(Configuration conf, Transaction txn) throws IOException { - OutputJobInfo outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, HCatUtil.serialize(txn)); - conf.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo)); - } - - /** - * Get the Revision number that will be assigned to this job's output data - * @param conf configuration of the job - * @return the revision number used - * @throws IOException - */ - static long getOutputRevision(Configuration conf) throws IOException { - return getWriteTransaction(conf).getRevisionNumber(); - } - - private static Map getHCatHBaseColumnMapping(HCatTableInfo hcatTableInfo) - throws IOException { - - HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns(); - StorerInfo storeInfo = hcatTableInfo.getStorerInfo(); - String hbaseColumnMapping = storeInfo.getProperties().getProperty( - HBaseSerDe.HBASE_COLUMNS_MAPPING); - - Map hcatHbaseColMap = new HashMap(); - List columnFamilies = new ArrayList(); - List columnQualifiers = new ArrayList(); - HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, - null, columnQualifiers, null); - - for (HCatFieldSchema column : hcatTableSchema.getFields()) { - int fieldPos = hcatTableSchema.getPosition(column.getName()); - String colFamily = columnFamilies.get(fieldPos); - if (colFamily.equals(HBaseSerDe.HBASE_KEY_COL) == false) { - hcatHbaseColMap.put(column.getName(), colFamily); - } - } - - return hcatHbaseColMap; - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java deleted file mode 100644 index 27e165ff162b..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.hbase.HBaseSerDe; -import org.apache.hadoop.mapred.JobConf; - -class HBaseUtil { - - private HBaseUtil() { - } - - /** - * Parses the HBase columns mapping to identify the column families, qualifiers - * and also caches the byte arrays corresponding to them. One of the HCat table - * columns maps to the HBase row key, by default the first column. - * - * @param columnMapping - the column mapping specification to be parsed - * @param colFamilies - the list of HBase column family names - * @param colFamiliesBytes - the corresponding byte array - * @param colQualifiers - the list of HBase column qualifier names - * @param colQualifiersBytes - the corresponding byte array - * @return the row key index in the column names list - * @throws IOException - */ - static int parseColumnMapping( - String columnMapping, - List colFamilies, - List colFamiliesBytes, - List colQualifiers, - List colQualifiersBytes) throws IOException { - - int rowKeyIndex = -1; - - if (colFamilies == null || colQualifiers == null) { - throw new IllegalArgumentException("Error: caller must pass in lists for the column families " + - "and qualifiers."); - } - - colFamilies.clear(); - colQualifiers.clear(); - - if (columnMapping == null) { - throw new IllegalArgumentException("Error: hbase.columns.mapping missing for this HBase table."); - } - - if (columnMapping.equals("") || columnMapping.equals(HBaseSerDe.HBASE_KEY_COL)) { - throw new IllegalArgumentException("Error: hbase.columns.mapping specifies only the HBase table" - + " row key. A valid Hive-HBase table must specify at least one additional column."); - } - - String[] mapping = columnMapping.split(","); - - for (int i = 0; i < mapping.length; i++) { - String elem = mapping[i]; - int idxFirst = elem.indexOf(":"); - int idxLast = elem.lastIndexOf(":"); - - if (idxFirst < 0 || !(idxFirst == idxLast)) { - throw new IllegalArgumentException("Error: the HBase columns mapping contains a badly formed " + - "column family, column qualifier specification."); - } - - if (elem.equals(HBaseSerDe.HBASE_KEY_COL)) { - rowKeyIndex = i; - colFamilies.add(elem); - colQualifiers.add(null); - } else { - String[] parts = elem.split(":"); - assert (parts.length > 0 && parts.length <= 2); - colFamilies.add(parts[0]); - - if (parts.length == 2) { - colQualifiers.add(parts[1]); - } else { - colQualifiers.add(null); - } - } - } - - if (rowKeyIndex == -1) { - colFamilies.add(0, HBaseSerDe.HBASE_KEY_COL); - colQualifiers.add(0, null); - rowKeyIndex = 0; - } - - if (colFamilies.size() != colQualifiers.size()) { - throw new IOException("Error in parsing the hbase columns mapping."); - } - - // populate the corresponding byte [] if the client has passed in a non-null list - if (colFamiliesBytes != null) { - colFamiliesBytes.clear(); - - for (String fam : colFamilies) { - colFamiliesBytes.add(Bytes.toBytes(fam)); - } - } - - if (colQualifiersBytes != null) { - colQualifiersBytes.clear(); - - for (String qual : colQualifiers) { - if (qual == null) { - colQualifiersBytes.add(null); - } else { - colQualifiersBytes.add(Bytes.toBytes(qual)); - } - } - } - - if (colFamiliesBytes != null && colQualifiersBytes != null) { - if (colFamiliesBytes.size() != colQualifiersBytes.size()) { - throw new IOException("Error in caching the bytes for the hbase column families " + - "and qualifiers."); - } - } - - return rowKeyIndex; - } - - /** - * Get delegation token from hbase and add it to JobConf - * @param job - * @throws IOException - */ - static void addHBaseDelegationToken(JobConf job) throws IOException { - if (User.isHBaseSecurityEnabled(job)) { - try { - User.getCurrent().obtainAuthTokenForJob(job); - } catch (InterruptedException e) { - throw new IOException("Error while obtaining hbase delegation token", e); - } - } - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java deleted file mode 100644 index 61a439ee79dd..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase; - -import java.io.Serializable; -import java.util.Map; - - -/** - * The class HCatTableSnapshot represents a snapshot of a hcatalog table. - * This class is intended to be opaque. This class would used only by the - * record readers to obtain knowledge about the revisions of a - * column to be filtered. - */ -public class HCatTableSnapshot implements Serializable { - - private static final long serialVersionUID = 1L; - private String tableName; - private String databaseName; - private Map columnMap; - private long latestRevision; - - HCatTableSnapshot(String databaseName, String tableName, Map columnMap, long latestRevision) { - this.tableName = tableName; - this.databaseName = databaseName; - this.columnMap = columnMap; - this.latestRevision = latestRevision; - } - - /** - * @return The name of the table in the snapshot. - */ - public String getTableName() { - return this.tableName; - } - - /** - * @return The name of the database to which the table snapshot belongs. - */ - public String getDatabaseName() { - return this.databaseName; - } - - /** - * @return The revision number of a column in a snapshot. - */ - long getRevision(String column) { - if (columnMap.containsKey(column)) - return this.columnMap.get(column); - return latestRevision; - } - - /** - * The method checks if the snapshot contains information about a data column. - * - * @param column The data column of the table - * @return true, if successful - */ - boolean containsColumn(String column) { - return this.columnMap.containsKey(column); - } - - /** - * @return latest committed revision when snapshot was taken - */ - long getLatestRevision() { - return latestRevision; - } - - @Override - public String toString() { - String snapshot = " Database Name: " + this.databaseName + " Table Name : " + tableName + - "Latest Revision: " + latestRevision + " Column revision : " + columnMap.toString(); - return snapshot; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java deleted file mode 100644 index 6f18846eef14..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hcatalog.mapreduce.InputJobInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The Class HbaseSnapshotRecordReader implements logic for filtering records - * based on snapshot. - */ -class HbaseSnapshotRecordReader implements RecordReader { - - static final Logger LOG = LoggerFactory.getLogger(HbaseSnapshotRecordReader.class); - private final InputJobInfo inpJobInfo; - private final Configuration conf; - private final int maxRevisions = 1; - private ResultScanner scanner; - private Scan scan; - private HTable htable; - private TableSnapshot snapshot; - private Iterator resultItr; - private Set allAbortedTransactions; - private DataOutputBuffer valueOut = new DataOutputBuffer(); - private DataInputBuffer valueIn = new DataInputBuffer(); - - HbaseSnapshotRecordReader(InputJobInfo inputJobInfo, Configuration conf) throws IOException { - this.inpJobInfo = inputJobInfo; - this.conf = conf; - String snapshotString = conf.get(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY); - HCatTableSnapshot hcatSnapshot = (HCatTableSnapshot) HCatUtil - .deserialize(snapshotString); - this.snapshot = HBaseRevisionManagerUtil.convertSnapshot(hcatSnapshot, - inpJobInfo.getTableInfo()); - } - - public void init() throws IOException { - restart(scan.getStartRow()); - } - - public void restart(byte[] firstRow) throws IOException { - allAbortedTransactions = getAbortedTransactions(Bytes.toString(htable.getTableName()), scan); - long maxValidRevision = getMaximumRevision(scan, snapshot); - while (allAbortedTransactions.contains(maxValidRevision)) { - maxValidRevision--; - } - Scan newScan = new Scan(scan); - newScan.setStartRow(firstRow); - //TODO: See if filters in 0.92 can be used to optimize the scan - //TODO: Consider create a custom snapshot filter - //TODO: Make min revision a constant in RM - newScan.setTimeRange(0, maxValidRevision + 1); - newScan.setMaxVersions(); - this.scanner = this.htable.getScanner(newScan); - resultItr = this.scanner.iterator(); - } - - private Set getAbortedTransactions(String tableName, Scan scan) throws IOException { - Set abortedTransactions = new HashSet(); - RevisionManager rm = null; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - byte[][] families = scan.getFamilies(); - for (byte[] familyKey : families) { - String family = Bytes.toString(familyKey); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - tableName, family); - if (abortedWriteTransactions != null) { - for (FamilyRevision revision : abortedWriteTransactions) { - abortedTransactions.add(revision.getRevision()); - } - } - } - return abortedTransactions; - } finally { - HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm); - } - } - - private long getMaximumRevision(Scan scan, TableSnapshot snapshot) { - long maxRevision = 0; - byte[][] families = scan.getFamilies(); - for (byte[] familyKey : families) { - String family = Bytes.toString(familyKey); - long revision = snapshot.getRevision(family); - if (revision > maxRevision) - maxRevision = revision; - } - return maxRevision; - } - - /* - * @param htable The HTable ( of HBase) to use for the record reader. - * - */ - public void setHTable(HTable htable) { - this.htable = htable; - } - - /* - * @param scan The scan to be used for reading records. - * - */ - public void setScan(Scan scan) { - this.scan = scan; - } - - @Override - public ImmutableBytesWritable createKey() { - return new ImmutableBytesWritable(); - } - - @Override - public Result createValue() { - return new Result(); - } - - @Override - public long getPos() { - // This should be the ordinal tuple in the range; - // not clear how to calculate... - return 0; - } - - @Override - public float getProgress() throws IOException { - // Depends on the total number of tuples - return 0; - } - - @Override - public boolean next(ImmutableBytesWritable key, Result value) throws IOException { - if (this.resultItr == null) { - LOG.warn("The HBase result iterator is found null. It is possible" - + " that the record reader has already been closed."); - } else { - while (resultItr.hasNext()) { - Result temp = resultItr.next(); - Result hbaseRow = prepareResult(temp.list()); - if (hbaseRow != null) { - // Update key and value. Currently no way to avoid serialization/de-serialization - // as no setters are available. - key.set(hbaseRow.getRow()); - valueOut.reset(); - hbaseRow.write(valueOut); - valueIn.reset(valueOut.getData(), valueOut.getLength()); - value.readFields(valueIn); - return true; - } - - } - } - return false; - } - - private Result prepareResult(List keyvalues) { - - List finalKeyVals = new ArrayList(); - Map> qualValMap = new HashMap>(); - for (KeyValue kv : keyvalues) { - byte[] cf = kv.getFamily(); - byte[] qualifier = kv.getQualifier(); - String key = Bytes.toString(cf) + ":" + Bytes.toString(qualifier); - List kvs; - if (qualValMap.containsKey(key)) { - kvs = qualValMap.get(key); - } else { - kvs = new ArrayList(); - } - - String family = Bytes.toString(kv.getFamily()); - //Ignore aborted transactions - if (allAbortedTransactions.contains(kv.getTimestamp())) { - continue; - } - - long desiredTS = snapshot.getRevision(family); - if (kv.getTimestamp() <= desiredTS) { - kvs.add(kv); - } - qualValMap.put(key, kvs); - } - - Set keys = qualValMap.keySet(); - for (String cf : keys) { - List kvs = qualValMap.get(cf); - if (maxRevisions <= kvs.size()) { - for (int i = 0; i < maxRevisions; i++) { - finalKeyVals.add(kvs.get(i)); - } - } else { - finalKeyVals.addAll(kvs); - } - } - - if (finalKeyVals.size() == 0) { - return null; - } else { - KeyValue[] kvArray = new KeyValue[finalKeyVals.size()]; - finalKeyVals.toArray(kvArray); - return new Result(kvArray); - } - } - - /* - * @see org.apache.hadoop.hbase.mapred.TableRecordReader#close() - */ - @Override - public void close() { - this.resultItr = null; - this.scanner.close(); - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java deleted file mode 100644 index 72c50ec3bd77..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; -import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.mapreduce.PutSortReducer; -import org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.JobStatus; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.DEFAULT_PATH; - - -/** - * MapReduce job which reads a series of Puts stored in a sequence file - * and imports the data into HBase. It needs to create the necessary HBase - * regions using HFileOutputFormat and then notify the correct region servers - * to doBulkLoad(). This will be used After an MR job has written the SequenceFile - * and data needs to be bulk loaded onto HBase. - */ -class ImportSequenceFile { - private final static Logger LOG = LoggerFactory.getLogger(ImportSequenceFile.class); - private final static String NAME = "HCatImportSequenceFile"; - private final static String IMPORTER_WORK_DIR = "_IMPORTER_MR_WORK_DIR"; - - - private static class SequenceFileImporter extends Mapper { - - @Override - public void map(ImmutableBytesWritable rowKey, Put value, - Context context) - throws IOException { - try { - context.write(new ImmutableBytesWritable(value.getRow()), value); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - private static class ImporterOutputFormat extends HFileOutputFormat { - @Override - public OutputCommitter getOutputCommitter(TaskAttemptContext context) throws IOException { - final OutputCommitter baseOutputCommitter = super.getOutputCommitter(context); - - return new OutputCommitter() { - @Override - public void setupJob(JobContext jobContext) throws IOException { - baseOutputCommitter.setupJob(jobContext); - } - - @Override - public void setupTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.setupTask(taskContext); - } - - @Override - public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { - return baseOutputCommitter.needsTaskCommit(taskContext); - } - - @Override - public void commitTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.commitTask(taskContext); - } - - @Override - public void abortTask(TaskAttemptContext taskContext) throws IOException { - baseOutputCommitter.abortTask(taskContext); - } - - @Override - public void abortJob(JobContext jobContext, JobStatus.State state) throws IOException { - try { - baseOutputCommitter.abortJob(jobContext, state); - } finally { - cleanupScratch(jobContext); - } - } - - @Override - public void commitJob(JobContext jobContext) throws IOException { - try { - baseOutputCommitter.commitJob(jobContext); - Configuration conf = jobContext.getConfiguration(); - try { - //import hfiles - new LoadIncrementalHFiles(conf) - .doBulkLoad(HFileOutputFormat.getOutputPath(jobContext), - new HTable(conf, - conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY))); - } catch (Exception e) { - throw new IOException("BulkLoad failed.", e); - } - } finally { - cleanupScratch(jobContext); - } - } - - @Override - public void cleanupJob(JobContext context) throws IOException { - try { - baseOutputCommitter.cleanupJob(context); - } finally { - cleanupScratch(context); - } - } - - private void cleanupScratch(JobContext context) throws IOException { - FileSystem fs = FileSystem.get(context.getConfiguration()); - fs.delete(HFileOutputFormat.getOutputPath(context), true); - } - }; - } - } - - private static Job createSubmittableJob(Configuration conf, String tableName, Path inputDir, Path scratchDir, boolean localMode) - throws IOException { - Job job = new Job(conf, NAME + "_" + tableName); - job.setJarByClass(SequenceFileImporter.class); - FileInputFormat.setInputPaths(job, inputDir); - job.setInputFormatClass(SequenceFileInputFormat.class); - job.setMapperClass(SequenceFileImporter.class); - - HTable table = new HTable(conf, tableName); - job.setReducerClass(PutSortReducer.class); - FileOutputFormat.setOutputPath(job, scratchDir); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(Put.class); - HFileOutputFormat.configureIncrementalLoad(job, table); - //override OutputFormatClass with our own so we can include cleanup in the committer - job.setOutputFormatClass(ImporterOutputFormat.class); - - //local mode doesn't support symbolic links so we have to manually set the actual path - if (localMode) { - String partitionFile = null; - for (URI uri : DistributedCache.getCacheFiles(job.getConfiguration())) { - if (DEFAULT_PATH.equals(uri.getFragment())) { - partitionFile = uri.toString(); - break; - } - } - partitionFile = partitionFile.substring(0, partitionFile.lastIndexOf("#")); - job.getConfiguration().set(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString()); - } - - return job; - } - - /** - * Method to run the Importer MapReduce Job. Normally will be called by another MR job - * during OutputCommitter.commitJob(). - * @param parentContext JobContext of the parent job - * @param tableName name of table to bulk load data into - * @param InputDir path of SequenceFile formatted data to read - * @param scratchDir temporary path for the Importer MR job to build the HFiles which will be imported - * @return - */ - static boolean runJob(JobContext parentContext, String tableName, Path InputDir, Path scratchDir) { - Configuration parentConf = parentContext.getConfiguration(); - Configuration conf = new Configuration(); - for (Map.Entry el : parentConf) { - if (el.getKey().startsWith("hbase.")) - conf.set(el.getKey(), el.getValue()); - if (el.getKey().startsWith("mapred.cache.archives")) - conf.set(el.getKey(), el.getValue()); - } - - //Inherit jar dependencies added to distributed cache loaded by parent job - conf.set("mapred.job.classpath.archives", parentConf.get("mapred.job.classpath.archives", "")); - conf.set("mapreduce.job.cache.archives.visibilities", parentConf.get("mapreduce.job.cache.archives.visibilities", "")); - - //Temporary fix until hbase security is ready - //We need the written HFile to be world readable so - //hbase regionserver user has the privileges to perform a hdfs move - if (parentConf.getBoolean("hadoop.security.authorization", false)) { - FsPermission.setUMask(conf, FsPermission.valueOf("----------")); - } - - conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - conf.setBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, false); - - boolean localMode = "local".equals(conf.get("mapred.job.tracker")); - - boolean success = false; - try { - FileSystem fs = FileSystem.get(parentConf); - Path workDir = new Path(new Job(parentConf).getWorkingDirectory(), IMPORTER_WORK_DIR); - if (!fs.mkdirs(workDir)) - throw new IOException("Importer work directory already exists: " + workDir); - Job job = createSubmittableJob(conf, tableName, InputDir, scratchDir, localMode); - job.setWorkingDirectory(workDir); - job.getCredentials().addAll(parentContext.getCredentials()); - success = job.waitForCompletion(true); - fs.delete(workDir, true); - //We only cleanup on success because failure might've been caused by existence of target directory - if (localMode && success) { - new ImporterOutputFormat().getOutputCommitter(org.apache.hadoop.mapred.HCatMapRedUtil.createTaskAttemptContext(conf, new TaskAttemptID())).commitJob(job); - } - } catch (InterruptedException e) { - LOG.error("ImportSequenceFile Failed", e); - } catch (ClassNotFoundException e) { - LOG.error("ImportSequenceFile Failed", e); - } catch (IOException e) { - LOG.error("ImportSequenceFile Failed", e); - } - return success; - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java deleted file mode 100644 index 7f2418a7f867..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ResultConverter.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hcatalog.data.HCatRecord; - -import java.io.IOException; - -/** - * Interface used to define conversion of HCatRecord to and from Native HBase write (Put) and read (Result) objects. - * How the actual mapping is defined between an HBase Table's schema and an HCatalog Table's schema - * is up to the underlying implementation - */ -interface ResultConverter { - - /** - * convert HCatRecord instance to an HBase Put, used when writing out data. - * @param record instance to convert - * @return converted Put instance - * @throws IOException - */ - Put convert(HCatRecord record) throws IOException; - - /** - * convert HBase Result to HCatRecord instance, used when reading data. - * @param result instance to convert - * @return converted Result instance - * @throws IOException - */ - HCatRecord convert(Result result) throws IOException; - - /** - * Returns the hbase columns that are required for the scan. - * @return String containing hbase columns delimited by space. - * @throws IOException - */ - String getHBaseScanColumns() throws IOException; - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java deleted file mode 100644 index b5103fc1afde..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/FamilyRevision.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - - -/** - * A FamiliyRevision class consists of a revision number and a expiration - * timestamp. When a write transaction starts, the transaction - * object is appended to the transaction list of the each column - * family and stored in the corresponding znode. When a write transaction is - * committed, the transaction object is removed from the list. - */ -public class FamilyRevision implements - Comparable { - - private long revision; - - private long timestamp; - - /** - * Create a FamilyRevision object - * @param rev revision number - * @param ts expiration timestamp - */ - FamilyRevision(long rev, long ts) { - this.revision = rev; - this.timestamp = ts; - } - - public long getRevision() { - return revision; - } - - public long getExpireTimestamp() { - return timestamp; - } - - void setExpireTimestamp(long ts) { - timestamp = ts; - } - - @Override - public String toString() { - String description = "revision: " + revision + " ts: " + timestamp; - return description; - } - - @Override - public int compareTo(FamilyRevision o) { - long d = revision - o.getRevision(); - return (d < 0) ? -1 : (d > 0) ? 1 : 0; - } - - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java deleted file mode 100644 index a427544ae046..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/IDGenerator.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.nio.charset.Charset; - -import org.apache.hcatalog.hbase.snapshot.lock.LockListener; -import org.apache.hcatalog.hbase.snapshot.lock.WriteLock; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * This class generates revision id's for transactions. - */ -class IDGenerator implements LockListener { - - private ZooKeeper zookeeper; - private String zNodeDataLoc; - private String zNodeLockBasePath; - private long id; - private static final Logger LOG = LoggerFactory.getLogger(IDGenerator.class); - - IDGenerator(ZooKeeper zookeeper, String tableName, String idGenNode) - throws IOException { - this.zookeeper = zookeeper; - this.zNodeDataLoc = idGenNode; - this.zNodeLockBasePath = PathUtil.getLockManagementNode(idGenNode); - } - - /** - * This method obtains a revision id for a transaction. - * - * @return revision ID - * @throws IOException - */ - public long obtainID() throws IOException { - WriteLock wLock = new WriteLock(zookeeper, zNodeLockBasePath, Ids.OPEN_ACL_UNSAFE); - wLock.setLockListener(this); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException("Unable to obtain lock to obtain id."); - } else { - id = incrementAndReadCounter(); - } - } catch (KeeperException e) { - LOG.warn("Exception while obtaining lock for ID.", e); - throw new IOException("Exception while obtaining lock for ID.", e); - } catch (InterruptedException e) { - LOG.warn("Exception while obtaining lock for ID.", e); - throw new IOException("Exception while obtaining lock for ID.", e); - } finally { - wLock.unlock(); - } - return id; - } - - /** - * This method reads the latest revision ID that has been used. The ID - * returned by this method cannot be used for transaction. - * @return revision ID - * @throws IOException - */ - public long readID() throws IOException { - long curId; - try { - Stat stat = new Stat(); - byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); - curId = Long.parseLong(new String(data, Charset.forName("UTF-8"))); - } catch (KeeperException e) { - LOG.warn("Exception while reading current revision id.", e); - throw new IOException("Exception while reading current revision id.", e); - } catch (InterruptedException e) { - LOG.warn("Exception while reading current revision id.", e); - throw new IOException("Exception while reading current revision id.", e); - } - - return curId; - } - - - private long incrementAndReadCounter() throws IOException { - - long curId, usedId; - try { - Stat stat = new Stat(); - byte[] data = zookeeper.getData(this.zNodeDataLoc, false, stat); - usedId = Long.parseLong((new String(data, Charset.forName("UTF-8")))); - curId = usedId + 1; - String lastUsedID = String.valueOf(curId); - zookeeper.setData(this.zNodeDataLoc, lastUsedID.getBytes(Charset.forName("UTF-8")), -1); - - } catch (KeeperException e) { - LOG.warn("Exception while incrementing revision id.", e); - throw new IOException("Exception while incrementing revision id. ", e); - } catch (InterruptedException e) { - LOG.warn("Exception while incrementing revision id.", e); - throw new IOException("Exception while incrementing revision id. ", e); - } - - return curId; - } - - /* - * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() - */ - @Override - public void lockAcquired() { - - - } - - /* - * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() - */ - @Override - public void lockReleased() { - - } - - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java deleted file mode 100644 index 993308bfaf53..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/PathUtil.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - - -/** - * The PathUtil class is a utility class to provide information about various - * znode paths. The following is the znode structure used for storing information. - * baseDir/ClockNode - * baseDir/TrasactionBasePath - * baseDir/TrasactionBasePath/TableA/revisionID - * baseDir/TrasactionBasePath/TableA/columnFamily-1 - * baseDir/TrasactionBasePath/TableA/columnFamily-1/runnningTxns - * baseDir/TrasactionBasePath/TableA/columnFamily-1/abortedTxns - * baseDir/TrasactionBasePath/TableB/revisionID - * baseDir/TrasactionBasePath/TableB/columnFamily-1 - * baseDir/TrasactionBasePath/TableB/columnFamily-1/runnningTxns - * baseDir/TrasactionBasePath/TableB/columnFamily-1/abortedTxns - - */ -public class PathUtil { - - static final String DATA_DIR = "/data"; - static final String CLOCK_NODE = "/clock"; - - /** - * This method returns the data path associated with the currently - * running transactions of a given table and column/column family. - * @param baseDir - * @param tableName - * @param columnFamily - * @return The path of the running transactions data. - */ - static String getRunningTxnInfoPath(String baseDir, String tableName, - String columnFamily) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName + "/" + columnFamily - + "/runningTxns"; - return path; - } - - /** - * This method returns the data path associated with the aborted - * transactions of a given table and column/column family. - * @param baseDir The base directory for revision management. - * @param tableName The name of the table. - * @param columnFamily - * @return The path of the aborted transactions data. - */ - static String getAbortInformationPath(String baseDir, String tableName, - String columnFamily) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName + "/" + columnFamily - + "/abortData"; - return path; - } - - /** - * Gets the revision id node for a given table. - * - * @param baseDir the base dir for revision management. - * @param tableName the table name - * @return the revision id node path. - */ - static String getRevisionIDNode(String baseDir, String tableName) { - String rmBasePath = getTransactionBasePath(baseDir); - String revisionIDNode = rmBasePath + "/" + tableName + "/idgen"; - return revisionIDNode; - } - - /** - * Gets the lock management node for any znode that needs to be locked. - * - * @param path the path of the znode. - * @return the lock management node path. - */ - static String getLockManagementNode(String path) { - String lockNode = path + "_locknode_"; - return lockNode; - } - - /** - * This method returns the base path for the transaction data. - * - * @param baseDir The base dir for revision management. - * @return The base path for the transaction data. - */ - static String getTransactionBasePath(String baseDir) { - String txnBaseNode = baseDir + DATA_DIR; - return txnBaseNode; - } - - /** - * Gets the txn data path for a given table. - * - * @param baseDir the base dir for revision management. - * @param tableName the table name - * @return the txn data path for the table. - */ - static String getTxnDataPath(String baseDir, String tableName) { - String txnBasePath = getTransactionBasePath(baseDir); - String path = txnBasePath + "/" + tableName; - return path; - } - - /** - * This method returns the data path for clock node. - * - * @param baseDir - * @return The data path for clock. - */ - static String getClockPath(String baseDir) { - String clockNode = baseDir + CLOCK_NODE; - return clockNode; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java deleted file mode 100644 index 4d6fa80b4c53..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot; - -public class RMConstants { - public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class"; - - public static final String WRITE_TRANSACTION_TIMEOUT = "revision.manager.writeTxn.timeout"; - - public static final String ZOOKEEPER_HOSTLIST = "revision.manager.zk.hostList"; - - public static final String ZOOKEEPER_DATADIR = "revision.manager.zk.dataDir"; -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java deleted file mode 100644 index 4a6f8424f61c..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; - -import java.io.IOException; -import java.util.List; - -/** - * This interface provides APIs for implementing revision management. - */ -public interface RevisionManager { - /** - * Version property required by HBase to use this interface - * for CoprocessorProtocol / RPC. - */ - public static final long VERSION = 1L; // do not change - - /** - * Initialize the revision manager. - */ - public void initialize(Configuration conf); - - /** - * Opens the revision manager. - * - * @throws IOException - */ - public void open() throws IOException; - - /** - * Closes the revision manager. - * - * @throws IOException - */ - public void close() throws IOException; - - /** - * Setup revision management for a newly created hbase table. - * @param table the hbase table name - * @param columnFamilies the column families in the table - */ - public void createTable(String table, List columnFamilies) throws IOException; - - /** - * Remove table data from revision manager for a dropped table. - * @param table the hbase table name - */ - public void dropTable(String table) throws IOException; - - /** - * Start the write transaction. - * - * @param table - * @param families - * @return a new Transaction - * @throws IOException - */ - public Transaction beginWriteTransaction(String table, List families) - throws IOException; - - /** - * Start the write transaction. - * - * @param table - * @param families - * @param keepAlive - * @return a new Transaction - * @throws IOException - */ - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException; - - /** - * Commit the write transaction. - * - * @param transaction - * @throws IOException - */ - public void commitWriteTransaction(Transaction transaction) - throws IOException; - - /** - * Abort the write transaction. - * - * @param transaction - * @throws IOException - */ - public void abortWriteTransaction(Transaction transaction) - throws IOException; - - /** - * Get the list of aborted Transactions for a column family - * - * @param table the table name - * @param columnFamily the column family name - * @return a list of aborted WriteTransactions - * @throws java.io.IOException - */ - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException; - - /** - * Create the latest snapshot of the table. - * - * @param tableName - * @return a new snapshot - * @throws IOException - */ - public TableSnapshot createSnapshot(String tableName) throws IOException; - - /** - * Create the snapshot of the table using the revision number. - * - * @param tableName - * @param revision - * @return a new snapshot - * @throws IOException - */ - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException; - - /** - * Extends the expiration of a transaction by the time indicated by keep alive. - * - * @param transaction - * @throws IOException - */ - public void keepAlive(Transaction transaction) throws IOException; - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java deleted file mode 100644 index d5c432964532..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; - -public class RevisionManagerConfiguration { - - - public static Configuration addResources(Configuration conf) { - conf.addDefaultResource("revision-manager-default.xml"); - conf.addResource("revision-manager-site.xml"); - return conf; - } - - /** - * Creates a Configuration with Revision Manager resources - * @return a Configuration with Revision Manager resources - */ - public static Configuration create() { - Configuration conf = new Configuration(); - return addResources(conf); - } - - /** - * Creates a clone of passed configuration. - * @param that Configuration to clone. - * @return a Configuration created with the revision-manager-*.xml files plus - * the given configuration. - */ - public static Configuration create(final Configuration that) { - Configuration conf = create(); - //we need to merge things instead of doing new Configuration(that) - //because of a bug in Configuration wherein the config - //set on the MR fronted will get loaded on the backend as resouce called job.xml - //hence adding resources on the backed could potentially overwrite properties - //set on the frontend which we shouldn't be doing here - HBaseConfiguration.merge(conf, that); - return conf; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java deleted file mode 100644 index 49d9ad1e7058..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of RevisionManager as HBase RPC endpoint. This class will control the lifecycle of - * and delegate to the actual RevisionManager implementation and make it available as a service - * hosted in the HBase region server (instead of running it in the client (storage handler). - * In the case of {@link ZKBasedRevisionManager} now only the region servers need write access to - * manage revision data. - */ -public class RevisionManagerEndpoint extends BaseEndpointCoprocessor implements RevisionManagerProtocol { - - private static final Logger LOGGER = - LoggerFactory.getLogger(RevisionManagerEndpoint.class.getName()); - - private RevisionManager rmImpl = null; - - @Override - public void start(CoprocessorEnvironment env) { - super.start(env); - try { - Configuration conf = RevisionManagerConfiguration.create(env.getConfiguration()); - String className = conf.get(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, - ZKBasedRevisionManager.class.getName()); - LOGGER.debug("Using Revision Manager implementation: {}", className); - rmImpl = RevisionManagerFactory.getOpenedRevisionManager(className, conf); - } catch (IOException e) { - LOGGER.error("Failed to initialize revision manager", e); - } - } - - @Override - public void stop(CoprocessorEnvironment env) { - if (rmImpl != null) { - try { - rmImpl.close(); - } catch (IOException e) { - LOGGER.warn("Error closing revision manager.", e); - } - } - super.stop(env); - } - - @Override - public void initialize(Configuration conf) { - // do nothing, HBase controls life cycle - } - - @Override - public void open() throws IOException { - // do nothing, HBase controls life cycle - } - - @Override - public void close() throws IOException { - // do nothing, HBase controls life cycle - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - rmImpl.createTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - rmImpl.dropTable(table); - } - - @Override - public Transaction beginWriteTransaction(String table, List families) - throws IOException { - return rmImpl.beginWriteTransaction(table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - return rmImpl.beginWriteTransaction(table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) - throws IOException { - rmImpl.commitWriteTransaction(transaction); - } - - @Override - public void abortWriteTransaction(Transaction transaction) - throws IOException { - rmImpl.abortWriteTransaction(transaction); - } - - @Override - public TableSnapshot createSnapshot(String tableName) throws IOException { - return rmImpl.createSnapshot(tableName); - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException { - return rmImpl.createSnapshot(tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - rmImpl.keepAlive(transaction); - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - return rmImpl.getAbortedWriteTransactions(table, columnFamily); - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java deleted file mode 100644 index c6ee50ec1e47..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * This class is nothing but a delegate for the enclosed proxy, - * which is created upon setting the configuration. - */ -public class RevisionManagerEndpointClient implements RevisionManager, Configurable { - - private Configuration conf = null; - private RevisionManager rmProxy; - - @Override - public Configuration getConf() { - return this.conf; - } - - @Override - public void setConf(Configuration arg0) { - this.conf = arg0; - } - - @Override - public void initialize(Configuration conf) { - // do nothing - } - - @Override - public void open() throws IOException { - // clone to adjust RPC settings unique to proxy - Configuration clonedConf = new Configuration(conf); - // conf.set("hbase.ipc.client.connect.max.retries", "0"); - // conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1); - clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC - HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME); - rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class, - Bytes.toBytes("anyRow")); - rmProxy.open(); - } - - @Override - public void close() throws IOException { - rmProxy.close(); - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - rmProxy.createTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - rmProxy.dropTable(table); - } - - @Override - public Transaction beginWriteTransaction(String table, List families) throws IOException { - return rmProxy.beginWriteTransaction(table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, List families, long keepAlive) - throws IOException { - return rmProxy.beginWriteTransaction(table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) throws IOException { - rmProxy.commitWriteTransaction(transaction); - } - - @Override - public void abortWriteTransaction(Transaction transaction) throws IOException { - rmProxy.abortWriteTransaction(transaction); - } - - @Override - public List getAbortedWriteTransactions(String table, String columnFamily) - throws IOException { - return rmProxy.getAbortedWriteTransactions(table, columnFamily); - } - - @Override - public TableSnapshot createSnapshot(String tableName) throws IOException { - return rmProxy.createSnapshot(tableName); - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { - return rmProxy.createSnapshot(tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - rmProxy.keepAlive(transaction); - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java deleted file mode 100644 index 4d75b421fde8..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; - -/** - * Utility to instantiate the revision manager (not a true factory actually). - * Depends on HBase configuration to resolve ZooKeeper connection (when ZK is used). - */ -public class RevisionManagerFactory { - - public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class"; - - /** - * Gets an instance of revision manager. - * - * @param conf The configuration required to created the revision manager. - * @return the revision manager An instance of revision manager. - * @throws IOException Signals that an I/O exception has occurred. - */ - private static RevisionManager getRevisionManager(String className, Configuration conf) throws IOException { - - RevisionManager revisionMgr; - ClassLoader classLoader = Thread.currentThread() - .getContextClassLoader(); - if (classLoader == null) { - classLoader = RevisionManagerFactory.class.getClassLoader(); - } - try { - Class revisionMgrClass = Class - .forName(className, true, classLoader).asSubclass(RevisionManager.class); - revisionMgr = (RevisionManager) revisionMgrClass.newInstance(); - revisionMgr.initialize(conf); - } catch (ClassNotFoundException e) { - throw new IOException( - "The implementation class of revision manager not found.", - e); - } catch (InstantiationException e) { - throw new IOException( - "Exception encountered during instantiating revision manager implementation.", - e); - } catch (IllegalAccessException e) { - throw new IOException( - "IllegalAccessException encountered during instantiating revision manager implementation.", - e); - } catch (IllegalArgumentException e) { - throw new IOException( - "IllegalArgumentException encountered during instantiating revision manager implementation.", - e); - } - return revisionMgr; - } - - /** - * Internally used by endpoint implementation to instantiate from different configuration setting. - * @param className - * @param conf - * @return - * @throws IOException - */ - static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException { - - RevisionManager revisionMgr = RevisionManagerFactory.getRevisionManager(className, conf); - if (revisionMgr instanceof Configurable) { - ((Configurable) revisionMgr).setConf(conf); - } - revisionMgr.open(); - return revisionMgr; - } - - /** - * Gets an instance of revision manager which is opened. - * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS}, - * default is {@link ZKBasedRevisionManager}. - * @param conf revision manager configuration - * @return RevisionManager An instance of revision manager. - * @throws IOException - */ - public static RevisionManager getOpenedRevisionManager(Configuration conf) throws IOException { - String className = conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS, - ZKBasedRevisionManager.class.getName()); - return getOpenedRevisionManager(className, conf); - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java deleted file mode 100644 index 4cbde74a0e95..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; - -/** - * Interface marker to implement RevisionManager as Coprocessor. - * (needs to extend CoprocessorProtocol) - */ -public interface RevisionManagerProtocol extends RevisionManager, - CoprocessorProtocol { - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java deleted file mode 100644 index fa941577f94d..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * The snapshot for a table and a list of column families. - */ -public class TableSnapshot implements Serializable { - - private String name; - - private Map cfRevisionMap; - - private long latestRevision; - - - public TableSnapshot(String name, Map cfRevMap, long latestRevision) { - this.name = name; - if (cfRevMap == null) { - throw new IllegalArgumentException("revision map cannot be null"); - } - this.cfRevisionMap = cfRevMap; - this.latestRevision = latestRevision; - } - - /** - * Gets the table name. - * - * @return String The name of the table. - */ - public String getTableName() { - return name; - } - - /** - * Gets the column families. - * - * @return List A list of column families associated with the snapshot. - */ - public List getColumnFamilies(){ - return new ArrayList(this.cfRevisionMap.keySet()); - } - - /** - * Gets the revision. - * - * @param familyName The name of the column family. - * @return the revision - */ - public long getRevision(String familyName){ - if(cfRevisionMap.containsKey(familyName)) - return cfRevisionMap.get(familyName); - return latestRevision; - } - - /** - * @return the latest committed revision when this snapshot was taken - */ - public long getLatestRevision() { - return latestRevision; - } - - @Override - public String toString() { - String snapshot = "Table Name : " + name +" Latest Revision: " + latestRevision - + " Column Familiy revision : " + cfRevisionMap.toString(); - return snapshot; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java deleted file mode 100644 index 1d17ca517782..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/Transaction.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * This class is responsible for storing information related to - * transactions. - */ -public class Transaction implements Serializable { - - private String tableName; - private List columnFamilies = new ArrayList(); - private long timeStamp; - private long keepAlive; - private long revision; - - - Transaction(String tableName, List columnFamilies, long revision, long timestamp) { - this.tableName = tableName; - this.columnFamilies = columnFamilies; - this.timeStamp = timestamp; - this.revision = revision; - } - - /** - * @return The revision number associated with a transaction. - */ - public long getRevisionNumber() { - return this.revision; - } - - /** - * @return The table name associated with a transaction. - */ - public String getTableName() { - return tableName; - } - - /** - * @return The column families associated with a transaction. - */ - public List getColumnFamilies() { - return columnFamilies; - } - - /** - * @return The expire timestamp associated with a transaction. - */ - long getTransactionExpireTimeStamp() { - return this.timeStamp + this.keepAlive; - } - - void setKeepAlive(long seconds) { - this.keepAlive = seconds; - } - - /** - * Gets the keep alive value. - * - * @return long The keep alive value for the transaction. - */ - public long getKeepAliveValue() { - return this.keepAlive; - } - - /** - * Gets the family revision info. - * - * @return FamilyRevision An instance of FamilyRevision associated with the transaction. - */ - FamilyRevision getFamilyRevisionInfo() { - return new FamilyRevision(revision, getTransactionExpireTimeStamp()); - } - - /** - * Keep alive transaction. This methods extends the expire timestamp of a - * transaction by the "keep alive" amount. - */ - void keepAliveTransaction() { - this.timeStamp = this.timeStamp + this.keepAlive; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("Revision : "); - sb.append(this.getRevisionNumber()); - sb.append(" Timestamp : "); - sb.append(this.getTransactionExpireTimeStamp()); - sb.append("\n").append("Table : "); - sb.append(this.tableName).append("\n"); - sb.append("Column Families : "); - sb.append(this.columnFamilies.toString()); - return sb.toString(); - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java deleted file mode 100644 index f4556d1ab720..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java +++ /dev/null @@ -1,461 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hcatalog.hbase.snapshot.lock.LockListener; -import org.apache.hcatalog.hbase.snapshot.lock.WriteLock; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The service for providing revision management to Hbase tables. - */ -public class ZKBasedRevisionManager implements RevisionManager { - - private static final Logger LOG = LoggerFactory.getLogger(ZKBasedRevisionManager.class); - private String zkHostList; - private String baseDir; - private ZKUtil zkUtil; - private long writeTxnTimeout; - - - /* - * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#initialize() - */ - @Override - public void initialize(Configuration conf) { - conf = new Configuration(conf); - if (conf.get(RMConstants.ZOOKEEPER_HOSTLIST) == null) { - String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM); - int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, - HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT); - String[] splits = zkHostList.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - sb.append(','); - } - sb.deleteCharAt(sb.length() - 1); - conf.set(RMConstants.ZOOKEEPER_HOSTLIST, sb.toString()); - } - this.zkHostList = conf.get(RMConstants.ZOOKEEPER_HOSTLIST); - this.baseDir = conf.get(RMConstants.ZOOKEEPER_DATADIR); - this.writeTxnTimeout = Long.parseLong(conf.get(RMConstants.WRITE_TRANSACTION_TIMEOUT)); - } - - /** - * Open a ZooKeeper connection - * @throws java.io.IOException - */ - - public void open() throws IOException { - zkUtil = new ZKUtil(zkHostList, this.baseDir); - zkUtil.createRootZNodes(); - LOG.info("Created root znodes for revision manager."); - } - - /** - * Close Zookeeper connection - */ - public void close() { - zkUtil.closeZKConnection(); - } - - private void checkInputParams(String table, List families) { - if (table == null) { - throw new IllegalArgumentException( - "The table name must be specified for reading."); - } - if (families == null || families.isEmpty()) { - throw new IllegalArgumentException( - "At least one column family should be specified for reading."); - } - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - zkUtil.createRootZNodes(); - zkUtil.setUpZnodesForTable(table, columnFamilies); - } - - @Override - public void dropTable(String table) throws IOException { - zkUtil.deleteZNodes(table); - } - - /* @param table - /* @param families - /* @param keepAlive - /* @return - /* @throws IOException - * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List, long) - */ - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - - checkInputParams(table, families); - zkUtil.setUpZnodesForTable(table, families); - long nextId = zkUtil.nextId(table); - long expireTimestamp = zkUtil.getTimeStamp(); - Transaction transaction = new Transaction(table, families, nextId, - expireTimestamp); - if (keepAlive != -1) { - transaction.setKeepAlive(keepAlive); - } else { - transaction.setKeepAlive(writeTxnTimeout); - } - - refreshTransactionList(transaction.getTableName()); - String lockPath = prepareLockNode(table); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while beginning transaction. " - + transaction.toString()); - } else { - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, table, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.APPEND); - } - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - return transaction; - } - - /* @param table The table name. - /* @param families The column families involved in the transaction. - /* @return transaction The transaction which was started. - /* @throws IOException - * @see org.apache.hcatalog.hbase.snapshot.RevisionManager#beginWriteTransaction(java.lang.String, java.util.List) - */ - public Transaction beginWriteTransaction(String table, List families) - throws IOException { - return beginWriteTransaction(table, families, -1); - } - - /** - * This method commits a write transaction. - * @param transaction The revision information associated with transaction. - * @throws java.io.IOException - */ - public void commitWriteTransaction(Transaction transaction) throws IOException { - refreshTransactionList(transaction.getTableName()); - - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while commiting transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.REMOVE); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - LOG.info("Write Transaction committed: " + transaction.toString()); - } - - /** - * This method aborts a write transaction. - * @param transaction - * @throws java.io.IOException - */ - public void abortWriteTransaction(Transaction transaction) throws IOException { - - refreshTransactionList(transaction.getTableName()); - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while aborting transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction - .getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.REMOVE); - path = PathUtil.getAbortInformationPath(baseDir, - tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.APPEND); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - LOG.info("Write Transaction aborted: " + transaction.toString()); - } - - - /* @param transaction - /* @throws IOException - * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#keepAlive(org.apache.hcatalog.hbase.snapshot.Transaction) - */ - public void keepAlive(Transaction transaction) - throws IOException { - - refreshTransactionList(transaction.getTableName()); - transaction.keepAliveTransaction(); - String lockPath = prepareLockNode(transaction.getTableName()); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock for keep alive of transaction. " - + transaction.toString()); - } else { - String tableName = transaction.getTableName(); - List colFamilies = transaction.getColumnFamilies(); - FamilyRevision revisionData = transaction.getFamilyRevisionInfo(); - for (String cfamily : colFamilies) { - String path = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cfamily); - zkUtil.updateData(path, revisionData, - ZKUtil.UpdateMode.KEEP_ALIVE); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - } - - /* This method allows the user to create latest snapshot of a - /* table. - /* @param tableName The table whose snapshot is being created. - /* @return TableSnapshot An instance of TableSnaphot - /* @throws IOException - * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String) - */ - public TableSnapshot createSnapshot(String tableName) throws IOException { - refreshTransactionList(tableName); - long latestID = zkUtil.currentID(tableName); - HashMap cfMap = new HashMap(); - List columnFamilyNames = zkUtil.getColumnFamiliesOfTable(tableName); - - for (String cfName : columnFamilyNames) { - String cfPath = PathUtil.getRunningTxnInfoPath(baseDir, tableName, cfName); - List tranxList = zkUtil.getTransactionList(cfPath); - long version; - if (!tranxList.isEmpty()) { - Collections.sort(tranxList); - // get the smallest running Transaction ID - long runningVersion = tranxList.get(0).getRevision(); - version = runningVersion - 1; - } else { - version = latestID; - } - cfMap.put(cfName, version); - } - - TableSnapshot snapshot = new TableSnapshot(tableName, cfMap, latestID); - LOG.debug("Created snapshot For table: " + tableName + " snapshot: " + snapshot); - return snapshot; - } - - /* This method allows the user to create snapshot of a - /* table with a given revision number. - /* @param tableName - /* @param revision - /* @return TableSnapshot - /* @throws IOException - * @see org.apache.hcatalog.hbase.snapshot.RevsionManager#createSnapshot(java.lang.String, long) - */ - public TableSnapshot createSnapshot(String tableName, long revision) throws IOException { - - long currentID = zkUtil.currentID(tableName); - if (revision > currentID) { - throw new IOException( - "The revision specified in the snapshot is higher than the current revision of the table."); - } - refreshTransactionList(tableName); - HashMap cfMap = new HashMap(); - List columnFamilies = zkUtil.getColumnFamiliesOfTable(tableName); - - for (String cf : columnFamilies) { - cfMap.put(cf, revision); - } - - return new TableSnapshot(tableName, cfMap, revision); - } - - /** - * Get the list of in-progress Transactions for a column family - * @param table the table name - * @param columnFamily the column family name - * @return a list of in-progress WriteTransactions - * @throws java.io.IOException - */ - List getRunningTransactions(String table, - String columnFamily) throws IOException { - String path = PathUtil.getRunningTxnInfoPath(baseDir, table, - columnFamily); - return zkUtil.getTransactionList(path); - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - String path = PathUtil.getAbortInformationPath(baseDir, table, columnFamily); - return zkUtil.getTransactionList(path); - } - - private void refreshTransactionList(String tableName) throws IOException { - String lockPath = prepareLockNode(tableName); - WriteLock wLock = new WriteLock(zkUtil.getSession(), lockPath, - Ids.OPEN_ACL_UNSAFE); - RMLockListener myLockListener = new RMLockListener(); - wLock.setLockListener(myLockListener); - try { - boolean lockGrabbed = wLock.lock(); - if (lockGrabbed == false) { - //TO DO : Let this request queue up and try obtaining lock. - throw new IOException( - "Unable to obtain lock while refreshing transactions of table " - + tableName + "."); - } else { - List cfPaths = zkUtil - .getColumnFamiliesOfTable(tableName); - for (String cf : cfPaths) { - String runningDataPath = PathUtil.getRunningTxnInfoPath( - baseDir, tableName, cf); - zkUtil.refreshTransactions(runningDataPath); - } - - } - } catch (KeeperException e) { - throw new IOException("Exception while obtaining lock.", e); - } catch (InterruptedException e) { - throw new IOException("Exception while obtaining lock.", e); - } finally { - wLock.unlock(); - } - - } - - private String prepareLockNode(String tableName) throws IOException { - String txnDataPath = PathUtil.getTxnDataPath(this.baseDir, tableName); - String lockPath = PathUtil.getLockManagementNode(txnDataPath); - zkUtil.ensurePathExists(lockPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - return lockPath; - } - - /* - * This class is a listener class for the locks used in revision management. - * TBD: Use the following class to signal that that the lock is actually - * been granted. - */ - class RMLockListener implements LockListener { - - /* - * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockAcquired() - */ - @Override - public void lockAcquired() { - - } - - /* - * @see org.apache.hcatalog.hbase.snapshot.lock.LockListener#lockReleased() - */ - @Override - public void lockReleased() { - - } - - } - - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java deleted file mode 100644 index e52584d7990b..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java +++ /dev/null @@ -1,519 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.apache.thrift.TBase; -import org.apache.thrift.TDeserializer; -import org.apache.thrift.TSerializer; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.ZooKeeper.States; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class ZKUtil { - - private int DEFAULT_SESSION_TIMEOUT = 1000000; - private ZooKeeper zkSession; - private String baseDir; - private String connectString; - private static final Logger LOG = LoggerFactory.getLogger(ZKUtil.class); - - static enum UpdateMode { - APPEND, REMOVE, KEEP_ALIVE - } - - ; - - ZKUtil(String connection, String baseDir) { - this.connectString = connection; - this.baseDir = baseDir; - } - - /** - * This method creates znodes related to table. - * - * @param table The name of the table. - * @param families The list of column families of the table. - * @throws IOException - */ - void setUpZnodesForTable(String table, List families) - throws IOException { - - String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, table); - ensurePathExists(transactionDataTablePath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - for (String cf : families) { - String runningDataPath = PathUtil.getRunningTxnInfoPath( - this.baseDir, table, cf); - ensurePathExists(runningDataPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String abortDataPath = PathUtil.getAbortInformationPath( - this.baseDir, table, cf); - ensurePathExists(abortDataPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } - - } - - /** - * This method ensures that a given path exists in zookeeper. If the path - * does not exists, it creates one. - * - * @param path The path of znode that is required to exist. - * @param data The data to be associated with the znode. - * @param acl The ACLs required. - * @param flags The CreateMode for the znode. - * @throws IOException - */ - void ensurePathExists(String path, byte[] data, List acl, - CreateMode flags) throws IOException { - String[] dirs = path.split("/"); - String parentPath = ""; - for (String subDir : dirs) { - if (subDir.equals("") == false) { - parentPath = parentPath + "/" + subDir; - try { - Stat stat = getSession().exists(parentPath, false); - if (stat == null) { - getSession().create(parentPath, data, acl, flags); - } - } catch (Exception e) { - throw new IOException("Exception while creating path " - + parentPath, e); - } - } - } - - } - - /** - * This method returns a list of columns of a table which were used in any - * of the transactions. - * - * @param tableName The name of table. - * @return List The list of column families in table. - * @throws IOException - */ - List getColumnFamiliesOfTable(String tableName) throws IOException { - String path = PathUtil.getTxnDataPath(baseDir, tableName); - List children = null; - List columnFamlies = new ArrayList(); - try { - children = getSession().getChildren(path, false); - } catch (KeeperException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining columns of table.", e); - } catch (InterruptedException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining columns of table.", e); - } - - for (String child : children) { - if ((child.contains("idgen") == false) - && (child.contains("_locknode_") == false)) { - columnFamlies.add(child); - } - } - return columnFamlies; - } - - /** - * This method returns a time stamp for use by the transactions. - * - * @return long The current timestamp in zookeeper. - * @throws IOException - */ - long getTimeStamp() throws IOException { - long timeStamp; - Stat stat; - String clockPath = PathUtil.getClockPath(this.baseDir); - ensurePathExists(clockPath, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - try { - getSession().exists(clockPath, false); - stat = getSession().setData(clockPath, null, -1); - - } catch (KeeperException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining timestamp ", e); - } catch (InterruptedException e) { - LOG.warn("Caught: ", e); - throw new IOException("Exception while obtaining timestamp ", e); - } - timeStamp = stat.getMtime(); - return timeStamp; - } - - /** - * This method returns the next revision number to be used for any - * transaction purposes. - * - * @param tableName The name of the table. - * @return revision number The revision number last used by any transaction. - * @throws IOException - */ - long nextId(String tableName) throws IOException { - String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); - ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String lockNode = PathUtil.getLockManagementNode(idNode); - ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); - long id = idf.obtainID(); - return id; - } - - /** - * The latest used revision id of the table. - * - * @param tableName The name of the table. - * @return the long The revision number to use by any transaction. - * @throws IOException Signals that an I/O exception has occurred. - */ - long currentID(String tableName) throws IOException { - String idNode = PathUtil.getRevisionIDNode(this.baseDir, tableName); - ensurePathExists(idNode, Bytes.toBytes("0"), Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - String lockNode = PathUtil.getLockManagementNode(idNode); - ensurePathExists(lockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - IDGenerator idf = new IDGenerator(getSession(), tableName, idNode); - long id = idf.readID(); - return id; - } - - /** - * This methods retrieves the list of transaction information associated - * with each column/column family of a table. - * - * @param path The znode path - * @return List of FamilyRevision The list of transactions in the given path. - * @throws IOException - */ - List getTransactionList(String path) - throws IOException { - - byte[] data = getRawData(path, new Stat()); - ArrayList wtxnList = new ArrayList(); - if (data == null) { - return wtxnList; - } - StoreFamilyRevisionList txnList = new StoreFamilyRevisionList(); - deserialize(txnList, data); - Iterator itr = txnList.getRevisionListIterator(); - - while (itr.hasNext()) { - StoreFamilyRevision wtxn = itr.next(); - wtxnList.add(new FamilyRevision(wtxn.getRevision(), wtxn - .getTimestamp())); - } - - return wtxnList; - } - - /** - * This method returns the data associated with the path in zookeeper. - * - * @param path The znode path - * @param stat Zookeeper stat - * @return byte array The data stored in the znode. - * @throws IOException - */ - byte[] getRawData(String path, Stat stat) throws IOException { - byte[] data = null; - try { - data = getSession().getData(path, false, stat); - } catch (Exception e) { - throw new IOException( - "Exception while obtaining raw data from zookeeper path " - + path, e); - } - return data; - } - - /** - * This method created the basic znodes in zookeeper for revision - * management. - * - * @throws IOException - */ - void createRootZNodes() throws IOException { - String txnBaseNode = PathUtil.getTransactionBasePath(this.baseDir); - String clockNode = PathUtil.getClockPath(this.baseDir); - ensurePathExists(txnBaseNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - ensurePathExists(clockNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } - - /** - * This method closes the zookeeper session. - */ - void closeZKConnection() { - if (zkSession != null) { - try { - zkSession.close(); - } catch (InterruptedException e) { - LOG.warn("Close failed: ", e); - } - zkSession = null; - LOG.info("Disconnected to ZooKeeper"); - } - } - - /** - * This method returns a zookeeper session. If the current session is closed, - * then a new session is created. - * - * @return ZooKeeper An instance of zookeeper client. - * @throws IOException - */ - ZooKeeper getSession() throws IOException { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - synchronized (this) { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - zkSession = new ZooKeeper(this.connectString, - this.DEFAULT_SESSION_TIMEOUT, new ZKWatcher()); - } - } - } - return zkSession; - } - - /** - * This method updates the transaction data related to a znode. - * - * @param path The path to the transaction data. - * @param updateTx The FamilyRevision to be updated. - * @param mode The mode to update like append, update, remove. - * @throws IOException - */ - void updateData(String path, FamilyRevision updateTx, UpdateMode mode) - throws IOException { - - if (updateTx == null) { - throw new IOException( - "The transaction to be updated found to be null."); - } - List currentData = getTransactionList(path); - List newData = new ArrayList(); - boolean dataFound = false; - long updateVersion = updateTx.getRevision(); - for (FamilyRevision tranx : currentData) { - if (tranx.getRevision() != updateVersion) { - newData.add(tranx); - } else { - dataFound = true; - } - } - switch (mode) { - case REMOVE: - if (dataFound == false) { - throw new IOException( - "The transaction to be removed not found in the data."); - } - LOG.info("Removed trasaction : " + updateTx.toString()); - break; - case KEEP_ALIVE: - if (dataFound == false) { - throw new IOException( - "The transaction to be kept alove not found in the data. It might have been expired."); - } - newData.add(updateTx); - LOG.info("keep alive of transaction : " + updateTx.toString()); - break; - case APPEND: - if (dataFound == true) { - throw new IOException( - "The data to be appended already exists."); - } - newData.add(updateTx); - LOG.info("Added transaction : " + updateTx.toString()); - break; - } - - // For serialization purposes. - List newTxnList = new ArrayList(); - for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), - wtxn.getExpireTimestamp()); - newTxnList.add(newTxn); - } - StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); - byte[] newByteData = serialize(wtxnList); - - Stat stat = null; - try { - stat = zkSession.setData(path, newByteData, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } - - if (stat != null) { - LOG.info("Transaction list stored at " + path + "."); - } - - } - - /** - * Refresh transactions on a given transaction data path. - * - * @param path The path to the transaction data. - * @throws IOException Signals that an I/O exception has occurred. - */ - void refreshTransactions(String path) throws IOException { - List currentData = getTransactionList(path); - List newData = new ArrayList(); - - for (FamilyRevision tranx : currentData) { - if (tranx.getExpireTimestamp() > getTimeStamp()) { - newData.add(tranx); - } - } - - if (newData.equals(currentData) == false) { - List newTxnList = new ArrayList(); - for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), - wtxn.getExpireTimestamp()); - newTxnList.add(newTxn); - } - StoreFamilyRevisionList wtxnList = new StoreFamilyRevisionList(newTxnList); - byte[] newByteData = serialize(wtxnList); - - try { - zkSession.setData(path, newByteData, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while updating trasactional data. ", e); - } - - } - - } - - /** - * Delete table znodes. - * - * @param tableName the hbase table name - * @throws IOException Signals that an I/O exception has occurred. - */ - void deleteZNodes(String tableName) throws IOException { - String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, - tableName); - deleteRecursively(transactionDataTablePath); - } - - void deleteRecursively(String path) throws IOException { - try { - List children = getSession().getChildren(path, false); - if (children.size() != 0) { - for (String child : children) { - deleteRecursively(path + "/" + child); - } - } - getSession().delete(path, -1); - } catch (KeeperException e) { - throw new IOException( - "Exception while deleting path " + path + ".", e); - } catch (InterruptedException e) { - throw new IOException( - "Exception while deleting path " + path + ".", e); - } - } - - /** - * This method serializes a given instance of TBase object. - * - * @param obj An instance of TBase - * @return byte array The serialized data. - * @throws IOException - */ - static byte[] serialize(TBase obj) throws IOException { - if (obj == null) - return new byte[0]; - try { - TSerializer serializer = new TSerializer( - new TBinaryProtocol.Factory()); - byte[] bytes = serializer.serialize(obj); - return bytes; - } catch (Exception e) { - throw new IOException("Serialization error: ", e); - } - } - - - /** - * This method deserializes the given byte array into the TBase object. - * - * @param obj An instance of TBase - * @param data Output of deserialization. - * @throws IOException - */ - static void deserialize(TBase obj, byte[] data) throws IOException { - if (data == null || data.length == 0) - return; - try { - TDeserializer deserializer = new TDeserializer( - new TBinaryProtocol.Factory()); - deserializer.deserialize(obj, data); - } catch (Exception e) { - throw new IOException("Deserialization error: " + e.getMessage(), e); - } - } - - private class ZKWatcher implements Watcher { - public void process(WatchedEvent event) { - switch (event.getState()) { - case Expired: - LOG.info("The client session has expired. Try opening a new " - + "session and connecting again."); - zkSession = null; - break; - default: - - } - } - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java deleted file mode 100644 index 3c5f95bd67d6..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/LockListener.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -/** - * This class has two methods which are call - * back methods when a lock is acquired and - * when the lock is released. - * This class has been used as-is from the zookeeper 3.3.4 recipes minor changes - * in the package name. - */ -public interface LockListener { - /** - * call back called when the lock - * is acquired - */ - public void lockAcquired(); - - /** - * call back called when the lock is - * released. - */ - public void lockReleased(); -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java deleted file mode 100644 index 0f975896c654..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ProtocolSupport.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * A base class for protocol implementations which provides a number of higher - * level helper methods for working with ZooKeeper along with retrying synchronous - * operations if the connection to ZooKeeper closes such as - * {@link #retryOperation(ZooKeeperOperation)} - * This class has been used as-is from the zookeeper 3.4.0 recipes with - * changes in the retry delay, retry count values and package name. - */ -class ProtocolSupport { - private static final Logger LOG = LoggerFactory.getLogger(ProtocolSupport.class); - - protected final ZooKeeper zookeeper; - private AtomicBoolean closed = new AtomicBoolean(false); - private long retryDelay = 500L; - private int retryCount = 3; - private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - - public ProtocolSupport(ZooKeeper zookeeper) { - this.zookeeper = zookeeper; - } - - /** - * Closes this strategy and releases any ZooKeeper resources; but keeps the - * ZooKeeper instance open - */ - public void close() { - if (closed.compareAndSet(false, true)) { - doClose(); - } - } - - /** - * return zookeeper client instance - * @return zookeeper client instance - */ - public ZooKeeper getZookeeper() { - return zookeeper; - } - - /** - * return the acl its using - * @return the acl. - */ - public List getAcl() { - return acl; - } - - /** - * set the acl - * @param acl the acl to set to - */ - public void setAcl(List acl) { - this.acl = acl; - } - - /** - * get the retry delay in milliseconds - * @return the retry delay - */ - public long getRetryDelay() { - return retryDelay; - } - - /** - * Sets the time waited between retry delays - * @param retryDelay the retry delay - */ - public void setRetryDelay(long retryDelay) { - this.retryDelay = retryDelay; - } - - /** - * Allow derived classes to perform - * some custom closing operations to release resources - */ - protected void doClose() { - } - - - /** - * Perform the given operation, retrying if the connection fails - * @return object. it needs to be cast to the callee's expected - * return type. - */ - protected Object retryOperation(ZooKeeperOperation operation) - throws KeeperException, InterruptedException { - KeeperException exception = null; - for (int i = 0; i < retryCount; i++) { - try { - return operation.execute(); - } catch (KeeperException.SessionExpiredException e) { - LOG.warn("Session expired for: " + zookeeper + " so reconnecting due to: " + e, e); - throw e; - } catch (KeeperException.ConnectionLossException e) { - if (exception == null) { - exception = e; - } - LOG.debug("Attempt " + i + " failed with connection loss so " + - "attempting to reconnect: " + e, e); - retryDelay(i); - } - } - throw exception; - } - - /** - * Ensures that the given path exists with no data, the current - * ACL and no flags - * @param path - */ - protected void ensurePathExists(String path) { - ensureExists(path, null, acl, CreateMode.PERSISTENT); - } - - /** - * Ensures that the given path exists with the given data, ACL and flags - * @param path - * @param acl - * @param flags - */ - protected void ensureExists(final String path, final byte[] data, - final List acl, final CreateMode flags) { - try { - retryOperation(new ZooKeeperOperation() { - public boolean execute() throws KeeperException, InterruptedException { - Stat stat = zookeeper.exists(path, false); - if (stat != null) { - return true; - } - zookeeper.create(path, data, acl, flags); - return true; - } - }); - } catch (KeeperException e) { - LOG.warn("Caught: " + e, e); - } catch (InterruptedException e) { - LOG.warn("Caught: " + e, e); - } - } - - /** - * Returns true if this protocol has been closed - * @return true if this protocol is closed - */ - protected boolean isClosed() { - return closed.get(); - } - - /** - * Performs a retry delay if this is not the first attempt - * @param attemptCount the number of the attempts performed so far - */ - protected void retryDelay(int attemptCount) { - if (attemptCount > 0) { - try { - Thread.sleep(attemptCount * retryDelay); - } catch (InterruptedException e) { - LOG.debug("Failed to sleep: " + e, e); - } - } - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java deleted file mode 100644 index 6838fe9aaefd..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/WriteLock.java +++ /dev/null @@ -1,303 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; - -import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL; - -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * A protocol to implement an exclusive - * write lock or to elect a leader.

You invoke {@link #lock()} to - * start the process of grabbing the lock; you may get the lock then or it may be - * some time later.

You can register a listener so that you are invoked - * when you get the lock; otherwise you can ask if you have the lock - * by calling {@link #isOwner()} - * This class has been used as-is from the zookeeper 3.4.0 recipes. The only change - * made is a TODO for sorting using suffixes and the package name. - */ -public class WriteLock extends ProtocolSupport { - private static final Logger LOG = LoggerFactory.getLogger(WriteLock.class); - - private final String dir; - private String id; - private ZNodeName idName; - private String ownerId; - private String lastChildId; - private byte[] data = {0x12, 0x34}; - private LockListener callback; - private LockZooKeeperOperation zop; - - /** - * zookeeper contructor for writelock - * @param zookeeper zookeeper client instance - * @param dir the parent path you want to use for locking - * @param acl the acls that you want to use for all the paths, - * if null world read/write is used. - */ - public WriteLock(ZooKeeper zookeeper, String dir, List acl) { - super(zookeeper); - this.dir = dir; - if (acl != null) { - setAcl(acl); - } - this.zop = new LockZooKeeperOperation(); - } - - /** - * zookeeper contructor for writelock with callback - * @param zookeeper the zookeeper client instance - * @param dir the parent path you want to use for locking - * @param acl the acls that you want to use for all the paths - * @param callback the call back instance - */ - public WriteLock(ZooKeeper zookeeper, String dir, List acl, - LockListener callback) { - this(zookeeper, dir, acl); - this.callback = callback; - } - - /** - * return the current locklistener - * @return the locklistener - */ - public LockListener getLockListener() { - return this.callback; - } - - /** - * register a different call back listener - * @param callback the call back instance - */ - public void setLockListener(LockListener callback) { - this.callback = callback; - } - - /** - * Removes the lock or associated znode if - * you no longer require the lock. this also - * removes your request in the queue for locking - * in case you do not already hold the lock. - * @throws RuntimeException throws a runtime exception - * if it cannot connect to zookeeper. - */ - public synchronized void unlock() throws RuntimeException { - - if (!isClosed() && id != null) { - // we don't need to retry this operation in the case of failure - // as ZK will remove ephemeral files and we don't wanna hang - // this process when closing if we cannot reconnect to ZK - try { - - ZooKeeperOperation zopdel = new ZooKeeperOperation() { - public boolean execute() throws KeeperException, - InterruptedException { - zookeeper.delete(id, -1); - return Boolean.TRUE; - } - }; - zopdel.execute(); - } catch (InterruptedException e) { - LOG.warn("Caught: " + e, e); - //set that we have been interrupted. - Thread.currentThread().interrupt(); - } catch (KeeperException.NoNodeException e) { - // do nothing - } catch (KeeperException e) { - LOG.warn("Caught: " + e, e); - throw (RuntimeException) new RuntimeException(e.getMessage()). - initCause(e); - } finally { - if (callback != null) { - callback.lockReleased(); - } - id = null; - } - } - } - - /** - * the watcher called on - * getting watch while watching - * my predecessor - */ - private class LockWatcher implements Watcher { - public void process(WatchedEvent event) { - // lets either become the leader or watch the new/updated node - LOG.debug("Watcher fired on path: " + event.getPath() + " state: " + - event.getState() + " type " + event.getType()); - try { - lock(); - } catch (Exception e) { - LOG.warn("Failed to acquire lock: " + e, e); - } - } - } - - /** - * a zoookeeper operation that is mainly responsible - * for all the magic required for locking. - */ - private class LockZooKeeperOperation implements ZooKeeperOperation { - - /** find if we have been created earler if not create our node - * - * @param prefix the prefix node - * @param zookeeper teh zookeeper client - * @param dir the dir paretn - * @throws KeeperException - * @throws InterruptedException - */ - private void findPrefixInChildren(String prefix, ZooKeeper zookeeper, String dir) - throws KeeperException, InterruptedException { - List names = zookeeper.getChildren(dir, false); - for (String name : names) { - if (name.startsWith(prefix)) { - id = name; - if (LOG.isDebugEnabled()) { - LOG.debug("Found id created last time: " + id); - } - break; - } - } - if (id == null) { - id = zookeeper.create(dir + "/" + prefix, data, - getAcl(), EPHEMERAL_SEQUENTIAL); - - if (LOG.isDebugEnabled()) { - LOG.debug("Created id: " + id); - } - } - - } - - /** - * the command that is run and retried for actually - * obtaining the lock - * @return if the command was successful or not - */ - public boolean execute() throws KeeperException, InterruptedException { - do { - if (id == null) { - long sessionId = zookeeper.getSessionId(); - String prefix = "x-" + sessionId + "-"; - // lets try look up the current ID if we failed - // in the middle of creating the znode - findPrefixInChildren(prefix, zookeeper, dir); - idName = new ZNodeName(id); - } - if (id != null) { - List names = zookeeper.getChildren(dir, false); - if (names.isEmpty()) { - LOG.warn("No children in: " + dir + " when we've just " + - "created one! Lets recreate it..."); - // lets force the recreation of the id - id = null; - } else { - // lets sort them explicitly (though they do seem to come back in order ususally :) - SortedSet sortedNames = new TreeSet(); - for (String name : names) { - //TODO: Just use the suffix to sort. - sortedNames.add(new ZNodeName(dir + "/" + name)); - } - ownerId = sortedNames.first().getName(); - SortedSet lessThanMe = sortedNames.headSet(idName); - if (!lessThanMe.isEmpty()) { - ZNodeName lastChildName = lessThanMe.last(); - lastChildId = lastChildName.getName(); - if (LOG.isDebugEnabled()) { - LOG.debug("watching less than me node: " + lastChildId); - } - Stat stat = zookeeper.exists(lastChildId, new LockWatcher()); - if (stat != null) { - return Boolean.FALSE; - } else { - LOG.warn("Could not find the" + - " stats for less than me: " + lastChildName.getName()); - } - } else { - if (isOwner()) { - if (callback != null) { - callback.lockAcquired(); - } - return Boolean.TRUE; - } - } - } - } - } - while (id == null); - return Boolean.FALSE; - } - } - - ; - - /** - * Attempts to acquire the exclusive write lock returning whether or not it was - * acquired. Note that the exclusive lock may be acquired some time later after - * this method has been invoked due to the current lock owner going away. - */ - public synchronized boolean lock() throws KeeperException, InterruptedException { - if (isClosed()) { - return false; - } - ensurePathExists(dir); - - return (Boolean) retryOperation(zop); - } - - /** - * return the parent dir for lock - * @return the parent dir used for locks. - */ - public String getDir() { - return dir; - } - - /** - * Returns true if this node is the owner of the - * lock (or the leader) - */ - public boolean isOwner() { - return id != null && ownerId != null && id.equals(ownerId); - } - - /** - * return the id for this lock - * @return the id for this lock - */ - public String getId() { - return this.id; - } -} - diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java deleted file mode 100644 index 51f0f1895265..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZNodeName.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents an ephemeral znode name which has an ordered sequence number - * and can be sorted in order - * This class has been used as-is from the zookeeper 3.4.0 recipes with a - * change in package name. - */ -public class ZNodeName implements Comparable { - private final String name; - private String prefix; - private int sequence = -1; - private static final Logger LOG = LoggerFactory.getLogger(ZNodeName.class); - - public ZNodeName(String name) { - if (name == null) { - throw new NullPointerException("id cannot be null"); - } - this.name = name; - this.prefix = name; - int idx = name.lastIndexOf('-'); - if (idx >= 0) { - this.prefix = name.substring(0, idx); - try { - this.sequence = Integer.parseInt(name.substring(idx + 1)); - // If an exception occurred we misdetected a sequence suffix, - // so return -1. - } catch (NumberFormatException e) { - LOG.info("Number format exception for " + idx, e); - } catch (ArrayIndexOutOfBoundsException e) { - LOG.info("Array out of bounds for " + idx, e); - } - } - } - - @Override - public String toString() { - return name.toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ZNodeName sequence = (ZNodeName) o; - - if (!name.equals(sequence.name)) return false; - - return true; - } - - @Override - public int hashCode() { - return name.hashCode() + 37; - } - - public int compareTo(ZNodeName that) { - int answer = this.prefix.compareTo(that.prefix); - if (answer == 0) { - int s1 = this.sequence; - int s2 = that.sequence; - if (s1 == -1 && s2 == -1) { - return this.name.compareTo(that.name); - } - answer = s1 == -1 ? 1 : s2 == -1 ? -1 : s1 - s2; - } - return answer; - } - - /** - * Returns the name of the znode - */ - public String getName() { - return name; - } - - /** - * Returns the sequence number - */ - public int getZNodeName() { - return sequence; - } - - /** - * Returns the text prefix before the sequence number - */ - public String getPrefix() { - return prefix; - } -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java deleted file mode 100644 index 929112541be4..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/lock/ZooKeeperOperation.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.KeeperException; - -/** - * A callback object which can be used for implementing retry-able operations in the - * {@link org.apache.hcatalog.hbase.snapshot.lock.ProtocolSupport} class - * This class has been used as-is from the zookeeper 3.4.0 with change in the - * package name . - */ -public interface ZooKeeperOperation { - - /** - * Performs the operation - which may be involved multiple times if the connection - * to ZooKeeper closes during this operation - * - * @return the result of the operation or null - * @throws KeeperException - * @throws InterruptedException - */ - public boolean execute() throws KeeperException, InterruptedException; -} diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java deleted file mode 100644 index 822b2b6041e2..000000000000 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Provides a revision manager for data stored in HBase that can be used to implement repeatable reads. - * The component is designed to be usable for revision management of data stored in HBase in general, - * independent and not limited to HCatalog. It is used by the HCatalog HBase storage handler, implementation depends on HBase 0.92+. - *

- * For more information please see - * Snapshots and Repeatable reads for HBase Tables. - * @since 0.4 - */ -package org.apache.hcatalog.hbase.snapshot; diff --git a/hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml b/hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml deleted file mode 100644 index f5eca5831a4f..000000000000 --- a/hcatalog/storage-handlers/hbase/src/resources/revision-manager-default.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - revision.manager.impl.class - org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager - - Which revision manager implementation to use. - - - - revision.manager.endpoint.impl.class - org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager - - - - revision.manager.zk.dataDir - /revision-management - zookeeper base directory - - - revision.manager.writeTxn.timeout - 14400000 - timeout for write transactions in seconds - - diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java index d48160cd25f7..378f78133a00 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java @@ -177,7 +177,6 @@ public Context(String handle) { } catch (IOException e) { throw new IllegalStateException("Failed to generate testDir", e); } - System.out.println("Cluster work directory: " + testDir); } public void start() { @@ -201,7 +200,6 @@ public void stop() { cluster.stop(); cluster = null; } finally { - System.out.println("Trying to cleanup: " + testDir); try { FileSystem fs = FileSystem.get(jobConf); fs.delete(new Path(testDir), true); diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java deleted file mode 100644 index 54c65cde241d..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java +++ /dev/null @@ -1,631 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hcatalog.cli.HCatDriver; -import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.data.DefaultHCatRecord; -import org.apache.hcatalog.data.HCatRecord; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter; -import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapReadAbortedTransaction; -import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapWriteAbortTransaction; -import org.apache.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hcatalog.hbase.snapshot.Transaction; -import org.apache.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hcatalog.mapreduce.OutputJobInfo; - -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Tests components of HBaseHCatStorageHandler using ManyMiniCluster. - * Including ImprtSequenceFile and HBaseBulkOutputFormat - */ -public class TestHBaseBulkOutputFormat extends SkeletonHBaseTest { - private final static Logger LOG = LoggerFactory.getLogger(TestHBaseBulkOutputFormat.class); - - private final HiveConf allConf; - private final HCatDriver hcatDriver; - - public TestHBaseBulkOutputFormat() { - allConf = getHiveConf(); - allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); - allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); - - //Add hbase properties - for (Map.Entry el : getHbaseConf()) - allConf.set(el.getKey(), el.getValue()); - for (Map.Entry el : getJobConf()) - allConf.set(el.getKey(), el.getValue()); - - HBaseConfiguration.merge( - allConf, - RevisionManagerConfiguration.create()); - SessionState.start(new CliSessionState(allConf)); - hcatDriver = new HCatDriver(); - } - - public static class MapWriteOldMapper implements org.apache.hadoop.mapred.Mapper { - - @Override - public void close() throws IOException { - } - - @Override - public void configure(JobConf job) { - } - - @Override - public void map(LongWritable key, Text value, - OutputCollector output, - Reporter reporter) throws IOException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - output.collect(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); - } - - } - - public static class MapWrite extends Mapper { - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - context.write(new ImmutableBytesWritable(Bytes.toBytes(vals[0])), put); - } - } - - public static class MapHCatWrite extends Mapper { - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - } - } - - @Test - public void hbaseBulkOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "hbaseBulkOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - - //create table - conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - conf.set("yarn.scheduler.capacity.root.queues", "default"); - conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:one,spanish:uno", - "2,english:two,spanish:dos", - "3,english:three,spanish:tres"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - Path interPath = new Path(methodTestDir, "inter"); - //create job - JobConf job = new JobConf(conf); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWriteOldMapper.class); - - job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); - org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormat(HBaseBulkOutputFormat.class); - org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(job, interPath); - job.setOutputCommitter(HBaseBulkOutputCommitter.class); - - //manually create transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); - Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - HCatUtil.serialize(txn)); - job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, - HCatUtil.serialize(outputJobInfo)); - } finally { - rm.close(); - } - - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - //test if scratch directory was erased - assertFalse(FileSystem.get(job).exists(interPath)); - } - - @Test - public void importSequenceFileTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "importSequenceFileTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - - //create table - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:one,spanish:uno", - "2,english:two,spanish:dos", - "3,english:three,spanish:tres"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - Path interPath = new Path(methodTestDir, "inter"); - Path scratchPath = new Path(methodTestDir, "scratch"); - - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormatClass(SequenceFileOutputFormat.class); - SequenceFileOutputFormat.setOutputPath(job, interPath); - - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(Put.class); - - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(Put.class); - - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - - job = new Job(new Configuration(allConf), testName + "_importer"); - assertTrue(ImportSequenceFile.runJob(job, tableName, interPath, scratchPath)); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - //test if scratch directory was erased - assertFalse(FileSystem.get(job.getConfiguration()).exists(scratchPath)); - } - - @Test - public void bulkModeHCatOutputFormatTest() throws Exception { - String testName = "bulkModeHCatOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - LOG.info("starting: " + testName); - - String databaseName = testName.toLowerCase(); - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - //create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapHCatWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - - job.setOutputFormatClass(HCatOutputFormat.class); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - assertTrue(job.waitForCompletion(true)); - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); - for (String el : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(el)); - } - } finally { - rm.close(); - } - - //verify - HTable table = new HTable(conf, databaseName + "." + tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - } - - @Test - public void bulkModeHCatOutputFormatTestWithDefaultDB() throws Exception { - String testName = "bulkModeHCatOutputFormatTestWithDefaultDB"; - Path methodTestDir = new Path(getTestDir(), testName); - - String databaseName = "default"; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - - //create job - Job job = new Job(conf, testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapHCatWrite.class); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - - - job.setOutputFormatClass(HCatOutputFormat.class); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, tableName, null); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - - assertTrue(job.waitForCompletion(true)); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - //test if load count is the same - assertEquals(data.length, index); - } - - @Test - public void bulkModeAbortTest() throws Exception { - String testName = "bulkModeAbortTest"; - Path methodTestDir = new Path(getTestDir(), testName); - String databaseName = testName.toLowerCase(); - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - - // include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir - + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES ('" + HBaseConstants.PROPERTY_BULK_OUTPUT_MODE_KEY + "'='true'," + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName - + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - // create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create( - new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - Path workingDir = new Path(methodTestDir, "mr_abort"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, - conf, workingDir, MapWriteAbortTransaction.class, - outputJobInfo, inputPath); - assertFalse(job.waitForCompletion(true)); - - // verify that revision manager has it as aborted transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName); - for (String family : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(family)); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - databaseName + "." + tableName, family); - assertEquals(1, abortedWriteTransactions.size()); - assertEquals(1, abortedWriteTransactions.get(0).getRevision()); - } - } finally { - rm.close(); - } - - //verify that hbase does not have any of the records. - //Since records are only written during commitJob, - //hbase should not have any records. - HTable table = new HTable(conf, databaseName + "." + tableName); - Scan scan = new Scan(); - scan.addFamily(Bytes.toBytes(familyName)); - ResultScanner scanner = table.getScanner(scan); - assertFalse(scanner.iterator().hasNext()); - - // verify that the storage handler input format returns empty results. - Path outputDir = new Path(getTestDir(), - "mapred/testHBaseTableBulkIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - job = new Job(conf, "hbase-bulk-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadAbortedTransaction.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, databaseName, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - } - - private Job configureJob(String jobName, Configuration conf, - Path workingDir, Class mapperClass, - OutputJobInfo outputJobInfo, Path inputPath) throws IOException { - Job job = new Job(conf, jobName); - job.setWorkingDirectory(workingDir); - job.setJarByClass(this.getClass()); - job.setMapperClass(mapperClass); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - job.setOutputFormatClass(HCatOutputFormat.class); - HCatOutputFormat.setOutput(job, outputJobInfo); - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - return job; - } - -} - diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java deleted file mode 100644 index 4dcef2fcdcc6..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java +++ /dev/null @@ -1,501 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapred.TableOutputFormat; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hcatalog.cli.HCatDriver; -import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.data.DefaultHCatRecord; -import org.apache.hcatalog.data.HCatRecord; -import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.hcatalog.hbase.snapshot.FamilyRevision; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hcatalog.hbase.snapshot.Transaction; -import org.apache.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hcatalog.mapreduce.HCatOutputFormat; -import org.apache.hcatalog.mapreduce.OutputJobInfo; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - -/** - * Test HBaseDirectOUtputFormat and HBaseHCatStorageHandler using a MiniCluster - */ -public class TestHBaseDirectOutputFormat extends SkeletonHBaseTest { - - private final HiveConf allConf; - private final HCatDriver hcatDriver; - - public TestHBaseDirectOutputFormat() { - allConf = getHiveConf(); - allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); - allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(), "warehouse").toString()); - - //Add hbase properties - for (Map.Entry el : getHbaseConf()) - allConf.set(el.getKey(), el.getValue()); - for (Map.Entry el : getJobConf()) - allConf.set(el.getKey(), el.getValue()); - HBaseConfiguration.merge( - allConf, - RevisionManagerConfiguration.create()); - SessionState.start(new CliSessionState(allConf)); - hcatDriver = new HCatDriver(); - } - - @Test - public void directOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { - String testName = "directOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - - String tableName = newTableName(testName).toLowerCase(); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - //create table - createTable(tableName, new String[]{familyName}); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:ONE,spanish:DOS", - "3,english:ONE,spanish:TRES"}; - - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile.txt")); - for (String line : data) - os.write(Bytes.toBytes(line + "\n")); - os.close(); - - //create job - JobConf job = new JobConf(conf); - job.setJobName(testName); - job.setWorkingDirectory(new Path(methodTestDir, "mr_work")); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapWrite.class); - - job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); - org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); - - job.setOutputFormat(HBaseDirectOutputFormat.class); - job.set(TableOutputFormat.OUTPUT_TABLE, tableName); - job.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName); - - //manually create transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); - Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName)); - outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY, - HCatUtil.serialize(txn)); - job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, - HCatUtil.serialize(outputJobInfo)); - } finally { - rm.close(); - } - - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - - //verify - HTable table = new HTable(conf, tableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - } - index++; - } - assertEquals(data.length, index); - } - - @Test - public void directHCatOutputFormatTest() throws Exception { - String testName = "directHCatOutputFormatTest"; - Path methodTestDir = new Path(getTestDir(), testName); - - String databaseName = testName; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - //Table name will be lower case unless specified by hbase.table.name property - String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - - //include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES (" + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + ":spanish')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:ONE,spanish:DOS", - "3,english:ONE,spanish:TRES"}; - - // input/output settings - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - //create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create(new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - //create job - Path workingDir = new Path(methodTestDir, "mr_work"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, conf, workingDir, MapHCatWrite.class, - outputJobInfo, inputPath); - assertTrue(job.waitForCompletion(true)); - - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); - for (String el : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(el)); - } - } finally { - rm.close(); - } - - //verify - HTable table = new HTable(conf, hbaseTableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int index = 0; - for (Result result : scanner) { - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0).getTimestamp()); - } - index++; - } - assertEquals(data.length, index); - } - - @Test - public void directModeAbortTest() throws Exception { - String testName = "directModeAbortTest"; - Path methodTestDir = new Path(getTestDir(), testName); - String databaseName = testName; - String dbDir = new Path(methodTestDir, "DB_" + testName).toString(); - String tableName = newTableName(testName); - String familyName = "my_family"; - byte[] familyNameBytes = Bytes.toBytes(familyName); - //Table name as specified by hbase.table.name property - String hbaseTableName = tableName; - - // include hbase config in conf file - Configuration conf = new Configuration(allConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); - - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + dbDir - + "'"; - String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + - "(key int, english string, spanish string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" + - "TBLPROPERTIES (" + - "'hbase.columns.mapping'=':key," + familyName + ":english," + familyName + - ":spanish','hbase.table.name'='" + hbaseTableName + "')"; - - assertEquals(0, hcatDriver.run(dbquery).getResponseCode()); - assertEquals(0, hcatDriver.run(tableQuery).getResponseCode()); - - String data[] = {"1,english:ONE,spanish:UNO", - "2,english:TWO,spanish:DOS", - "3,english:THREE,spanish:TRES"}; - - Path inputPath = new Path(methodTestDir, "mr_input"); - getFileSystem().mkdirs(inputPath); - // create multiple files so we can test with multiple mappers - for (int i = 0; i < data.length; i++) { - FSDataOutputStream os = getFileSystem().create( - new Path(inputPath, "inputFile" + i + ".txt")); - os.write(Bytes.toBytes(data[i] + "\n")); - os.close(); - } - - Path workingDir = new Path(methodTestDir, "mr_abort"); - OutputJobInfo outputJobInfo = OutputJobInfo.create(databaseName, - tableName, null); - Job job = configureJob(testName, conf, workingDir, MapWriteAbortTransaction.class, - outputJobInfo, inputPath); - assertFalse(job.waitForCompletion(true)); - - // verify that revision manager has it as aborted transaction - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf); - try { - TableSnapshot snapshot = rm.createSnapshot(hbaseTableName); - for (String family : snapshot.getColumnFamilies()) { - assertEquals(1, snapshot.getRevision(family)); - List abortedWriteTransactions = rm.getAbortedWriteTransactions( - hbaseTableName, family); - assertEquals(1, abortedWriteTransactions.size()); - assertEquals(1, abortedWriteTransactions.get(0).getRevision()); - } - } finally { - rm.close(); - } - - // verify that hbase has the records of the successful maps. - HTable table = new HTable(conf, hbaseTableName); - Scan scan = new Scan(); - scan.addFamily(familyNameBytes); - ResultScanner scanner = table.getScanner(scan); - int count = 0; - for (Result result : scanner) { - String key = Bytes.toString(result.getRow()); - assertNotSame(MapWriteAbortTransaction.failedKey, key); - int index = Integer.parseInt(key) - 1; - String vals[] = data[index].toString().split(","); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - assertTrue(result.containsColumn(familyNameBytes, Bytes.toBytes(pair[0]))); - assertEquals(pair[1], - Bytes.toString(result.getValue(familyNameBytes, Bytes.toBytes(pair[0])))); - assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0) - .getTimestamp()); - } - count++; - } - assertEquals(data.length - 1, count); - - // verify that the inputformat returns empty results. - Path outputDir = new Path(getTestDir(), - "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - job = new Job(conf, "hbase-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadAbortedTransaction.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, databaseName, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - } - - private Job configureJob(String jobName, Configuration conf, - Path workingDir, Class mapperClass, - OutputJobInfo outputJobInfo, Path inputPath) throws IOException { - Job job = new Job(conf, jobName); - job.setWorkingDirectory(workingDir); - job.setJarByClass(this.getClass()); - job.setMapperClass(mapperClass); - - job.setInputFormatClass(TextInputFormat.class); - TextInputFormat.setInputPaths(job, inputPath); - job.setOutputFormatClass(HCatOutputFormat.class); - HCatOutputFormat.setOutput(job, outputJobInfo); - String txnString = job.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY); - //Test passing in same OutputJobInfo multiple times and verify 1 transaction is created - String jobString = job.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); - outputJobInfo = (OutputJobInfo) HCatUtil.deserialize(jobString); - Job job2 = new Job(conf); - HCatOutputFormat.setOutput(job2, outputJobInfo); - assertEquals(txnString, job2.getConfiguration().get(HBaseConstants.PROPERTY_WRITE_TXN_KEY)); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(HCatRecord.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(HCatRecord.class); - - job.setNumReduceTasks(0); - return job; - } - - public static class MapHCatWrite extends Mapper { - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - } - } - - public static class MapWrite implements org.apache.hadoop.mapred.Mapper { - - @Override - public void configure(JobConf job) { - } - - @Override - public void close() throws IOException { - } - - @Override - public void map(LongWritable key, Text value, - OutputCollector output, Reporter reporter) - throws IOException { - String vals[] = value.toString().split(","); - Put put = new Put(Bytes.toBytes(vals[0])); - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - put.add(Bytes.toBytes("my_family"), - Bytes.toBytes(pair[0]), - Bytes.toBytes(pair[1])); - } - output.collect(null, put); - } - } - - static class MapWriteAbortTransaction extends Mapper { - public static String failedKey; - private static int count = 0; - - @Override - public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { - OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); - HCatRecord record = new DefaultHCatRecord(3); - HCatSchema schema = jobInfo.getOutputSchema(); - String vals[] = value.toString().split(","); - record.setInteger("key", schema, Integer.parseInt(vals[0])); - synchronized (MapWriteAbortTransaction.class) { - if (count == 2) { - failedKey = vals[0]; - throw new IOException("Failing map to test abort"); - } - for (int i = 1; i < vals.length; i++) { - String pair[] = vals[i].split(":"); - record.set(pair[0], schema, pair[1]); - } - context.write(null, record); - count++; - } - - } - - } - - static class MapReadAbortedTransaction - extends - Mapper, Text> { - - @Override - public void run(Context context) throws IOException, - InterruptedException { - setup(context); - if (context.nextKeyValue()) { - map(context.getCurrentKey(), context.getCurrentValue(), context); - while (context.nextKeyValue()) { - map(context.getCurrentKey(), context.getCurrentValue(), - context); - } - throw new IOException("There should have been no records"); - } - cleanup(context); - } - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - } - } -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java index 9d29e2baf5ea..e8065fcb4c01 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,12 +9,11 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hcatalog.hbase; @@ -40,11 +39,14 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.hbase.HBaseSerDe; +import org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; @@ -67,9 +69,6 @@ import org.apache.hcatalog.data.HCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; -import org.apache.hcatalog.hbase.snapshot.Transaction; import org.apache.hcatalog.mapreduce.HCatInputFormat; import org.apache.hcatalog.mapreduce.InputJobInfo; import org.apache.hcatalog.mapreduce.PartInfo; @@ -77,19 +76,19 @@ public class TestHBaseInputFormat extends SkeletonHBaseTest { - private static HiveConf hcatConf; + private static HiveConf hcatConf; private static HCatDriver hcatDriver; - private final byte[] FAMILY = Bytes.toBytes("testFamily"); + private final byte[] FAMILY = Bytes.toBytes("testFamily"); private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1"); private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2"); - public TestHBaseInputFormat() throws Exception { + public TestHBaseInputFormat() throws Exception { hcatConf = getHiveConf(); hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); + HCatSemanticAnalyzer.class.getName()); URI fsuri = getFileSystem().getUri(); Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); + getTestDir()); hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); @@ -100,9 +99,6 @@ public TestHBaseInputFormat() throws Exception { hcatConf.set(el.getKey(), el.getValue()); } } - HBaseConfiguration.merge(hcatConf, - RevisionManagerConfiguration.create()); - SessionState.start(new CliSessionState(hcatConf)); hcatDriver = new HCatDriver(); @@ -112,70 +108,24 @@ public TestHBaseInputFormat() throws Exception { private List generatePuts(int num, String tableName) throws IOException { List columnFamilies = Arrays.asList("testFamily"); - RevisionManager rm = null; List myPuts; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); myPuts = new ArrayList(); for (int i = 1; i <= num; i++) { Put put = new Put(Bytes.toBytes("testRow")); put.add(FAMILY, QUALIFIER1, i, Bytes.toBytes("textValue-" + i)); put.add(FAMILY, QUALIFIER2, i, Bytes.toBytes("textValue-" + i)); myPuts.add(put); - Transaction tsx = rm.beginWriteTransaction(tableName, - columnFamilies); - rm.commitWriteTransaction(tsx); } - } finally { - if (rm != null) - rm.close(); - } - - return myPuts; + return myPuts; } - private void populateHBaseTable(String tName, int revisions) throws IOException { + private void populateHBaseTable(String tName, int revisions) throws IOException { List myPuts = generatePuts(revisions, tName); HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); table.put(myPuts); } - private long populateHBaseTableQualifier1(String tName, int value, Boolean commit) - throws IOException { - List columnFamilies = Arrays.asList("testFamily"); - RevisionManager rm = null; - List myPuts = new ArrayList(); - long revision; - try { - rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies); - - Put put = new Put(Bytes.toBytes("testRow")); - revision = tsx.getRevisionNumber(); - put.add(FAMILY, QUALIFIER1, revision, - Bytes.toBytes("textValue-" + value)); - myPuts.add(put); - - // If commit is null it is left as a running transaction - if (commit != null) { - if (commit) { - rm.commitWriteTransaction(tsx); - } else { - rm.abortWriteTransaction(tsx); - } - } - } finally { - if (rm != null) - rm.close(); - } - HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); - table.put(myPuts); - return revision; - } - - @Test + @Test public void TestHBaseTableReadMR() throws Exception { String tableName = newTableName("MyTable"); String databaseName = newTableName("MyDatabase"); @@ -184,11 +134,11 @@ public void TestHBaseTableReadMR() throws Exception { String db_dir = getTestDir() + "/hbasedb"; String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" - + db_dir + "'"; + + db_dir + "'"; String tableQuery = "CREATE TABLE " + databaseName + "." + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')"; + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')" ; CommandProcessorResponse responseOne = hcatDriver.run(dbquery); assertEquals(0, responseOne.getResponseCode()); @@ -202,8 +152,10 @@ public void TestHBaseTableReadMR() throws Exception { populateHBaseTable(hbaseTableName, 5); Configuration conf = new Configuration(hcatConf); conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - + HCatUtil.serialize(getHiveConf().getAllProperties())); + + conf.set(HBaseSerDe.HBASE_TABLE_NAME,hbaseTableName); + conf.set(TableInputFormat.INPUT_TABLE, hbaseTableName); // output settings Path outputDir = new Path(getTestDir(), "mapred/testHbaseTableMRRead"); FileSystem fs = getFileSystem(); @@ -217,7 +169,9 @@ public void TestHBaseTableReadMR() throws Exception { MapReadHTable.resetCounters(); job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job.getConfiguration(), databaseName, tableName); + InputJobInfo inputJobInfo = InputJobInfo.create(databaseName, tableName, + null); + HCatInputFormat.setInput(job, inputJobInfo); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, outputDir); job.setMapOutputKeyClass(BytesWritable.class); @@ -231,7 +185,7 @@ public void TestHBaseTableReadMR() throws Exception { assertFalse(MapReadHTable.error); assertEquals(MapReadHTable.count, 1); - String dropTableQuery = "DROP TABLE " + hbaseTableName; + String dropTableQuery = "DROP TABLE " + hbaseTableName ; CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); assertEquals(0, responseThree.getResponseCode()); @@ -250,11 +204,11 @@ public void TestHBaseTableProjectionReadMR() throws Exception { //Table name as specified by hbase.table.name property String hbaseTableName = "MyDB_" + tableName; String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " - + "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=" - + "':key,testFamily:testQualifier1,testFamily:testQualifier2'," - + "'hbase.table.name'='" + hbaseTableName + "')"; + + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=" + + "':key,testFamily:testQualifier1,testFamily:testQualifier2')" + + " TBLPROPERTIES ('hbase.table.name'='" + hbaseTableName+ "')" ; CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); assertEquals(0, responseTwo.getResponseCode()); @@ -267,8 +221,8 @@ public void TestHBaseTableProjectionReadMR() throws Exception { Configuration conf = new Configuration(hcatConf); conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - + HCatUtil.serialize(getHiveConf().getAllProperties())); + // output settings Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); FileSystem fs = getFileSystem(); @@ -280,8 +234,10 @@ public void TestHBaseTableProjectionReadMR() throws Exception { job.setJarByClass(this.getClass()); job.setMapperClass(MapReadProjHTable.class); job.setInputFormatClass(HCatInputFormat.class); + InputJobInfo inputJobInfo = InputJobInfo.create( + MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, null); HCatInputFormat.setOutputSchema(job, getProjectionSchema()); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); + HCatInputFormat.setInput(job, inputJobInfo); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, outputDir); job.setMapOutputKeyClass(BytesWritable.class); @@ -293,7 +249,7 @@ public void TestHBaseTableProjectionReadMR() throws Exception { assertFalse(MapReadProjHTable.error); assertEquals(MapReadProjHTable.count, 1); - String dropTableQuery = "DROP TABLE " + tableName; + String dropTableQuery = "DROP TABLE " + tableName ; CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); assertEquals(0, responseThree.getResponseCode()); @@ -301,209 +257,21 @@ public void TestHBaseTableProjectionReadMR() throws Exception { assertFalse(isHbaseTableThere); } - @Test - public void TestHBaseInputFormatProjectionReadMR() throws Exception { - - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 5); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - // output settings - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableProjectionReadMR"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - // create job - JobConf job = new JobConf(conf); - job.setJobName("hbase-scan-column"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadProjectionHTable.class); - job.setInputFormat(HBaseInputFormat.class); - - //Configure projection schema - job.set(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA, HCatUtil.serialize(getProjectionSchema())); - Job newJob = new Job(job); - HCatInputFormat.setInput(newJob, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - String inputJobString = newJob.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo info = (InputJobInfo) HCatUtil.deserialize(inputJobString); - job.set(HCatConstants.HCAT_KEY_JOB_INFO, inputJobString); - for (PartInfo partinfo : info.getPartitions()) { - for (Entry entry : partinfo.getJobProperties().entrySet()) - job.set(entry.getKey(), entry.getValue()); - } - assertEquals("testFamily:testQualifier1", job.get(TableInputFormat.SCAN_COLUMNS)); - - job.setOutputFormat(org.apache.hadoop.mapred.TextOutputFormat.class); - org.apache.hadoop.mapred.TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - - RunningJob runJob = JobClient.runJob(job); - runJob.waitForCompletion(); - assertTrue(runJob.isSuccessful()); - assertFalse(MapReadProjHTable.error); - assertEquals(MapReadProjHTable.count, 1); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - @Test - public void TestHBaseTableIgnoreAbortedTransactions() throws Exception { - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 5); - populateHBaseTableQualifier1(tableName, 6, false); - populateHBaseTableQualifier1(tableName, 7, false); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - Job job = new Job(conf, "hbase-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadHTable.class); - MapReadHTable.resetCounters(); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - // Verify that the records do not contain aborted transaction - // revisions 6 and 7 for testFamily:testQualifier1 and - // fetches revision 5 for both testQualifier1 and testQualifier2 - assertFalse(MapReadHTable.error); - assertEquals(1, MapReadHTable.count); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - @Test - public void TestHBaseTableIgnoreAbortedAndRunningTransactions() throws Exception { - String tableName = newTableName("mytable"); - String tableQuery = "CREATE TABLE " + tableName - + "(key string, testqualifier1 string, testqualifier2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key," + - "testFamily:testQualifier1,testFamily:testQualifier2')"; - - CommandProcessorResponse responseTwo = hcatDriver.run(tableQuery); - assertEquals(0, responseTwo.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists(tableName); - assertTrue(doesTableExist); - - populateHBaseTable(tableName, 2); - populateHBaseTableQualifier1(tableName, 3, Boolean.TRUE); //Committed transaction - populateHBaseTableQualifier1(tableName, 4, null); //Running transaction - populateHBaseTableQualifier1(tableName, 5, Boolean.FALSE); //Aborted transaction - populateHBaseTableQualifier1(tableName, 6, Boolean.TRUE); //Committed transaction - populateHBaseTableQualifier1(tableName, 7, null); //Running Transaction - populateHBaseTableQualifier1(tableName, 8, Boolean.FALSE); //Aborted Transaction - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - - Path outputDir = new Path(getTestDir(), "mapred/testHBaseTableIgnoreAbortedTransactions"); - FileSystem fs = getFileSystem(); - if (fs.exists(outputDir)) { - fs.delete(outputDir, true); - } - Job job = new Job(conf, "hbase-running-aborted-transaction"); - job.setJarByClass(this.getClass()); - job.setMapperClass(MapReadHTableRunningAbort.class); - job.setInputFormatClass(HCatInputFormat.class); - HCatInputFormat.setInput(job, MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName); - job.setOutputFormatClass(TextOutputFormat.class); - TextOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputKeyClass(BytesWritable.class); - job.setMapOutputValueClass(Text.class); - job.setOutputKeyClass(BytesWritable.class); - job.setOutputValueClass(Text.class); - job.setNumReduceTasks(0); - assertTrue(job.waitForCompletion(true)); - // Verify that the records do not contain running and aborted transaction - // and it fetches revision 2 for testQualifier1 and testQualifier2 - assertFalse(MapReadHTableRunningAbort.error); - assertEquals(1, MapReadHTableRunningAbort.count); - - String dropTableQuery = "DROP TABLE " + tableName; - CommandProcessorResponse responseThree = hcatDriver.run(dropTableQuery); - assertEquals(0, responseThree.getResponseCode()); - - boolean isHbaseTableThere = hAdmin.tableExists(tableName); - assertFalse(isHbaseTableThere); - } - - + static class MapReadHTable - extends - Mapper, Text> { + extends + Mapper, Text> { static boolean error = false; static int count = 0; @Override public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); + Context context) throws IOException, InterruptedException { boolean correctValues = (value.size() == 3) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-5") - && (value.get(2).toString()).equalsIgnoreCase("textValue-5"); + && (value.get(0).toString()).equalsIgnoreCase("testRow") + && (value.get(1).toString()).equalsIgnoreCase("textValue-5") + && (value.get(2).toString()).equalsIgnoreCase("textValue-5"); if (correctValues == false) { error = true; @@ -518,75 +286,17 @@ public static void resetCounters() { } static class MapReadProjHTable - extends - Mapper, Text> { + extends + Mapper, Text> { static boolean error = false; static int count = 0; - @Override public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); + Context context) throws IOException, InterruptedException { boolean correctValues = (value.size() == 2) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-5"); - - if (correctValues == false) { - error = true; - } - count++; - } - } - - static class MapReadProjectionHTable - implements org.apache.hadoop.mapred.Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void configure(JobConf job) { - } - - @Override - public void close() throws IOException { - } - - @Override - public void map(ImmutableBytesWritable key, Result result, - OutputCollector, Text> output, Reporter reporter) - throws IOException { - System.out.println("Result " + result.toString()); - List list = result.list(); - boolean correctValues = (list.size() == 1) - && (Bytes.toString(list.get(0).getRow())).equalsIgnoreCase("testRow") - && (Bytes.toString(list.get(0).getValue())).equalsIgnoreCase("textValue-5") - && (Bytes.toString(list.get(0).getFamily())).equalsIgnoreCase("testFamily") - && (Bytes.toString(list.get(0).getQualifier())).equalsIgnoreCase("testQualifier1"); - - if (correctValues == false) { - error = true; - } - count++; - } - } - - static class MapReadHTableRunningAbort - extends - Mapper, Text> { - - static boolean error = false; - static int count = 0; - - @Override - public void map(ImmutableBytesWritable key, HCatRecord value, - Context context) throws IOException, InterruptedException { - System.out.println("HCat record value" + value.toString()); - boolean correctValues = (value.size() == 3) - && (value.get(0).toString()).equalsIgnoreCase("testRow") - && (value.get(1).toString()).equalsIgnoreCase("textValue-3") - && (value.get(2).toString()).equalsIgnoreCase("textValue-2"); + && (value.get(0).toString()).equalsIgnoreCase("testRow") + && (value.get(1).toString()).equalsIgnoreCase("textValue-5"); if (correctValues == false) { error = true; @@ -599,9 +309,9 @@ private HCatSchema getProjectionSchema() throws HCatException { HCatSchema schema = new HCatSchema(new ArrayList()); schema.append(new HCatFieldSchema("key", HCatFieldSchema.Type.STRING, - "")); + "")); schema.append(new HCatFieldSchema("testqualifier1", - HCatFieldSchema.Type.STRING, "")); + HCatFieldSchema.Type.STRING, "")); return schema; } diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseStorageHandler.java similarity index 66% rename from hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java rename to hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseStorageHandler.java index a97ab4ed41f0..72d2da20c6fc 100644 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseStorageHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,12 +9,11 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hcatalog.hbase; @@ -38,12 +37,10 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hcatalog.cli.HCatDriver; import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hcatalog.hbase.snapshot.RevisionManager; -import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; import org.apache.zookeeper.KeeperException.NoNodeException; import org.junit.Test; -public class TestHBaseHCatStorageHandler extends SkeletonHBaseTest { +public class TestHiveHBaseStorageHandler extends SkeletonHBaseTest { private static HiveConf hcatConf; private static HCatDriver hcatDriver; @@ -66,10 +63,7 @@ public void Initialize() throws Exception { hcatConf.set(el.getKey(), el.getValue()); } } - HBaseConfiguration.merge( - hcatConf, - RevisionManagerConfiguration.create()); - + SessionState.start(new CliSessionState(hcatConf)); hcatDriver = new HCatDriver(); @@ -82,8 +76,8 @@ public void testTableCreateDrop() throws Exception { hcatDriver.run("drop table test_table"); CommandProcessorResponse response = hcatDriver .run("create table test_table(key int, value string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); assertEquals(0, response.getResponseCode()); @@ -92,21 +86,30 @@ public void testTableCreateDrop() throws Exception { assertTrue(doesTableExist); - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("test_table", "cf1"); - hcatDriver.run("drop table test_table"); doesTableExist = hAdmin.tableExists("test_table"); assertTrue(doesTableExist == false); - try { - rm.getAbortedWriteTransactions("test_table", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); + } + public void testHBaseTableCreateDrop() throws Exception { + Initialize(); + + hcatDriver.run("drop table test_table"); + CommandProcessorResponse response = hcatDriver + .run("create table test_table(key int, value string) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + + assertEquals(0, response.getResponseCode()); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists("test_table"); + + assertTrue(doesTableExist); + + hcatDriver.run("drop table test_table"); + doesTableExist = hAdmin.tableExists("test_table"); + assertTrue(doesTableExist == false); } @@ -117,33 +120,20 @@ public void testTableCreateDropDifferentCase() throws Exception { hcatDriver.run("drop table test_Table"); CommandProcessorResponse response = hcatDriver .run("create table test_Table(key int, value string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); assertEquals(0, response.getResponseCode()); - //HBase table gets created with lower case unless specified as a table property. + //HBase table gets created with the specific case HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); boolean doesTableExist = hAdmin.tableExists("test_table"); assertTrue(doesTableExist); - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("test_table", "cf1"); - hcatDriver.run("drop table test_table"); doesTableExist = hAdmin.tableExists("test_table"); assertTrue(doesTableExist == false); - - try { - rm.getAbortedWriteTransactions("test_table", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); - } @Test @@ -153,9 +143,9 @@ public void testTableCreateDropCaseSensitive() throws Exception { hcatDriver.run("drop table test_Table"); CommandProcessorResponse response = hcatDriver .run("create table test_Table(key int, value string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val'," + - " 'hbase.table.name'='CaseSensitiveTable')"); + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')" + + " TBLPROPERTIES ('hbase.table.name'='CaseSensitiveTable')"); assertEquals(0, response.getResponseCode()); @@ -164,22 +154,11 @@ public void testTableCreateDropCaseSensitive() throws Exception { assertTrue(doesTableExist); - RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf); - rm.open(); - //Should be able to successfully query revision manager - rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); hcatDriver.run("drop table test_table"); doesTableExist = hAdmin.tableExists("CaseSensitiveTable"); assertTrue(doesTableExist == false); - try { - rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1"); - } catch (Exception e) { - assertTrue(e.getCause() instanceof NoNodeException); - } - rm.close(); - } @Test @@ -189,8 +168,8 @@ public void testTableDropNonExistent() throws Exception { hcatDriver.run("drop table mytable"); CommandProcessorResponse response = hcatDriver .run("create table mytable(key int, value string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); assertEquals(0, response.getResponseCode()); @@ -207,7 +186,7 @@ public void testTableDropNonExistent() throws Exception { assertTrue(doesTableExist == false); CommandProcessorResponse responseTwo = hcatDriver.run("drop table mytable"); - assertTrue(responseTwo.getResponseCode() == 0); + assertTrue(responseTwo.getResponseCode() != 0); } @@ -229,9 +208,9 @@ public void testTableCreateExternal() throws Exception { hcatDriver.run("drop table mytabletwo"); CommandProcessorResponse response = hcatDriver .run("create external table mytabletwo(key int, valueone string, valuetwo string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,familyone:val,familytwo:val'," + - "'hbase.table.name'='testTable')"); + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,familyone:val,familytwo:val') " + + " TBLPROPERTIES ('hbase.table.name'='testTable')"); assertEquals(0, response.getResponseCode()); diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseTableOutputFormat.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseTableOutputFormat.java new file mode 100644 index 000000000000..c53b2a6b7544 --- /dev/null +++ b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHiveHBaseTableOutputFormat.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hcatalog.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapred.TableOutputFormat; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.hbase.HBaseSerDe; +import org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hcatalog.cli.HCatDriver; +import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hcatalog.common.ErrorType; +import org.apache.hcatalog.common.HCatConstants; +import org.apache.hcatalog.common.HCatException; +import org.apache.hcatalog.common.HCatUtil; +import org.apache.hcatalog.data.DefaultHCatRecord; +import org.apache.hcatalog.data.HCatRecord; +import org.apache.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hcatalog.mapreduce.InputJobInfo; +import org.apache.hcatalog.mapreduce.OutputJobInfo; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +/** + * Test HBaseDirectOUtputFormat and HBaseStorageHandler using a MiniCluster + */ +public class TestHiveHBaseTableOutputFormat extends SkeletonHBaseTest { + + private final HiveConf allConf; + private final HCatDriver hcatDriver; + + public TestHiveHBaseTableOutputFormat() { + allConf = getHiveConf(); + allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + HCatSemanticAnalyzer.class.getName()); + allConf.set(HiveConf.ConfVars.HADOOPFS.varname, getFileSystem().getUri().toString()); + allConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(getTestDir(),"warehouse").toString()); + + //Add hbase properties + for (Map.Entry el : getHbaseConf()) + if (el.getKey().startsWith("hbase.")) { + allConf.set(el.getKey(), el.getValue()); + } + SessionState.start(new CliSessionState(allConf)); + hcatDriver = new HCatDriver(); + } + + @Test + public void directOutputFormatTest() throws IOException, ClassNotFoundException, InterruptedException { + String testName = "directOutputFormatTest"; + Path methodTestDir = new Path(getTestDir(),testName); + + String tableName = newTableName(testName).toLowerCase(); + String familyName = "my_family"; + byte[] familyNameBytes = Bytes.toBytes(familyName); + + //include hbase config in conf file + Configuration conf = new Configuration(allConf); + conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(allConf.getAllProperties())); + + //create table + createTable(tableName,new String[]{familyName}); + + String data[] = {"1,english:ONE,spanish:UNO", + "2,english:TWO,spanish:DOS", + "3,english:THREE,spanish:TRES"}; + + // input/output settings + Path inputPath = new Path(methodTestDir,"mr_input"); + getFileSystem().mkdirs(inputPath); + FSDataOutputStream os = getFileSystem().create(new Path(inputPath,"inputFile.txt")); + for(String line: data) + os.write(Bytes.toBytes(line + "\n")); + os.close(); + + //create job + JobConf job = new JobConf(conf); + job.setJobName(testName); + job.setWorkingDirectory(new Path(methodTestDir,"mr_work")); + job.setJarByClass(this.getClass()); + job.setMapperClass(MapWrite.class); + + job.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class); + org.apache.hadoop.mapred.TextInputFormat.setInputPaths(job, inputPath); + // why we need to set all the 3 properties?? + job.setOutputFormat(HiveHBaseTableOutputFormat.class); + job.set(HBaseSerDe.HBASE_TABLE_NAME,tableName); + job.set(TableOutputFormat.OUTPUT_TABLE, tableName); + job.set(HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX+".hbase.mapreduce.outputTableName", tableName); + + try { + OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null); + job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO, + HCatUtil.serialize(outputJobInfo)); + } catch (Exception ex) { + throw new IOException("Serialization error " + ex.getMessage(), ex); + } + + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + job.setNumReduceTasks(0); + System.getProperty("java.classpath"); + RunningJob runJob = JobClient.runJob(job); + runJob.waitForCompletion(); + assertTrue(runJob.isSuccessful()); + + //verify + HTable table = new HTable(conf, tableName); + Scan scan = new Scan(); + scan.addFamily(familyNameBytes); + ResultScanner scanner = table.getScanner(scan); + int index=0; + for(Result result: scanner) { + String vals[] = data[index].toString().split(","); + for(int i=1;i mapperClass, + OutputJobInfo outputJobInfo, Path inputPath) throws IOException { + + try { + //now setting the schema + HiveConf hiveConf = HCatUtil.getHiveConf(conf); + HiveMetaStoreClient client = HCatUtil.getHiveClient(hiveConf); + Table table = client.getTable(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName()); + StorageDescriptor tblSD = table.getSd(); + if (tblSD == null) { + throw new HCatException( + "Cannot construct partition info from an empty storage descriptor."); + } + HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(tblSD.getCols())); + outputJobInfo.setOutputSchema(tableSchema); + } + catch(Exception e) { + if( e instanceof HCatException ) { + throw (HCatException) e; + } else { + throw new HCatException(ErrorType.ERROR_SET_OUTPUT, e); + } + } + conf.set(HBaseSerDe.HBASE_TABLE_NAME,outputJobInfo.getDatabaseName()+ "." + outputJobInfo.getTableName()); + conf.set(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,outputJobInfo.getDatabaseName()+ "." + outputJobInfo.getTableName()); + conf.set(TableOutputFormat.OUTPUT_TABLE, outputJobInfo.getDatabaseName() + "."+ outputJobInfo.getTableName()); + conf.set(HCatConstants.HCAT_DEFAULT_TOPIC_PREFIX+".hbase.mapreduce.outputTableName", outputJobInfo.getDatabaseName() + "." + outputJobInfo.getTableName()); + conf.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,HCatUtil.serialize(outputJobInfo)); + + Job job = new Job(conf, jobName); + job.setWorkingDirectory(workingDir); + job.setJarByClass(this.getClass()); + job.setMapperClass(mapperClass); + + job.setInputFormatClass(TextInputFormat.class); + TextInputFormat.setInputPaths(job, inputPath); + //job.setOutputFormatClass(HiveHBaseTableOutputFormat.class); + job.setOutputFormatClass(HCatOutputFormat.class); + HCatOutputFormat.setOutput(job, outputJobInfo); + job.setMapOutputKeyClass(BytesWritable.class); + job.setMapOutputValueClass(HCatRecord.class); + job.setOutputKeyClass(BytesWritable.class); + job.setOutputValueClass(HCatRecord.class); + + job.setNumReduceTasks(0); + return job; + } + + public static class MapHCatWrite extends Mapper { + + @Override + public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { + OutputJobInfo jobInfo = (OutputJobInfo)HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO)); + HCatRecord record = new DefaultHCatRecord(3); + HCatSchema schema = jobInfo.getOutputSchema(); + String vals[] = value.toString().split(","); + record.setInteger("key",schema,Integer.parseInt(vals[0])); + for(int i=1;i { + + @Override + public void configure(JobConf job) { + } + + @Override + public void close() throws IOException { + } + + @Override + public void map(LongWritable key, Text value, + OutputCollector output, Reporter reporter) + throws IOException { + String vals[] = value.toString().split(","); + Put put = new Put(Bytes.toBytes(vals[0])); + for(int i=1;i el : getHbaseConf()) { + if (el.getKey().startsWith("hbase.")) { + hcatConf.set(el.getKey(), el.getValue()); + } + } + + driver = new Driver(hcatConf); + SessionState.start(new CliSessionState(hcatConf)); + + } + + private void populateHBaseTable(String tName) throws IOException { + List myPuts = generatePuts(tName); + HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName)); + table.put(myPuts); + } + + private List generatePuts(String tableName) throws IOException { + + List columnFamilies = Arrays.asList("testFamily"); + List myPuts; + myPuts = new ArrayList(); + for (int i = 1; i <=10; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.add(FAMILY, QUALIFIER1, 1, Bytes.toBytes("textA-" + i)); + put.add(FAMILY, QUALIFIER2, 1, Bytes.toBytes("textB-" + i)); + myPuts.add(put); + } + return myPuts; + } + + public static void createTestDataFile(String filename) throws IOException { + FileWriter writer = null; + int LOOP_SIZE = 10; + float f = -100.1f; + try { + File file = new File(filename); + file.deleteOnExit(); + writer = new FileWriter(file); + + for (int i =1; i <= LOOP_SIZE; i++) { + writer.write(i+ "\t" +(f+i)+ "\t" + "textB-" + i + "\n"); + } + } finally { + if (writer != null) { + writer.close(); + } + } + + } + + @Test + public void testPigHBaseSchema() throws Exception { + Initialize(); + + String tableName = newTableName("MyTable"); + String databaseName = newTableName("MyDatabase"); + //Table name will be lower case unless specified by hbase.table.name property + String hbaseTableName = "testTable"; + String db_dir = getTestDir() + "/hbasedb"; + + String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + + db_dir + "'"; + + String deleteQuery = "DROP TABLE "+databaseName+"."+tableName; + + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key float, testqualifier1 string, testqualifier2 int) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')" + + " TBLPROPERTIES ('hbase.table.name'='"+hbaseTableName+"')"; + + CommandProcessorResponse responseOne = driver.run(deleteQuery); + assertEquals(0, responseOne.getResponseCode()); + + + CommandProcessorResponse responseTwo = driver.run(dbQuery); + assertEquals(0, responseTwo.getResponseCode()); + + + CommandProcessorResponse responseThree = driver.run(tableQuery); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(hbaseTableName); + assertTrue(doesTableExist); + + PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hcatalog.pig.HCatLoader();"); + + Schema dumpedASchema = server.dumpSchema("A"); + + List fields = dumpedASchema.getFields(); + assertEquals(3, fields.size()); + + assertEquals(DataType.FLOAT,fields.get(0).type); + assertEquals("key",fields.get(0).alias.toLowerCase()); + + assertEquals( DataType.CHARARRAY,fields.get(1).type); + assertEquals("testQualifier1".toLowerCase(), fields.get(1).alias.toLowerCase()); + + assertEquals( DataType.INTEGER,fields.get(2).type); + assertEquals("testQualifier2".toLowerCase(), fields.get(2).alias.toLowerCase()); + + } + + + @Test + public void testPigFilterProjection() throws Exception { + Initialize(); + + String tableName = newTableName("MyTable"); + String databaseName = newTableName("MyDatabase"); + //Table name will be lower case unless specified by hbase.table.name property + String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); + String db_dir = getTestDir() + "/hbasedb"; + + String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + + db_dir + "'"; + + String deleteQuery = "DROP TABLE "+databaseName+"."+tableName; + + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, testqualifier1 string, testqualifier2 string) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')" + + " TBLPROPERTIES ('hbase.table.default.storage.type'='binary')"; + + CommandProcessorResponse responseOne = driver.run(deleteQuery); + assertEquals(0, responseOne.getResponseCode()); + + + CommandProcessorResponse responseTwo = driver.run(dbQuery); + assertEquals(0, responseTwo.getResponseCode()); + + + CommandProcessorResponse responseThree = driver.run(tableQuery); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(hbaseTableName); + assertTrue(doesTableExist); + + populateHBaseTable(hbaseTableName); + + Configuration conf = new Configuration(getHbaseConf()); + HTable table = new HTable(conf, hbaseTableName); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("testFamily")); + ResultScanner scanner = table.getScanner(scan); + int index=1; + + PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + server.registerQuery("A = load '"+databaseName+"."+tableName+"' using org.apache.hcatalog.pig.HCatLoader();"); + server.registerQuery("B = filter A by key < 5;"); + server.registerQuery("C = foreach B generate key,testqualifier2;"); + Iterator itr = server.openIterator("C"); + //verify if the filter is correct and returns 2 rows and contains 2 columns and the contents match + while(itr.hasNext()){ + Tuple t = itr.next(); + assertTrue(t.size() == 2); + assertTrue(t.get(0).getClass() == Integer.class); + assertEquals(index,t.get(0)); + assertTrue(t.get(1).getClass() == String.class); + assertEquals("textB-"+index,t.get(1)); + index++; + } + assertEquals(index-1,4); + } + + @Test + public void testPigPopulation() throws Exception { + Initialize(); + + String tableName = newTableName("MyTable"); + String databaseName = newTableName("MyDatabase"); + //Table name will be lower case unless specified by hbase.table.name property + String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); + String db_dir = getTestDir() + "/hbasedb"; + String POPTXT_FILE_NAME = db_dir+"testfile.txt"; + float f = -100.1f; + + String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + + db_dir + "'"; + + String deleteQuery = "DROP TABLE "+databaseName+"."+tableName; + + String tableQuery = "CREATE TABLE " + databaseName + "." + tableName + + "(key int, testqualifier1 float, testqualifier2 string) STORED BY " + + "'org.apache.hadoop.hive.hbase.HBaseStorageHandler'" + + " WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,testFamily:testQualifier1,testFamily:testQualifier2')" + + " TBLPROPERTIES ('hbase.table.default.storage.type'='binary')"; + + + String selectQuery = "SELECT * from "+databaseName.toLowerCase()+"."+tableName.toLowerCase(); + + + CommandProcessorResponse responseOne = driver.run(deleteQuery); + assertEquals(0, responseOne.getResponseCode()); + + + CommandProcessorResponse responseTwo = driver.run(dbQuery); + assertEquals(0, responseTwo.getResponseCode()); + + + CommandProcessorResponse responseThree = driver.run(tableQuery); + + HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); + boolean doesTableExist = hAdmin.tableExists(hbaseTableName); + assertTrue(doesTableExist); + + + createTestDataFile(POPTXT_FILE_NAME); + + PigServer server = new PigServer(ExecType.LOCAL,hcatConf.getAllProperties()); + server.registerQuery("A = load '"+POPTXT_FILE_NAME+"' using PigStorage() as (key:int, testqualifier1:float, testqualifier2:chararray);"); + server.registerQuery("B = filter A by (key > 2) AND (key < 8) ;"); + server.registerQuery("store B into '"+databaseName.toLowerCase()+"."+tableName.toLowerCase()+"' using org.apache.hcatalog.pig.HCatStorer();"); + server.registerQuery("C = load '"+databaseName.toLowerCase()+"."+tableName.toLowerCase()+"' using org.apache.hcatalog.pig.HCatLoader();"); + // Schema should be same + Schema dumpedBSchema = server.dumpSchema("C"); + + List fields = dumpedBSchema.getFields(); + assertEquals(3, fields.size()); + + assertEquals(DataType.INTEGER,fields.get(0).type); + assertEquals("key",fields.get(0).alias.toLowerCase()); + + assertEquals( DataType.FLOAT,fields.get(1).type); + assertEquals("testQualifier1".toLowerCase(), fields.get(1).alias.toLowerCase()); + + assertEquals( DataType.CHARARRAY,fields.get(2).type); + assertEquals("testQualifier2".toLowerCase(), fields.get(2).alias.toLowerCase()); + + //Query the hbase table and check the key is valid and only 5 are present + Configuration conf = new Configuration(getHbaseConf()); + HTable table = new HTable(conf, hbaseTableName); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("testFamily")); + byte[] familyNameBytes = Bytes.toBytes("testFamily"); + ResultScanner scanner = table.getScanner(scan); + int index=3; + int count=0; + for(Result result: scanner) { + //key is correct + assertEquals(index,Bytes.toInt(result.getRow())); + //first column exists + assertTrue(result.containsColumn(familyNameBytes,Bytes.toBytes("testQualifier1"))); + //value is correct + assertEquals((index+f),Bytes.toFloat(result.getValue(familyNameBytes,Bytes.toBytes("testQualifier1"))),0); + + //second column exists + assertTrue(result.containsColumn(familyNameBytes,Bytes.toBytes("testQualifier2"))); + //value is correct + assertEquals(("textB-"+index).toString(),Bytes.toString(result.getValue(familyNameBytes,Bytes.toBytes("testQualifier2")))); + index++; + count++; + } + // 5 rows should be returned + assertEquals(count,5); + + //Check if hive returns results correctly + driver.run(selectQuery); + ArrayList result = new ArrayList(); + driver.getResults(result); + //Query using the hive command line + assertEquals(5, result.size()); + Iterator itr = result.iterator(); + for(int i = 3; i <= 7; i++) { + String tokens[] = itr.next().split("\\s+"); + assertEquals(i,Integer.parseInt(tokens[0])); + assertEquals(i+f,Float.parseFloat(tokens[1]),0); + assertEquals(("textB-"+i).toString(),tokens[2]); + } + + //delete the table from the database + CommandProcessorResponse responseFour = driver.run(deleteQuery); + assertEquals(0, responseFour.getResponseCode()); + + } + +} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java deleted file mode 100644 index f689cc9193cc..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase; - -import static org.junit.Assert.assertEquals; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hcatalog.cli.HCatDriver; -import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hcatalog.common.HCatConstants; -import org.apache.hcatalog.common.HCatUtil; -import org.apache.hcatalog.hbase.snapshot.TableSnapshot; -import org.apache.hcatalog.mapreduce.HCatInputFormat; -import org.apache.hcatalog.mapreduce.InputJobInfo; -import org.junit.Test; - -public class TestSnapshots extends SkeletonHBaseTest { - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - - public void Initialize() throws Exception { - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - @Test - public void TestSnapshotConversion() throws Exception { - Initialize(); - String tableName = newTableName("mytableOne"); - String databaseName = newTableName("mydatabase"); - String fullyQualTableName = databaseName + "." + tableName; - String db_dir = getTestDir() + "/hbasedb"; - String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" - + db_dir + "'"; - String tableQuery = "CREATE TABLE " + fullyQualTableName - + "(key string, value1 string, value2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf2:q2')"; - - CommandProcessorResponse cmdResponse = hcatDriver.run(dbquery); - assertEquals(0, cmdResponse.getResponseCode()); - cmdResponse = hcatDriver.run(tableQuery); - assertEquals(0, cmdResponse.getResponseCode()); - - Configuration conf = new Configuration(hcatConf); - conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, - HCatUtil.serialize(getHiveConf().getAllProperties())); - Job job = new Job(conf); - Properties properties = new Properties(); - properties.setProperty(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY, "dummysnapshot"); - HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); - String modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - InputJobInfo inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); - - Map revMap = new HashMap(); - revMap.put("cf1", 3L); - revMap.put("cf2", 5L); - TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); - HCatTableSnapshot hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); - - assertEquals(hcatSnapshot.getRevision("value1"), 3); - assertEquals(hcatSnapshot.getRevision("value2"), 5); - - String dropTable = "DROP TABLE " + fullyQualTableName; - cmdResponse = hcatDriver.run(dropTable); - assertEquals(0, cmdResponse.getResponseCode()); - - tableName = newTableName("mytableTwo"); - fullyQualTableName = databaseName + "." + tableName; - tableQuery = "CREATE TABLE " + fullyQualTableName - + "(key string, value1 string, value2 string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:q1,cf1:q2')"; - cmdResponse = hcatDriver.run(tableQuery); - assertEquals(0, cmdResponse.getResponseCode()); - revMap.clear(); - revMap.put("cf1", 3L); - hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1); - HCatInputFormat.setInput(job, databaseName, tableName).setProperties(properties); - modifiedInputInfo = job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO); - inputInfo = (InputJobInfo) HCatUtil.deserialize(modifiedInputInfo); - hcatSnapshot = HBaseRevisionManagerUtil.convertSnapshot(hbaseSnapshot, inputInfo.getTableInfo()); - assertEquals(hcatSnapshot.getRevision("value1"), 3); - assertEquals(hcatSnapshot.getRevision("value2"), 3); - - dropTable = "DROP TABLE " + fullyQualTableName; - cmdResponse = hcatDriver.run(dropTable); - assertEquals(0, cmdResponse.getResponseCode()); - - String dropDatabase = "DROP DATABASE IF EXISTS " + databaseName + "CASCADE"; - cmdResponse = hcatDriver.run(dropDatabase); - assertEquals(0, cmdResponse.getResponseCode()); - } -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java deleted file mode 100644 index 353402beb4ba..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/IDGenClient.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -public class IDGenClient extends Thread { - - String connectionStr; - String base_dir; - ZKUtil zkutil; - Random sleepTime = new Random(); - int runtime; - HashMap idMap; - String tableName; - - IDGenClient(String connectionStr, String base_dir, int time, String tableName) { - super(); - this.connectionStr = connectionStr; - this.base_dir = base_dir; - this.zkutil = new ZKUtil(connectionStr, base_dir); - this.runtime = time; - idMap = new HashMap(); - this.tableName = tableName; - } - - /* - * @see java.lang.Runnable#run() - */ - @Override - public void run() { - long startTime = System.currentTimeMillis(); - int timeElapsed = 0; - while( timeElapsed <= runtime){ - try { - long id = zkutil.nextId(tableName); - idMap.put(System.currentTimeMillis(), id); - - int sTime = sleepTime.nextInt(2); - Thread.sleep(sTime * 100); - } catch (Exception e) { - e.printStackTrace(); - } - - timeElapsed = (int) Math.ceil((System.currentTimeMillis() - startTime)/(double)1000); - } - - } - - Map getIdMap(){ - return idMap; - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java deleted file mode 100644 index 9b0cd01057aa..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; - -import org.apache.hcatalog.hbase.SkeletonHBaseTest; -import org.junit.Assert; -import org.junit.Test; - -public class TestIDGenerator extends SkeletonHBaseTest { - - @Test - public void testIDGeneration() throws Exception { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - - String tableName = "myTable"; - long initId = zkutil.nextId(tableName); - for (int i = 0; i < 10; i++) { - long id = zkutil.nextId(tableName); - Assert.assertEquals(initId + (i + 1), id); - } - } - - @Test - public void testMultipleClients() throws InterruptedException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - ArrayList clients = new ArrayList(); - - for (int i = 0; i < 5; i++) { - IDGenClient idClient = new IDGenClient(sb.toString(), "/rm_base", 10, "testTable"); - clients.add(idClient); - } - - for (IDGenClient idClient : clients) { - idClient.run(); - } - - for (IDGenClient idClient : clients) { - idClient.join(); - } - - HashMap idMap = new HashMap(); - for (IDGenClient idClient : clients) { - idMap.putAll(idClient.getIdMap()); - } - - ArrayList keys = new ArrayList(idMap.keySet()); - Collections.sort(keys); - int startId = 1; - for (Long key : keys) { - Long id = idMap.get(key); - System.out.println("Key: " + key + " Value " + id); - assertTrue(id == startId); - startId++; - - } - } -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java deleted file mode 100644 index 114895aaf28c..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java +++ /dev/null @@ -1,260 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hcatalog.hbase.SkeletonHBaseTest; -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.junit.Test; - -public class TestRevisionManager extends SkeletonHBaseTest { - - @Test - public void testBasicZNodeCreation() throws IOException, KeeperException, InterruptedException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf001", "cf002", "cf003"); - - zkutil.createRootZNodes(); - ZooKeeper zk = zkutil.getSession(); - Stat tempTwo = zk.exists("/rm_base" + PathUtil.DATA_DIR, false); - assertTrue(tempTwo != null); - Stat tempThree = zk.exists("/rm_base" + PathUtil.CLOCK_NODE, false); - assertTrue(tempThree != null); - - zkutil.setUpZnodesForTable(tableName, columnFamilies); - String transactionDataTablePath = "/rm_base" + PathUtil.DATA_DIR + "/" + tableName; - Stat result = zk.exists(transactionDataTablePath, false); - assertTrue(result != null); - - for (String colFamiliy : columnFamilies) { - String cfPath = transactionDataTablePath + "/" + colFamiliy; - Stat resultTwo = zk.exists(cfPath, false); - assertTrue(resultTwo != null); - } - - } - - @Test - public void testCommitTransaction() throws IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); - Transaction txn = manager.beginWriteTransaction(tableName, - columnFamilies); - - List cfs = zkutil.getColumnFamiliesOfTable(tableName); - assertTrue(cfs.size() == columnFamilies.size()); - for (String cf : cfs) { - assertTrue(columnFamilies.contains(cf)); - } - - for (String colFamily : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamily); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision lightTxn = list.getRevisionList().get(0); - assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); - assertEquals(lightTxn.revision, txn.getRevisionNumber()); - - } - manager.commitWriteTransaction(txn); - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 0); - - } - - manager.close(); - } - - @Test - public void testAbortTransaction() throws IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String host = getHbaseConf().get("hbase.zookeeper.quorum"); - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - ZKUtil zkutil = new ZKUtil(host + ':' + port, "/rm_base"); - - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2", "cf3"); - Transaction txn = manager.beginWriteTransaction(tableName, columnFamilies); - List cfs = zkutil.getColumnFamiliesOfTable(tableName); - - assertTrue(cfs.size() == columnFamilies.size()); - for (String cf : cfs) { - assertTrue(columnFamilies.contains(cf)); - } - - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision lightTxn = list.getRevisionList().get(0); - assertEquals(lightTxn.timestamp, txn.getTransactionExpireTimeStamp()); - assertEquals(lightTxn.revision, txn.getRevisionNumber()); - - } - manager.abortWriteTransaction(txn); - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getRunningTxnInfoPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 0); - - } - - for (String colFamiliy : columnFamilies) { - String path = PathUtil.getAbortInformationPath("/rm_base", tableName, colFamiliy); - byte[] data = zkutil.getRawData(path, null); - StoreFamilyRevisionList list = new StoreFamilyRevisionList(); - ZKUtil.deserialize(list, data); - assertEquals(list.getRevisionListSize(), 1); - StoreFamilyRevision abortedTxn = list.getRevisionList().get(0); - assertEquals(abortedTxn.getRevision(), txn.getRevisionNumber()); - } - manager.close(); - } - - @Test - public void testKeepAliveTransaction() throws InterruptedException, IOException { - - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - String tableName = newTableName("testTable"); - List columnFamilies = Arrays.asList("cf1", "cf2"); - Transaction txn = manager.beginWriteTransaction(tableName, - columnFamilies, 40); - Thread.sleep(100); - try { - manager.commitWriteTransaction(txn); - } catch (Exception e) { - assertTrue(e instanceof IOException); - assertEquals(e.getMessage(), - "The transaction to be removed not found in the data."); - } - - } - - @Test - public void testCreateSnapshot() throws IOException { - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String host = getHbaseConf().get("hbase.zookeeper.quorum"); - Configuration conf = RevisionManagerConfiguration.create(getHbaseConf()); - conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - ZKBasedRevisionManager manager = new ZKBasedRevisionManager(); - manager.initialize(conf); - manager.open(); - String tableName = newTableName("testTable"); - List cfOne = Arrays.asList("cf1", "cf2"); - List cfTwo = Arrays.asList("cf2", "cf3"); - Transaction tsx1 = manager.beginWriteTransaction(tableName, cfOne); - Transaction tsx2 = manager.beginWriteTransaction(tableName, cfTwo); - TableSnapshot snapshotOne = manager.createSnapshot(tableName); - assertEquals(snapshotOne.getRevision("cf1"), 0); - assertEquals(snapshotOne.getRevision("cf2"), 0); - assertEquals(snapshotOne.getRevision("cf3"), 1); - - List cfThree = Arrays.asList("cf1", "cf3"); - Transaction tsx3 = manager.beginWriteTransaction(tableName, cfThree); - manager.commitWriteTransaction(tsx1); - TableSnapshot snapshotTwo = manager.createSnapshot(tableName); - assertEquals(snapshotTwo.getRevision("cf1"), 2); - assertEquals(snapshotTwo.getRevision("cf2"), 1); - assertEquals(snapshotTwo.getRevision("cf3"), 1); - - manager.commitWriteTransaction(tsx2); - TableSnapshot snapshotThree = manager.createSnapshot(tableName); - assertEquals(snapshotThree.getRevision("cf1"), 2); - assertEquals(snapshotThree.getRevision("cf2"), 3); - assertEquals(snapshotThree.getRevision("cf3"), 2); - manager.commitWriteTransaction(tsx3); - TableSnapshot snapshotFour = manager.createSnapshot(tableName); - assertEquals(snapshotFour.getRevision("cf1"), 3); - assertEquals(snapshotFour.getRevision("cf2"), 3); - assertEquals(snapshotFour.getRevision("cf3"), 3); - - } - - -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java deleted file mode 100644 index 301bf92da148..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -public class TestRevisionManagerConfiguration { - - @Test - public void testDefault() { - Configuration conf = RevisionManagerConfiguration.create(); - Assert.assertEquals("org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager", - conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS)); - } -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java deleted file mode 100644 index fe9ca40fd1fa..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hcatalog.hbase.SkeletonHBaseTest; -import org.junit.Assert; -import org.junit.Test; - -public class TestRevisionManagerEndpoint extends SkeletonHBaseTest { - - static { - // test case specific mini cluster settings - testConf = new Configuration(false); - testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, - "org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpoint", - "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); - testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName()); - } - - /** - * Mock implementation to test the protocol/serialization - */ - public static class MockRM implements RevisionManager { - - private static class Invocation { - Invocation(String methodName, Object ret, Object... args) { - this.methodName = methodName; - this.args = args; - this.ret = ret; - } - - String methodName; - Object[] args; - Object ret; - - private static boolean equals(Object obj1, Object obj2) { - if (obj1 == obj2) return true; - if (obj1 == null || obj2 == null) return false; - if (obj1 instanceof Transaction || obj1 instanceof TableSnapshot) { - return obj1.toString().equals(obj2.toString()); - } - return obj1.equals(obj2); - } - - @Override - public boolean equals(Object obj) { - Invocation other = (Invocation) obj; - if (this == other) return true; - if (other == null) return false; - if (this.args != other.args) { - if (this.args == null || other.args == null) return false; - if (this.args.length != other.args.length) return false; - for (int i = 0; i < args.length; i++) { - if (!equals(this.args[i], other.args[i])) return false; - } - } - return equals(this.ret, other.ret); - } - - @Override - public String toString() { - return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE). - append("method", this.methodName). - append("args", this.args). - append("returns", this.ret). - toString(); - } - } - - final static String DEFAULT_INSTANCE = "default"; - final static Map INSTANCES = new ConcurrentHashMap(); - Invocation lastCall; - boolean isOpen = false; - - private T recordCall(T result, Object... args) { - StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - lastCall = new Invocation(stackTrace[2].getMethodName(), result, args); - return result; - } - - @Override - public void initialize(Configuration conf) { - if (!INSTANCES.containsKey(DEFAULT_INSTANCE)) - INSTANCES.put(DEFAULT_INSTANCE, this); - } - - @Override - public void open() throws IOException { - isOpen = true; - } - - @Override - public void close() throws IOException { - isOpen = false; - } - - @Override - public void createTable(String table, List columnFamilies) throws IOException { - } - - @Override - public void dropTable(String table) throws IOException { - } - - @Override - public Transaction beginWriteTransaction(String table, - List families) throws IOException { - return recordCall(null, table, families); - } - - @Override - public Transaction beginWriteTransaction(String table, - List families, long keepAlive) throws IOException { - return recordCall(null, table, families, keepAlive); - } - - @Override - public void commitWriteTransaction(Transaction transaction) - throws IOException { - } - - @Override - public void abortWriteTransaction(Transaction transaction) - throws IOException { - } - - @Override - public List getAbortedWriteTransactions(String table, - String columnFamily) throws IOException { - return null; - } - - @Override - public TableSnapshot createSnapshot(String tableName) - throws IOException { - return null; - } - - @Override - public TableSnapshot createSnapshot(String tableName, long revision) - throws IOException { - TableSnapshot ret = new TableSnapshot(tableName, new HashMap(), revision); - return recordCall(ret, tableName, revision); - } - - @Override - public void keepAlive(Transaction transaction) throws IOException { - recordCall(null, transaction); - } - } - - @Test - public void testRevisionManagerProtocol() throws Throwable { - - Configuration conf = getHbaseConf(); - RevisionManager rm = RevisionManagerFactory.getOpenedRevisionManager( - RevisionManagerEndpointClient.class.getName(), conf); - - MockRM mockImpl = MockRM.INSTANCES.get(MockRM.DEFAULT_INSTANCE); - Assert.assertNotNull(mockImpl); - Assert.assertTrue(mockImpl.isOpen); - - Transaction t = new Transaction("t1", Arrays.asList("f1", "f2"), 0, 0); - MockRM.Invocation call = new MockRM.Invocation("keepAlive", null, t); - rm.keepAlive(t); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - t = new Transaction("t2", Arrays.asList("f21", "f22"), 0, 0); - call = new MockRM.Invocation("beginWriteTransaction", null, t.getTableName(), t.getColumnFamilies()); - call.ret = rm.beginWriteTransaction(t.getTableName(), t.getColumnFamilies()); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - call = new MockRM.Invocation("createSnapshot", null, "t3", 1L); - call.ret = rm.createSnapshot("t3", 1); - Assert.assertEquals(call.methodName, call, mockImpl.lastCall); - - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java deleted file mode 100644 index e423f6508133..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestThriftSerialization.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevision; -import org.apache.hcatalog.hbase.snapshot.transaction.thrift.StoreFamilyRevisionList; -import org.junit.Test; - -public class TestThriftSerialization { - - @Test - public void testLightWeightTransaction() { - StoreFamilyRevision trxn = new StoreFamilyRevision(0, 1000); - try { - - byte[] data = ZKUtil.serialize(trxn); - StoreFamilyRevision newWtx = new StoreFamilyRevision(); - ZKUtil.deserialize(newWtx, data); - - assertTrue(newWtx.getRevision() == trxn.getRevision()); - assertTrue(newWtx.getTimestamp() == trxn.getTimestamp()); - - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Test - public void testWriteTransactionList() { - List txnList = new ArrayList(); - long version; - long timestamp; - for (int i = 0; i < 10; i++) { - version = i; - timestamp = 1000 + i; - StoreFamilyRevision wtx = new StoreFamilyRevision(version, timestamp); - txnList.add(wtx); - } - - StoreFamilyRevisionList wList = new StoreFamilyRevisionList(txnList); - - try { - byte[] data = ZKUtil.serialize(wList); - StoreFamilyRevisionList newList = new StoreFamilyRevisionList(); - ZKUtil.deserialize(newList, data); - assertTrue(newList.getRevisionListSize() == wList.getRevisionListSize()); - - Iterator itr = newList.getRevisionListIterator(); - int i = 0; - while (itr.hasNext()) { - StoreFamilyRevision txn = itr.next(); - assertTrue(txn.getRevision() == i); - assertTrue(txn.getTimestamp() == (i + 1000)); - i++; - } - - } catch (IOException e) { - e.printStackTrace(); - } - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java deleted file mode 100644 index cb0e7cf3c159..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.net.URI; -import java.util.Map; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hcatalog.cli.HCatDriver; -import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hcatalog.hbase.SkeletonHBaseTest; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.Stat; -import org.junit.Test; - - -public class TestZNodeSetUp extends SkeletonHBaseTest { - - private static HiveConf hcatConf; - private static HCatDriver hcatDriver; - - public void Initialize() throws Exception { - - hcatConf = getHiveConf(); - hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, - HCatSemanticAnalyzer.class.getName()); - URI fsuri = getFileSystem().getUri(); - Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(), - getTestDir()); - hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString()); - hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString()); - - //Add hbase properties - - for (Map.Entry el : getHbaseConf()) { - if (el.getKey().startsWith("hbase.")) { - hcatConf.set(el.getKey(), el.getValue()); - } - } - HBaseConfiguration.merge(hcatConf, - RevisionManagerConfiguration.create()); - hcatConf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base"); - SessionState.start(new CliSessionState(hcatConf)); - hcatDriver = new HCatDriver(); - - } - - @Test - public void testBasicZNodeCreation() throws Exception { - - Initialize(); - int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181); - String servers = getHbaseConf().get("hbase.zookeeper.quorum"); - String[] splits = servers.split(","); - StringBuffer sb = new StringBuffer(); - for (String split : splits) { - sb.append(split); - sb.append(':'); - sb.append(port); - } - - hcatDriver.run("drop table test_table"); - CommandProcessorResponse response = hcatDriver - .run("create table test_table(key int, value string) STORED BY " + - "'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'" - + "TBLPROPERTIES ('hbase.columns.mapping'=':key,cf1:val')"); - - assertEquals(0, response.getResponseCode()); - - HBaseAdmin hAdmin = new HBaseAdmin(getHbaseConf()); - boolean doesTableExist = hAdmin.tableExists("test_table"); - assertTrue(doesTableExist); - - - ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base"); - ZooKeeper zk = zkutil.getSession(); - String tablePath = PathUtil.getTxnDataPath("/rm_base", "test_table"); - Stat tempTwo = zk.exists(tablePath, false); - assertTrue(tempTwo != null); - - String cfPath = PathUtil.getTxnDataPath("/rm_base", "test_table") + "/cf1"; - Stat tempThree = zk.exists(cfPath, false); - assertTrue(tempThree != null); - - hcatDriver.run("drop table test_table"); - - System.out.println("Table path : " + tablePath); - Stat tempFour = zk.exists(tablePath, false); - assertTrue(tempFour == null); - - } - -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/WriteLockTest.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/WriteLockTest.java deleted file mode 100644 index df1219b20ba9..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/WriteLockTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.test.ClientBase; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; - -/** - * test for writelock - * This class is taken from the zookeeper 3.4.0 as-is to test the zookeeper lock - * Recipe with a change in the package name. - */ -public class WriteLockTest extends ClientBase { - protected int sessionTimeout = 10 * 1000; - protected String dir = "/" + getClass().getName(); - protected WriteLock[] nodes; - protected CountDownLatch latch = new CountDownLatch(1); - private boolean restartServer = true; - private boolean workAroundClosingLastZNodeFails = true; - private boolean killLeader = true; - - @Test - public void testRun() throws Exception { - runTest(3); - } - - class LockCallback implements LockListener { - public void lockAcquired() { - latch.countDown(); - } - - public void lockReleased() { - - } - - } - - protected void runTest(int count) throws Exception { - nodes = new WriteLock[count]; - for (int i = 0; i < count; i++) { - ZooKeeper keeper = createClient(); - WriteLock leader = new WriteLock(keeper, dir, null); - leader.setLockListener(new LockCallback()); - nodes[i] = leader; - - leader.lock(); - } - - // lets wait for any previous leaders to die and one of our new - // nodes to become the new leader - latch.await(30, TimeUnit.SECONDS); - - WriteLock first = nodes[0]; - dumpNodes(count); - - // lets assert that the first election is the leader - Assert.assertTrue("The first znode should be the leader " + first.getId(), first.isOwner()); - - for (int i = 1; i < count; i++) { - WriteLock node = nodes[i]; - Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); - } - - if (count > 1) { - if (killLeader) { - System.out.println("Now killing the leader"); - // now lets kill the leader - latch = new CountDownLatch(1); - first.unlock(); - latch.await(30, TimeUnit.SECONDS); - //Thread.sleep(10000); - WriteLock second = nodes[1]; - dumpNodes(count); - // lets assert that the first election is the leader - Assert.assertTrue("The second znode should be the leader " + second.getId(), second.isOwner()); - - for (int i = 2; i < count; i++) { - WriteLock node = nodes[i]; - Assert.assertFalse("Node should not be the leader " + node.getId(), node.isOwner()); - } - } - - - if (restartServer) { - // now lets stop the server - System.out.println("Now stopping the server"); - stopServer(); - Thread.sleep(10000); - - // TODO lets assert that we are no longer the leader - dumpNodes(count); - - System.out.println("Starting the server"); - startServer(); - Thread.sleep(10000); - - for (int i = 0; i < count - 1; i++) { - System.out.println("Calling acquire for node: " + i); - nodes[i].lock(); - } - dumpNodes(count); - System.out.println("Now closing down..."); - } - } - } - - protected void dumpNodes(int count) { - for (int i = 0; i < count; i++) { - WriteLock node = nodes[i]; - System.out.println("node: " + i + " id: " + - node.getId() + " is leader: " + node.isOwner()); - } - } - - @After - public void tearDown() throws Exception { - if (nodes != null) { - for (int i = 0; i < nodes.length; i++) { - WriteLock node = nodes[i]; - if (node != null) { - System.out.println("Closing node: " + i); - node.close(); - if (workAroundClosingLastZNodeFails && i == nodes.length - 1) { - System.out.println("Not closing zookeeper: " + i + " due to bug!"); - } else { - System.out.println("Closing zookeeper: " + i); - node.getZookeeper().close(); - System.out.println("Closed zookeeper: " + i); - } - } - } - } - System.out.println("Now lets stop the server"); - super.tearDown(); - - } -} diff --git a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/ZNodeNameTest.java b/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/ZNodeNameTest.java deleted file mode 100644 index 899f5124afe7..000000000000 --- a/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/lock/ZNodeNameTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hcatalog.hbase.snapshot.lock; - -import junit.framework.TestCase; - -import java.util.SortedSet; -import java.util.TreeSet; - -import org.junit.Test; - -/** - * test for znodenames. This class is taken as-is from zookeeper lock recipe test. - * The package name has been changed. - */ -public class ZNodeNameTest extends TestCase { - @Test - public void testOrderWithSamePrefix() throws Exception { - String[] names = { "x-3", "x-5", "x-11", "x-1" }; - String[] expected = { "x-1", "x-3", "x-5", "x-11" }; - assertOrderedNodeNames(names, expected); - } - @Test - public void testOrderWithDifferentPrefixes() throws Exception { - String[] names = { "r-3", "r-2", "r-1", "w-2", "w-1" }; - String[] expected = { "r-1", "r-2", "r-3", "w-1", "w-2" }; - assertOrderedNodeNames(names, expected); - } - - protected void assertOrderedNodeNames(String[] names, String[] expected) { - int size = names.length; - assertEquals("The two arrays should be the same size!", names.length, expected.length); - SortedSet nodeNames = new TreeSet(); - for (String name : names) { - nodeNames.add(new ZNodeName(name)); - } - - int index = 0; - for (ZNodeName nodeName : nodeNames) { - String name = nodeName.getName(); - assertEquals("Node " + index, expected[index++], name); - } - } - -} diff --git a/ivy/libraries.properties b/ivy/libraries.properties index 6e25064b0167..9d464da5149c 100644 --- a/ivy/libraries.properties +++ b/ivy/libraries.properties @@ -44,7 +44,7 @@ commons-logging-api.version=1.0.4 commons-pool.version=1.5.4 derby.version=10.4.2.0 guava.version=11.0.2 -hbase.version=0.94.6.1 +hbase.version=0.94.5.6.1302190003 jackson.version=1.8.8 javaewah.version=0.3.2 jdo-api.version=2.3-ec diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 0bf6add0927f..a7b78867fff9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -926,8 +927,20 @@ public void augmentPlan() { public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { if (hiveOutputFormat == null) { - try { - hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance(); + try { + if (getConf().getTableInfo().getJobProperties() != null) { + //Setting only for Storage Handler + if (getConf().getTableInfo().getJobProperties().get("StorageHandlerOF") != null) { + job.set("StorageHandlerOF",getConf().getTableInfo().getJobProperties().get("StorageHandlerOF")); + hiveOutputFormat = ReflectionUtils.newInstance(conf.getTableInfo().getOutputFileFormatClass(),job); + } + else { + hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance(); + } + } + else { + hiveOutputFormat = conf.getTableInfo().getOutputFileFormatClass().newInstance(); + } } catch (Exception ex) { throw new IOException(ex); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java index 1b99781f9835..5bb73525bb1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -51,6 +52,7 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.ReflectionUtils; /** * An util class for various Hive file format tasks. @@ -70,10 +72,16 @@ public final class HiveFileFormatUtils { SequenceFileOutputFormat.class, HiveSequenceFileOutputFormat.class); } + static { + realoutputFormatMap = new HashMap< String, String >(); + } + @SuppressWarnings("unchecked") private static Map, Class> outputFormatSubstituteMap; + private static Map realoutputFormatMap; + /** * register a substitute. * @@ -93,15 +101,47 @@ public static synchronized void registerOutputFormatSubstitute( */ @SuppressWarnings("unchecked") public static synchronized Class getOutputFormatSubstitute( - Class origin) { + Class origin, boolean storagehandlerflag) { if (HiveOutputFormat.class.isAssignableFrom(origin)) { return (Class) origin; } Class result = outputFormatSubstituteMap .get(origin); + //register this output format into the map for the first time + if ((storagehandlerflag == true) && (result == null)) { + HiveFileFormatUtils.setRealOutputFormatClassName(HivePassThroughOutputFormat.HIVE_PASSTHROUGH_OF_CLASSNAME,origin.getName()); + result = HivePassThroughOutputFormat.class; + HiveFileFormatUtils.registerOutputFormatSubstitute((Class) origin,HivePassThroughOutputFormat.class); + } return result; } + /** + * get a RealOutputFormatClassName corresponding to the HivePassThroughOutputFormat + */ + @SuppressWarnings("unchecked") + public static synchronized String getRealOutputFormatClassName( + String origin) { + if (origin != null) { + return realoutputFormatMap.get(origin); + } + return null; + } + + /** + * set a RealOutputFormatClassName corresponding to the HivePassThroughOutputFormat + */ + public static synchronized void setRealOutputFormatClassName( + String origin, String destination) { + if ((origin != null) && (destination != null )){ + realoutputFormatMap.put(origin, destination); + } + else { + return; + } + } + + /** * get the final output path of a given FileOutputFormat. * @@ -215,9 +255,21 @@ private static boolean checkTextInputFormat(FileSystem fs, HiveConf conf, public static RecordWriter getHiveRecordWriter(JobConf jc, TableDesc tableInfo, Class outputClass, FileSinkDesc conf, Path outPath, Reporter reporter) throws HiveException { + boolean storagehandlerofhivepassthru = false; + HiveOutputFormat hiveOutputFormat; try { - HiveOutputFormat hiveOutputFormat = tableInfo - .getOutputFileFormatClass().newInstance(); + if (tableInfo.getJobProperties() != null) { + if (tableInfo.getJobProperties().get("StorageHandlerOF") != null) { + jc.set("StorageHandlerOF",tableInfo.getJobProperties().get("StorageHandlerOF")); + storagehandlerofhivepassthru = true; + } + } + if (storagehandlerofhivepassthru) { + hiveOutputFormat = ReflectionUtils.newInstance(tableInfo.getOutputFileFormatClass(),jc); + } + else { + hiveOutputFormat = tableInfo.getOutputFileFormatClass().newInstance(); + } boolean isCompressed = conf.getCompressed(); JobConf jc_output = jc; if (isCompressed) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java new file mode 100644 index 000000000000..9c50e05f539a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughOutputFormat.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * This pass through class is used to wrap OutputFormat implementations such that new OutputFormats not derived from + * HiveOutputFormat gets through the checker + */ + +public class HivePassThroughOutputFormat implements Configurable, HiveOutputFormat{ + + private OutputFormat, ? super Writable> actualOutputFormat; + private String actualOutputFormatClass = ""; + private Configuration conf; + private boolean initialized; + public static final String HIVE_PASSTHROUGH_OF_CLASS = "HivePassThroughOutputFormat.class"; + public static final String HIVE_PASSTHROUGH_OF_CLASSNAME = "org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat"; + + public HivePassThroughOutputFormat() { + //construct this class through ReflectionUtils from FileSinkOperator + this.actualOutputFormat = null; + this.initialized = false; + } + + private void CreateActualOF() throws IOException { + Class cls; + try { + int e; + if (actualOutputFormatClass != null) + { + cls = + (Class) Class.forName(actualOutputFormatClass, true, JavaUtils.getClassLoader()); + } else { + throw new RuntimeException("Null pointer detected in actualOutputFormatClass"); + } + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + OutputFormat, ? super Writable> actualOF = (OutputFormat) ReflectionUtils.newInstance(cls, this.getConf()); + this.actualOutputFormat = actualOF; + } + + @Override + public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException { + if (this.initialized == false) { + CreateActualOF(); + this.initialized = true; + } + this.actualOutputFormat.checkOutputSpecs(ignored, job); + } + + @Override + public org.apache.hadoop.mapred.RecordWriter getRecordWriter(FileSystem ignored, JobConf job, + String name, Progressable progress) throws IOException { + if (this.initialized == false) { + CreateActualOF(); + this.initialized = true; + } + return (RecordWriter) this.actualOutputFormat.getRecordWriter(ignored, job, name, progress); + } + + @Override + public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( + JobConf jc, Path finalOutPath, Class valueClass, boolean isCompressed, + Properties tableProperties, Progressable progress) throws IOException { + if (this.initialized == false) { + CreateActualOF(); + } + if (this.actualOutputFormat instanceof HiveOutputFormat) { + return ((HiveOutputFormat) this.actualOutputFormat).getHiveRecordWriter(jc, finalOutPath, valueClass, isCompressed, tableProperties, progress); + } + else { + FileSystem fs = finalOutPath.getFileSystem(jc); + HivePassThroughRecordWriter hivepassthroughrecordwriter = new HivePassThroughRecordWriter(this.actualOutputFormat.getRecordWriter(fs, jc, null, progress)); + return hivepassthroughrecordwriter; + } + } + + public Configuration getConf() { + return conf; + } + + public void setConf(Configuration config) { + if (config.get("StorageHandlerOF") != null) { + actualOutputFormatClass = config.get("StorageHandlerOF"); + } + this.conf = config; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java new file mode 100644 index 000000000000..90f592b4415b --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + + +public class HivePassThroughRecordWriter , V extends Writable> +implements RecordWriter { + + private final org.apache.hadoop.mapred.RecordWriter mWriter; + + public HivePassThroughRecordWriter(org.apache.hadoop.mapred.RecordWriter writer) { + this.mWriter = writer; + } + + @SuppressWarnings("unchecked") + public void write(Writable r) throws IOException { + mWriter.write(null, (V) r); + } + + public void close(boolean abort) throws IOException { + //close with null reporter + mWriter.close(null); + } +} + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index b678669cbe66..bbb62487c312 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -301,7 +301,7 @@ public void setInputFormatClass(Class inputFormatClass) { public void setOutputFormatClass(Class outputFormatClass) { this.outputFormatClass = outputFormatClass; tPartition.getSd().setOutputFormat(HiveFileFormatUtils - .getOutputFormatSubstitute(outputFormatClass).toString()); + .getOutputFormatSubstitute(outputFormatClass, false).toString()); } final public Class getInputFormatClass() @@ -339,7 +339,7 @@ final public Class getOutputFormatClass() JavaUtils.getClassLoader())); // Replace FileOutputFormat for backward compatibility if (!HiveOutputFormat.class.isAssignableFrom(c)) { - outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c); + outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c,false); } else { outputFormatClass = (Class)c; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 0920a9d7d28a..6fb462af7d88 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -311,7 +311,7 @@ final public Class getInputFormatClass() { final public Class getOutputFormatClass() { // Replace FileOutputFormat for backward compatibility - + boolean storagehandler = false; if (outputFormatClass == null) { try { String className = tTable.getSd().getOutputFormat(); @@ -326,7 +326,13 @@ final public Class getOutputFormatClass() { JavaUtils.getClassLoader()); } if (!HiveOutputFormat.class.isAssignableFrom(c)) { - outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c); + if (getStorageHandler() != null) { + storagehandler = true; + } + else { + storagehandler = false; + } + outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c,storagehandler); } else { outputFormatClass = (Class)c; } @@ -669,7 +675,7 @@ public void setOutputFormatClass(String name) throws HiveException { try { Class origin = Class.forName(name, true, JavaUtils.getClassLoader()); setOutputFormatClass(HiveFileFormatUtils - .getOutputFormatSubstitute(origin)); + .getOutputFormatSubstitute(origin,false)); } catch (ClassNotFoundException e) { throw new HiveException("Class not found: " + name, e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 25d04e1de945..83eb68664a42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -403,7 +403,7 @@ public void validate() Class origin = Class.forName(this.getOutputFormat(), true, JavaUtils.getClassLoader()); Class replaced = HiveFileFormatUtils - .getOutputFormatSubstitute(origin); + .getOutputFormatSubstitute(origin,false); if (replaced == null) { throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE .getMsg()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java index f0b16e4af876..a2e71dec9015 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java @@ -74,7 +74,7 @@ public PartitionDesc(final TableDesc table, this.inputFileFormatClass = inputFileFormatClass; if (outputFormat != null) { outputFileFormatClass = HiveFileFormatUtils - .getOutputFormatSubstitute(outputFormat); + .getOutputFormatSubstitute(outputFormat,false); } if (serdeClassName != null) { this.serdeClassName = serdeClassName; @@ -177,7 +177,7 @@ public Class getOutputFileFormatClass() { public void setOutputFileFormatClass(final Class outputFileFormatClass) { this.outputFileFormatClass = HiveFileFormatUtils - .getOutputFormatSubstitute(outputFileFormatClass); + .getOutputFormatSubstitute(outputFileFormatClass,false); } @Explain(displayName = "properties", normalExplain = false) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 771a5ac0965a..7b4a3868b5fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.plan; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -37,7 +38,9 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -722,6 +725,10 @@ private static void configureJobPropertiesForStorageHandler(boolean input, // for native tables, leave it null to avoid cluttering up // plans. if (!jobProperties.isEmpty()) { + if (tableDesc.getOutputFileFormatClass().getName() == HivePassThroughOutputFormat.HIVE_PASSTHROUGH_OF_CLASSNAME) { + // get the real output format when we register this for the table + jobProperties.put("StorageHandlerOF",HiveFileFormatUtils.getRealOutputFormatClassName(HivePassThroughOutputFormat.HIVE_PASSTHROUGH_OF_CLASSNAME)); + } tableDesc.setJobProperties(jobProperties); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java index a34e89e38453..bc8f048f7f7e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.mapred.InputFormat; @@ -51,7 +52,7 @@ public TableDesc(final Class serdeClass, deserializerClass = serdeClass; this.inputFileFormatClass = inputFileFormatClass; outputFileFormatClass = HiveFileFormatUtils - .getOutputFormatSubstitute(class1); + .getOutputFormatSubstitute(class1, false); this.properties = properties; serdeClassName = properties .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB); @@ -91,7 +92,7 @@ public Class getOutputFileFormatClass() { public void setOutputFileFormatClass(final Class outputFileFormatClass) { this.outputFileFormatClass = HiveFileFormatUtils - .getOutputFormatSubstitute(outputFileFormatClass); + .getOutputFormatSubstitute(outputFileFormatClass, false); } @Explain(displayName = "properties", normalExplain = false) @@ -141,7 +142,12 @@ public String getInputFileFormatClassName() { @Explain(displayName = "output format") public String getOutputFileFormatClassName() { - return getOutputFileFormatClass().getName(); + if (getOutputFileFormatClass().getName() == HivePassThroughOutputFormat.HIVE_PASSTHROUGH_OF_CLASSNAME) { + return HiveFileFormatUtils.getRealOutputFormatClassName(HivePassThroughOutputFormat.HIVE_PASSTHROUGH_OF_CLASSNAME); + } + else { + return getOutputFileFormatClass().getName(); + } } public boolean isNonNative() {