Skip to content

Commit

Permalink
KYLIN-892 compile with HDP 2.3, HBase 1.x
Browse files Browse the repository at this point in the history
  • Loading branch information
@liyang-kylin
liyang-kylin authored and Li, Yang committed Oct 23, 2015
1 parent 1318b73 commit 3623dd6
Show file tree
Hide file tree
Showing 8 changed files with 43 additions and 87 deletions.
17 changes: 7 additions & 10 deletions pom.xml
View file
Expand Up @@ -45,12 +45,13 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

<!-- Hadoop versions -->
<hadoop2.version>2.6.0</hadoop2.version>
<yarn.version>2.6.0</yarn.version>
<hadoop2.version>2.7.0</hadoop2.version>
<yarn.version>2.7.0</yarn.version>
<zookeeper.version>3.4.6</zookeeper.version>
<hive.version>0.14.0</hive.version>
<hive-hcatalog.version>0.14.0</hive-hcatalog.version>
<hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
<hive.version>1.1.0</hive.version>
<hive-hcatalog.version>1.1.0</hive-hcatalog.version>
<hbase-hadoop2.version>1.1.0</hbase-hadoop2.version>
<curator.version>2.7.1</curator.version>

<!-- Dependency versions -->
<antlr.version>3.4</antlr.version>
Expand Down Expand Up @@ -88,9 +89,6 @@
<!-- Calcite Version -->
<calcite.version>1.4.0-incubating</calcite.version>

<!-- Curator.version Version -->
<curator.version>2.6.0</curator.version>

<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
Expand Down Expand Up @@ -821,8 +819,7 @@
</configuration>
</plugin>

<!-- Override the parent assembly execution to customize the assembly
descriptor and final name. -->
<!-- Override the parent assembly execution to customize the assembly descriptor and final name. -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
Expand Down
10 changes: 8 additions & 2 deletions storage/src/main/java/org/apache/kylin/storage/hbase/RegionScannerAdapter.java
View file
Expand Up @@ -26,6 +26,7 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;

/**
* @author yangli9
Expand All @@ -50,7 +51,7 @@ public boolean next(List<Cell> results) throws IOException {
}

@Override
public boolean next(List<Cell> result, int limit) throws IOException {
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}

Expand All @@ -60,11 +61,16 @@ public boolean nextRaw(List<Cell> result) throws IOException {
}

@Override
public boolean nextRaw(List<Cell> result, int limit) throws IOException {
public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}

@Override
public int getBatch() {
return -1;
}

@Override
public void close() throws IOException {
scanner.close();
}
Expand Down
2 changes: 1 addition & 1 deletion storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
View file
Expand Up @@ -90,7 +90,7 @@ public void getRows(RpcController controller, IIProtos.IIRequest request, RpcCal
RegionScanner innerScanner = null;
HRegion region = null;
try {
region = env.getRegion();
region = (HRegion) env.getRegion();
innerScanner = region.getScanner(buildScan());
region.startRegionOperation();

Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
View file
Expand Up @@ -93,7 +93,7 @@ private RegionScanner doPostScannerObserver(final ObserverContext<RegionCoproces
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
HRegion region = ctxt.getEnvironment().getRegion();
HRegion region = (HRegion) ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
Expand Down
14 changes: 10 additions & 4 deletions ...src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
Expand Down Expand Up @@ -93,13 +94,18 @@ ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, Coproc
}

@Override
public int getBatch() {
return outerScanner.getBatch();
}

@Override
public boolean next(List<Cell> results) throws IOException {
return outerScanner.next(results);
}

@Override
public boolean next(List<Cell> result, int limit) throws IOException {
return outerScanner.next(result, limit);
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return outerScanner.next(result, scannerContext);
}

@Override
Expand All @@ -108,8 +114,8 @@ public boolean nextRaw(List<Cell> result) throws IOException {
}

@Override
public boolean nextRaw(List<Cell> result, int limit) throws IOException {
return outerScanner.nextRaw(result, limit);
public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return outerScanner.nextRaw(result, scannerContext);
}

@Override
Expand Down
10 changes: 8 additions & 2 deletions ...in/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverAggregationCache.java
View file
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
Expand Down Expand Up @@ -107,7 +108,7 @@ private void makeCells(Entry<AggrKey, MeasureAggregator[]> entry, List<Cell> res
}

@Override
public boolean next(List<Cell> result, int limit) throws IOException {
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}

Expand All @@ -117,10 +118,15 @@ public boolean nextRaw(List<Cell> result) throws IOException {
}

@Override
public boolean nextRaw(List<Cell> result, int limit) throws IOException {
public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}

@Override
public int getBatch() {
return innerScanner.getBatch();
}

@Override
public void close() throws IOException {
// AggregateRegionObserver.LOG.info("Kylin Scanner close()");
Expand Down
72 changes: 7 additions & 65 deletions ...java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
View file
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
Expand Down Expand Up @@ -223,102 +224,46 @@ public MockupRegionScanner(List<Cell> cellInputs) {
this.input = cellInputs;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
* .List)
*/
@Override
public boolean next(List<Cell> results) throws IOException {
return nextRaw(results);
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
* .List, int)
*/
@Override
public boolean next(List<Cell> result, int limit) throws IOException {
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}

/*
* (non-Javadoc)
*
* @see org.apache.hadoop.hbase.regionserver.InternalScanner#close()
*/
@Override
public void close() throws IOException {

}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#getRegionInfo()
*/
@Override
public HRegionInfo getRegionInfo() {
return null;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#isFilterDone()
*/
@Override
public boolean isFilterDone() throws IOException {
return false;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#reseek(byte[])
*/
@Override
public boolean reseek(byte[] row) throws IOException {
return false;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#getMaxResultSize()
*/
@Override
public long getMaxResultSize() {
return 0;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#getMvccReadPoint()
*/
@Override
public long getMvccReadPoint() {
return 0;
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
* .List)
*/
@Override
public boolean nextRaw(List<Cell> result) throws IOException {
if (i < input.size()) {
Expand All @@ -328,18 +273,15 @@ public boolean nextRaw(List<Cell> result) throws IOException {
return i < input.size();
}

/*
* (non-Javadoc)
*
* @see
* org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
* .List, int)
*/
@Override
public boolean nextRaw(List<Cell> result, int limit) throws IOException {
public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return nextRaw(result);
}

@Override
public int getBatch() {
return -1;
}
}

}
3 changes: 1 addition & 2 deletions storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
View file
Expand Up @@ -27,7 +27,6 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.service.HiveInterface;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;

Expand All @@ -47,7 +46,7 @@ public class HiveMiniClusterTest extends HiveJDBCClientTest {
public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse");
public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata");
public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp");
protected HiveInterface client;
//protected HiveInterface client;

protected MiniDFSCluster miniDFS;
protected MiniMRCluster miniMR;
Expand Down

0 comments on commit 3623dd6

Please sign in to comment.