/
HBaseTestUtil.java
130 lines (117 loc) · 4.38 KB
/
HBaseTestUtil.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.bigtop.itest.hbase.util;
import java.io.IOException;
import java.security.MessageDigest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.util.Bytes;
public class HBaseTestUtil {
public static int BLOCKSIZE = 64*1024;
public static String COMPRESSION =
Compression.Algorithm.NONE.getName();
private static String getTestPrefix() {
return String.valueOf(System.currentTimeMillis());
}
public static byte[] getTestTableName(String testName) {
return Bytes.toBytes(testName + "_" + getTestPrefix());
}
public static HTableDescriptor createTestTableDescriptor(String testName,
byte[] familyName) {
byte[] tableName = getTestTableName(testName);
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor(familyName));
return htd;
}
public static HBaseAdmin getAdmin()
throws MasterNotRunningException, ZooKeeperConnectionException {
return new HBaseAdmin(HBaseConfiguration.create());
}
public static FileSystem getClusterFileSystem() throws IOException {
return FileSystem.get(new Configuration());
}
public static Path getMROutputDir(String testName) throws IOException {
Path p = new Path(testName + "_" + getTestPrefix());
return p.makeQualified(getClusterFileSystem());
}
/**
* Create an HFile with the given number of rows between a given
* start key and end key.
*/
public static void createHFile(
Configuration conf,
FileSystem fs, Path path,
byte[] family, byte[] qualifier,
byte[] startKey, byte[] endKey, int numRows) throws IOException
{
HFile.Writer writer =
HFile.getWriterFactory(conf).createWriter(fs, path,
BLOCKSIZE, COMPRESSION,
KeyValue.KEY_COMPARATOR);
long now = System.currentTimeMillis();
try {
// subtract 2 since iterateOnSplits doesn't include boundary keys
for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows-2)) {
KeyValue kv = new KeyValue(key, family, qualifier, now, key);
writer.append(kv);
}
} finally {
writer.close();
}
}
/**
* Return the number of rows in the given table.
*/
public static int countRows(final HTable table) throws IOException {
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
int count = 0;
for (@SuppressWarnings("unused") Result res : results) {
count++;
}
results.close();
return count;
}
/**
* Return an md5 digest of the entire contents of a table.
*/
public static String checksumRows(final HTable table) throws Exception {
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
MessageDigest digest = MessageDigest.getInstance("MD5");
for (Result res : results) {
digest.update(res.getRow());
}
results.close();
return digest.toString();
}
}