Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

SHDP-109

remove use of HTable in favor of HTableInterface
  • Loading branch information...
commit ee0aad4af7c4e3cd4b7abba82521ef27270603de 1 parent 1ef5ec2
Costin Leau authored
4  gradle.properties
@@ -8,10 +8,10 @@ slf4jVersion = 1.6.6
8 8
 springVersion = 3.0.7.RELEASE
9 9
 springBatchVersion = 2.1.9.RELEASE
10 10
 springIntVersion = 2.1.2.RELEASE
11  
-hadoopVersion = 1.1.0
  11
+hadoopVersion = 1.0.4
12 12
 hiveVersion = 0.9.0
13 13
 pigVersion = 0.10.0
14  
-hbaseVersion = 0.92.1
  14
+hbaseVersion = 0.94.1
15 15
 jacksonVersion = 1.8.8
16 16
 commonsioVersion = 2.1
17 17
 cglibVersion = 2.2.2
2  src/main/java/org/springframework/data/hadoop/hbase/HbaseInterceptor.java
@@ -53,7 +53,7 @@ public Object invoke(MethodInvocation methodInvocation) throws Throwable {
53 53
 		for (String tableName : tableNames) {
54 54
 			if (!HbaseSynchronizationManager.hasResource(tableName)) {
55 55
 				boundTables.add(tableName);
56  
-				HTable table = HbaseUtils.getHTable(getTableFactory(), getCharset(), getConfiguration(), tableName);
  56
+				HTableInterface table = HbaseUtils.getHTable(getTableFactory(), getCharset(), getConfiguration(), tableName);
57 57
 				HbaseSynchronizationManager.bindResource(tableName, table);
58 58
 			}
59 59
 		}
32  src/main/java/org/springframework/data/hadoop/hbase/HbaseTemplate.java
@@ -21,6 +21,7 @@
21 21
 import org.apache.hadoop.conf.Configuration;
22 22
 import org.apache.hadoop.hbase.client.Get;
23 23
 import org.apache.hadoop.hbase.client.HTable;
  24
+import org.apache.hadoop.hbase.client.HTableInterface;
24 25
 import org.apache.hadoop.hbase.client.Result;
25 26
 import org.apache.hadoop.hbase.client.ResultScanner;
26 27
 import org.apache.hadoop.hbase.client.Scan;
@@ -50,7 +51,7 @@ public HbaseTemplate(Configuration configuration) {
50 51
 		Assert.notNull(action, "Callback object must not be null");
51 52
 		Assert.notNull(tableName, "No table specified");
52 53
 
53  
-		HTable table = getTable(tableName);
  54
+		HTableInterface table = getTable(tableName);
54 55
 
55 56
 		try {
56 57
 			boolean previousFlushSetting = applyFlushSetting(table);
@@ -70,26 +71,34 @@ public HbaseTemplate(Configuration configuration) {
70 71
 		}
71 72
 	}
72 73
 
73  
-	private HTable getTable(String tableName) {
  74
+	private HTableInterface getTable(String tableName) {
74 75
 		return HbaseUtils.getHTable(getTableFactory(), getCharset(), getConfiguration(), tableName);
75 76
 	}
76 77
 
77  
-	private void releaseTable(String tableName, HTable table) {
  78
+	private void releaseTable(String tableName, HTableInterface table) {
78 79
 		HbaseUtils.releaseTable(tableName, table);
79 80
 	}
80 81
 
81  
-	private boolean applyFlushSetting(HTable table) {
  82
+	private boolean applyFlushSetting(HTableInterface table) {
82 83
 		boolean autoFlush = table.isAutoFlush();
83  
-		table.setAutoFlush(this.autoFlush);
  84
+		if (table instanceof HTable) {
  85
+			((HTable) table).setAutoFlush(this.autoFlush);
  86
+		}
84 87
 		return autoFlush;
85 88
 	}
86 89
 
87  
-	private void flushIfNecessary(HTable table, boolean oldFlush) throws IOException {
  90
+	private void restoreFlushSettings(HTableInterface table, boolean oldFlush) {
  91
+		if (table instanceof HTable) {
  92
+			if (table.isAutoFlush() != oldFlush) {
  93
+				((HTable) table).setAutoFlush(oldFlush);
  94
+			}
  95
+		}
  96
+	}
  97
+
  98
+	private void flushIfNecessary(HTableInterface table, boolean oldFlush) throws IOException {
88 99
 		// TODO: check whether we can consider or not a table scope
89 100
 		table.flushCommits();
90  
-		if (table.isAutoFlush() != oldFlush) {
91  
-			table.setAutoFlush(oldFlush);
92  
-		}
  101
+		restoreFlushSettings(table, oldFlush);
93 102
 	}
94 103
 
95 104
 	public DataAccessException convertHbaseAccessException(Exception ex) {
@@ -114,7 +123,7 @@ public DataAccessException convertHbaseAccessException(Exception ex) {
114 123
 	public <T> T find(String tableName, final Scan scan, final ResultsExtractor<T> action) {
115 124
 		return execute(tableName, new TableCallback<T>() {
116 125
 			@Override
117  
-			public T doInTable(HTable htable) throws Throwable {
  126
+			public T doInTable(HTableInterface htable) throws Throwable {
118 127
 				ResultScanner scanner = htable.getScanner(scan);
119 128
 				try {
120 129
 					return action.extractData(scanner);
@@ -158,7 +167,7 @@ public T doInTable(HTable htable) throws Throwable {
158 167
 	public <T> T get(String tableName, final String rowName, final String familyName, final String qualifier, final RowMapper<T> mapper) {
159 168
 		return execute(tableName, new TableCallback<T>() {
160 169
 			@Override
161  
-			public T doInTable(HTable htable) throws Throwable {
  170
+			public T doInTable(HTableInterface htable) throws Throwable {
162 171
 				Get get = new Get(rowName.getBytes(getCharset()));
163 172
 				if (familyName != null) {
164 173
 					byte[] family = familyName.getBytes(getCharset());
@@ -184,5 +193,4 @@ public T doInTable(HTable htable) throws Throwable {
184 193
 	public void setAutoFlush(boolean autoFlush) {
185 194
 		this.autoFlush = autoFlush;
186 195
 	}
187  
-
188 196
 }
11  src/main/java/org/springframework/data/hadoop/hbase/HbaseUtils.java
@@ -23,7 +23,6 @@
23 23
 import org.apache.hadoop.hbase.client.HTableInterface;
24 24
 import org.apache.hadoop.hbase.client.HTableInterfaceFactory;
25 25
 import org.springframework.dao.DataAccessException;
26  
-import org.springframework.util.Assert;
27 26
 import org.springframework.util.StringUtils;
28 27
 
29 28
 /**
@@ -50,7 +49,7 @@ public static DataAccessException convertHbaseException(Exception ex) {
50 49
 	 * @param tableName table name
51 50
 	 * @return table instance
52 51
 	 */
53  
-	public static HTable getHTable(Configuration configuration, String tableName) {
  52
+	public static HTableInterface getHTable(Configuration configuration, String tableName) {
54 53
 		return getHTable(null, getCharset(null), configuration, tableName);
55 54
 	}
56 55
 
@@ -63,17 +62,15 @@ public static HTable getHTable(Configuration configuration, String tableName) {
63 62
 	 * @param tableName table name
64 63
 	 * @return table instance
65 64
 	 */
66  
-	public static HTable getHTable(HTableInterfaceFactory tableFactory, Charset charset, Configuration configuration, String tableName) {
  65
+	public static HTableInterface getHTable(HTableInterfaceFactory tableFactory, Charset charset, Configuration configuration, String tableName) {
67 66
 		if (HbaseSynchronizationManager.hasResource(tableName)) {
68 67
 			return (HTable) HbaseSynchronizationManager.getResource(tableName);
69 68
 		}
70 69
 
71  
-		HTable t = null;
  70
+		HTableInterface t = null;
72 71
 		try {
73 72
 			if (tableFactory != null) {
74  
-				HTableInterface table = tableFactory.createHTableInterface(configuration, tableName.getBytes(charset));
75  
-				Assert.isInstanceOf(HTable.class, table, "The table factory needs to create HTable instances");
76  
-				t = (HTable) table;
  73
+				t = tableFactory.createHTableInterface(configuration, tableName.getBytes(charset));
77 74
 			}
78 75
 			else {
79 76
 				t = new HTable(configuration, tableName.getBytes(charset));
4  src/main/java/org/springframework/data/hadoop/hbase/TableCallback.java
@@ -15,7 +15,7 @@
15 15
  */
16 16
 package org.springframework.data.hadoop.hbase;
17 17
 
18  
-import org.apache.hadoop.hbase.client.HTable;
  18
+import org.apache.hadoop.hbase.client.HTableInterface;
19 19
 
20 20
 /**
21 21
  * Callback interface for Hbase code. To be used with {@link HbaseTemplate}'s execution methods, often as anonymous classes within a method implementation without
@@ -32,5 +32,5 @@
32 32
 	 * @return a result object, or null if none
33 33
 	 * @throws Throwable thrown by the Hbase API
34 34
 	 */
35  
-	T doInTable(HTable table) throws Throwable;
  35
+	T doInTable(HTableInterface table) throws Throwable;
36 36
 }
5  src/test/java/org/springframework/data/hadoop/hbase/BasicHBaseTest.java
@@ -23,6 +23,7 @@
23 23
 import org.apache.hadoop.hbase.client.Get;
24 24
 import org.apache.hadoop.hbase.client.HBaseAdmin;
25 25
 import org.apache.hadoop.hbase.client.HTable;
  26
+import org.apache.hadoop.hbase.client.HTableInterface;
26 27
 import org.apache.hadoop.hbase.client.Put;
27 28
 import org.apache.hadoop.hbase.client.Result;
28 29
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -111,11 +112,11 @@ public void testHBaseConnection() throws Exception {
111 112
 	public void testTemplate() throws Exception {
112 113
 		assertTrue(HbaseSynchronizationManager.getTableNames().isEmpty());
113 114
 
114  
-		final HTable t = HbaseUtils.getHTable(config, tableName);
  115
+		final HTableInterface t = HbaseUtils.getHTable(config, tableName);
115 116
 
116 117
 		template.execute(tableName, new TableCallback<Object>() {
117 118
 			@Override
118  
-			public Object doInTable(HTable table) throws Throwable {
  119
+			public Object doInTable(HTableInterface table) throws Throwable {
119 120
 				assertSame(t, table);
120 121
 				Put p = new Put(Bytes.toBytes(rowName));
121 122
 				p.add(Bytes.toBytes(columnName), Bytes.toBytes(qualifier), Bytes.toBytes(value));
6  src/test/resources/org/springframework/data/hadoop/hadoop-ctx.xml
@@ -33,13 +33,13 @@
33 33
 	<!--  default id is 'hadoopConfiguration' /   -->
34 34
 	<!--  properties-location="s3.properties" -->
35 35
 	<!--  -->
36  
-	<hdp:configuration register-url-handler="false" >
  36
+	<hdp:configuration register-url-handler="false">
37 37
 		fs.default.name=${hd.fs}
38  
-		mapred.job.tracker=local
  38
+		mapred.job.tracker=${jt}
39 39
 		cfg=main
40 40
 	</hdp:configuration>
41 41
 
42  
-	<hdp:file-system user="hadoop" />
  42
+	<hdp:file-system />
43 43
 	
44 44
 	<!-- 
45 45
 	<bean id="fs-config" class="org.springframework.data.hadoop.configuration.ConfigurationFactoryBean">
13  src/test/resources/test.properties
@@ -4,13 +4,9 @@
4 4
 
5 5
 # Amazon EMR
6 6
 #hive.port=10003
7  
-#hd.fs=s3n://work-emr/tmp/
  7
+hd.fs=s3n://work-emr/tmp
8 8
 #jt=localhost:20001
9  
-#hd.fs=hdfs://ec2-54-242-45-228.compute-1.amazonaws.com:9000/
10  
-#jt=ec2-54-242-45-228.compute-1.amazonaws.com:9001
11  
-#jt=54.242.45.228:9001
12  
-#jt=localhost:20001
13  
-
  9
+# jt=10.80.205.79:9001
14 10
 
15 11
 # Apache Whirr - EC2
16 12
 #hd.fs=hdfs://xxx.amazonaws.com:8020
@@ -18,13 +14,12 @@
18 14
 
19 15
 # Default - Vanilla Installs
20 16
 hd.fs=hdfs://localhost:9000
21  
-jt=localhost:9001
22  
-#jt=
  17
+jt=local
23 18
 
24 19
 hive.host=localhost
25 20
 hive.port=12345
26 21
 hive.url=jdbc:hive://${hive.host}:${hive.port}
27  
-hd.host=localhost
  22
+#hd.host=localhost
28 23
 
29 24
 #path.cat=bin${file.separator}stream-bin${file.separator}cat
30 25
 #path.wc=bin${file.separator}stream-bin${file.separator}wc

0 notes on commit ee0aad4

Please sign in to comment.
Something went wrong with that request. Please try again.